cronwatch 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +20 -11
- data/lib/cronwatch/alerts/datadog.rb +1 -0
- data/lib/cronwatch/alerts/discord.rb +34 -2
- data/lib/cronwatch/alerts/email.rb +2 -0
- data/lib/cronwatch/alerts/honeybadger.rb +1 -0
- data/lib/cronwatch/alerts/postmark.rb +1 -0
- data/lib/cronwatch/alerts/provider.rb +2 -0
- data/lib/cronwatch/alerts/resend.rb +1 -0
- data/lib/cronwatch/alerts/rollbar.rb +1 -0
- data/lib/cronwatch/alerts/sentry.rb +1 -0
- data/lib/cronwatch/alerts/sigv4.rb +2 -0
- data/lib/cronwatch/alerts/slack.rb +1 -0
- data/lib/cronwatch/alerts/twilio.rb +1 -0
- data/lib/cronwatch/alerts/webhook.rb +18 -5
- data/lib/cronwatch/client.rb +275 -110
- data/lib/cronwatch/cron_pattern.rb +17 -8
- data/lib/cronwatch/deprecation.rb +18 -0
- data/lib/cronwatch/duration.rb +2 -0
- data/lib/cronwatch/environment.rb +109 -7
- data/lib/cronwatch/evaluate.rb +145 -10
- data/lib/cronwatch/flight.rb +6 -0
- data/lib/cronwatch/format.rb +1 -0
- data/lib/cronwatch/http.rb +21 -5
- data/lib/cronwatch/job.rb +7 -2
- data/lib/cronwatch/js.rb +74 -1
- data/lib/cronwatch/monitored.rb +6 -1
- data/lib/cronwatch/output.rb +28 -1
- data/lib/cronwatch/pg_cron.rb +29 -6
- data/lib/cronwatch/rails/active_job.rb +2 -2
- data/lib/cronwatch/schedule.rb +25 -1
- data/lib/cronwatch/scheduler.rb +79 -11
- data/lib/cronwatch/serialize.rb +1 -0
- data/lib/cronwatch/stats.rb +1 -0
- data/lib/cronwatch/stores/active_record.rb +37 -11
- data/lib/cronwatch/ticker.rb +14 -0
- data/lib/cronwatch/triage/anthropic.rb +2 -0
- data/lib/cronwatch/types.rb +199 -27
- data/lib/cronwatch/version.rb +1 -1
- data/lib/cronwatch/walker.rb +2 -0
- data/lib/cronwatch/web/app.rb +114 -37
- data/lib/cronwatch/web/html.rb +9 -5
- data/lib/cronwatch/web/origin.rb +2 -0
- data/lib/cronwatch/web/pwa.rb +2 -0
- data/lib/cronwatch/web/timeline.rb +2 -0
- data/lib/cronwatch/web.rb +1 -2
- data/lib/cronwatch/zone.rb +2 -0
- data/lib/cronwatch.rb +2 -0
- data/lib/generators/cronwatch/install/install_generator.rb +52 -8
- metadata +2 -1
data/lib/cronwatch/types.rb
CHANGED
|
@@ -7,6 +7,8 @@ module Cronwatch
|
|
|
7
7
|
# Ruby names are snake_case symbols; everything that leaves the process
|
|
8
8
|
# (store rows, JSON, webhook bodies) uses the SDK's camelCase names and
|
|
9
9
|
# string values. Each type's to_h is that JSON shape, and from_h reads it.
|
|
10
|
+
#
|
|
11
|
+
# @api private
|
|
10
12
|
module Naming
|
|
11
13
|
module_function
|
|
12
14
|
|
|
@@ -51,6 +53,8 @@ module Cronwatch
|
|
|
51
53
|
end
|
|
52
54
|
|
|
53
55
|
# Shared by the structs below.
|
|
56
|
+
#
|
|
57
|
+
# @api private
|
|
54
58
|
module Serializable
|
|
55
59
|
def as_json(*)
|
|
56
60
|
to_h
|
|
@@ -78,23 +82,51 @@ module Cronwatch
|
|
|
78
82
|
|
|
79
83
|
FIELDS.each_key { |field| define_method(field) { @fields[field] } }
|
|
80
84
|
|
|
85
|
+
private_constant :BY_JSON, :FIELDS
|
|
86
|
+
|
|
81
87
|
def initialize(fields = {})
|
|
82
88
|
@fields = {}
|
|
83
89
|
fields.each { |k, v| @fields[k.is_a?(Symbol) ? k : k.to_s] = v }
|
|
84
90
|
@fields.freeze
|
|
91
|
+
@unreadable = false
|
|
85
92
|
freeze
|
|
86
93
|
end
|
|
87
94
|
|
|
88
|
-
|
|
95
|
+
# A stored definition that is not a JSON object (null, a string, a
|
|
96
|
+
# number, an array, or text that does not parse, written by hand or by
|
|
97
|
+
# something else) reads as `{ name }` (`name` being the job's, when
|
|
98
|
+
# given) marked unreadable, so the one row does not stop every other job
|
|
99
|
+
# being listed; a check or a dashboard read reports that job and shows it
|
|
100
|
+
# as failing (see Client#evaluable). `tags` is kept only when it is a
|
|
101
|
+
# list of strings. Every other field is kept as stored.
|
|
102
|
+
def self.from_h(hash, name = nil)
|
|
89
103
|
return hash if hash.is_a?(JobDefinition)
|
|
104
|
+
return unreadable(name) unless hash.is_a?(Hash)
|
|
105
|
+
|
|
106
|
+
fields = hash.each_with_object({}) { |(k, v), out| out[BY_JSON[k.to_s] || k.to_s] = v }
|
|
107
|
+
tags = fields[:tags]
|
|
108
|
+
fields.delete(:tags) if fields.key?(:tags) && !(tags.is_a?(Array) && tags.all?(String))
|
|
109
|
+
new(fields)
|
|
110
|
+
end
|
|
90
111
|
|
|
91
|
-
|
|
112
|
+
# `{ name }`, marked unreadable. See from_h.
|
|
113
|
+
def self.unreadable(name)
|
|
114
|
+
definition = allocate
|
|
115
|
+
definition.instance_variable_set(:@fields, (name.nil? ? {} : { name: name }).freeze)
|
|
116
|
+
definition.instance_variable_set(:@unreadable, true)
|
|
117
|
+
definition.freeze
|
|
92
118
|
end
|
|
119
|
+
private_class_method :unreadable
|
|
93
120
|
|
|
94
121
|
def [](field)
|
|
95
122
|
@fields[field]
|
|
96
123
|
end
|
|
97
124
|
|
|
125
|
+
# True for a stored definition that was not a JSON object. See from_h.
|
|
126
|
+
def unreadable?
|
|
127
|
+
@unreadable
|
|
128
|
+
end
|
|
129
|
+
|
|
98
130
|
def key?(field)
|
|
99
131
|
@fields.key?(field)
|
|
100
132
|
end
|
|
@@ -146,25 +178,38 @@ module Cronwatch
|
|
|
146
178
|
new(
|
|
147
179
|
id: Naming.fetch(hash, "id"),
|
|
148
180
|
job: Naming.fetch(hash, "job"),
|
|
149
|
-
status: Naming.fetch(hash, "status")
|
|
181
|
+
status: Run.status_from(Naming.fetch(hash, "status")),
|
|
150
182
|
started_at: Naming.fetch(hash, "startedAt"),
|
|
151
183
|
finished_at: Naming.fetch(hash, "finishedAt"),
|
|
152
184
|
duration_ms: Naming.fetch(hash, "durationMs"),
|
|
153
185
|
error: Naming.fetch(hash, "error"),
|
|
154
186
|
output: Naming.fetch(hash, "output"),
|
|
155
|
-
metrics: (Naming.fetch(hash, "metrics")
|
|
187
|
+
metrics: Run.metrics_from(Naming.fetch(hash, "metrics")),
|
|
156
188
|
trigger: Naming.fetch(hash, "trigger", "run"),
|
|
157
189
|
)
|
|
158
190
|
end
|
|
159
191
|
|
|
160
192
|
def to_h
|
|
161
193
|
{
|
|
162
|
-
"id" => id, "job" => job, "status" => status.
|
|
194
|
+
"id" => id, "job" => job, "status" => status.is_a?(Symbol) ? status.to_s : status,
|
|
195
|
+
"startedAt" => started_at, "finishedAt" => finished_at,
|
|
163
196
|
"durationMs" => duration_ms, "error" => error, "output" => output,
|
|
164
|
-
"metrics" => (metrics
|
|
197
|
+
"metrics" => Run.metrics_from(metrics), "trigger" => trigger,
|
|
165
198
|
}
|
|
166
199
|
end
|
|
167
200
|
|
|
201
|
+
# A status as read: text as a Symbol, anything else (a foreign or
|
|
202
|
+
# damaged row's) kept as it came, which matches no known status.
|
|
203
|
+
def self.status_from(value)
|
|
204
|
+
value.is_a?(String) ? value.to_sym : value
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Stored metrics as a Hash with string keys. Anything else (a string or
|
|
208
|
+
# an array a foreign or corrupted row holds) reads as none.
|
|
209
|
+
def self.metrics_from(value)
|
|
210
|
+
value.is_a?(Hash) ? value.transform_keys(&:to_s) : {}
|
|
211
|
+
end
|
|
212
|
+
|
|
168
213
|
def running? = status == :running
|
|
169
214
|
def ok? = status == :ok
|
|
170
215
|
end
|
|
@@ -175,9 +220,10 @@ module Cronwatch
|
|
|
175
220
|
def self.from_h(hash)
|
|
176
221
|
return hash if hash.is_a?(StoredJob)
|
|
177
222
|
|
|
223
|
+
name = Naming.fetch(hash, "name")
|
|
178
224
|
new(
|
|
179
|
-
name:
|
|
180
|
-
definition: JobDefinition.from_h(Naming.fetch(hash, "definition")
|
|
225
|
+
name: name,
|
|
226
|
+
definition: JobDefinition.from_h(Naming.fetch(hash, "definition"), name),
|
|
181
227
|
created_at: Naming.fetch(hash, "createdAt"),
|
|
182
228
|
updated_at: Naming.fetch(hash, "updatedAt"),
|
|
183
229
|
)
|
|
@@ -190,6 +236,8 @@ module Cronwatch
|
|
|
190
236
|
|
|
191
237
|
# An alert before it has a title and message. See Format.compose_alert.
|
|
192
238
|
# `details` is a hash with snake_case symbol keys; its JSON is camelCase.
|
|
239
|
+
#
|
|
240
|
+
# @api private
|
|
193
241
|
AlertDraft = Struct.new(:type, :run, :details, keyword_init: true) do
|
|
194
242
|
include Serializable
|
|
195
243
|
|
|
@@ -220,77 +268,201 @@ module Cronwatch
|
|
|
220
268
|
@triage_tried = true
|
|
221
269
|
end
|
|
222
270
|
|
|
271
|
+
# Read leniently, since a queued alert a foreign writer left malformed
|
|
272
|
+
# must affect only its own job: a field of the wrong type is kept as it
|
|
273
|
+
# came (a `type` or `run` that is not what it should be, `details` that
|
|
274
|
+
# are not an object, an `at` that is not a number), and the retry drops
|
|
275
|
+
# such an alert as stale (Evaluate.stale_alert?).
|
|
223
276
|
def self.from_h(hash)
|
|
224
277
|
return hash if hash.is_a?(Alert)
|
|
225
278
|
|
|
226
279
|
run = Naming.fetch(hash, "run")
|
|
227
|
-
|
|
228
|
-
details
|
|
229
|
-
|
|
280
|
+
written = Naming.fetch(hash, "details") || {}
|
|
281
|
+
details = Naming.from_json_value(written)
|
|
282
|
+
if details.is_a?(Hash)
|
|
283
|
+
details[:after] = details[:after].map { |c| c.is_a?(String) ? c.to_sym : c } if details[:after].is_a?(Array)
|
|
284
|
+
details[:reason] = details[:reason].to_sym if details[:reason].is_a?(String)
|
|
285
|
+
end
|
|
286
|
+
type = Naming.fetch(hash, "type")
|
|
287
|
+
definition = Naming.fetch(hash, "definition")
|
|
230
288
|
alert = new(
|
|
231
|
-
type:
|
|
232
|
-
run: run
|
|
289
|
+
type: type.is_a?(String) ? type.to_sym : type,
|
|
290
|
+
run: run.is_a?(Hash) ? Run.from_h(run) : run,
|
|
233
291
|
details: details,
|
|
234
292
|
job: Naming.fetch(hash, "job"),
|
|
235
|
-
definition: JobDefinition.from_h(
|
|
293
|
+
definition: definition.nil? || definition.is_a?(Hash) ? JobDefinition.from_h(definition || {}) : definition,
|
|
236
294
|
title: Naming.fetch(hash, "title"),
|
|
237
295
|
message: Naming.fetch(hash, "message"),
|
|
238
296
|
at: Naming.fetch(hash, "at"),
|
|
239
297
|
)
|
|
240
298
|
alert.triage_result = Naming.fetch(hash, "triage") if Naming.present?(hash, "triage")
|
|
299
|
+
alert.keep_as_written(hash, written, details)
|
|
241
300
|
alert
|
|
242
301
|
end
|
|
243
302
|
|
|
303
|
+
# What a stored alert held that its members do not say again: the
|
|
304
|
+
# fields this version does not know (a newer release may add one), its
|
|
305
|
+
# details as written, since snake_case and back would turn a key spelled
|
|
306
|
+
# `a_b` into `aB`, and which known fields it left out. to_h writes the
|
|
307
|
+
# first two back, the details only while they are unchanged, and leaves
|
|
308
|
+
# out a field that was absent while it still holds what its absence
|
|
309
|
+
# read as, so a partial alert (a foreign row's) is written as it came,
|
|
310
|
+
# in the order it came. An alert with every field the SDK writes has
|
|
311
|
+
# them written back in the SDK's order, then the fields it does not
|
|
312
|
+
# know, since a store on Postgres jsonb gives an object's keys back in
|
|
313
|
+
# an order of its own (as JobState.sending_entry does for an entry).
|
|
314
|
+
#
|
|
315
|
+
# @api private
|
|
316
|
+
def keep_as_written(hash, written, details)
|
|
317
|
+
extra = hash.each_with_object({}) { |(k, v), out| out[k.to_s] = v unless Alert::KNOWN.include?(k.to_s) }
|
|
318
|
+
@extra = extra.empty? ? nil : extra
|
|
319
|
+
@written_details = [Marshal.load(Marshal.dump(details)), Alert.string_keys(written)] if written.is_a?(Hash)
|
|
320
|
+
absent = (Alert::KNOWN - ["triage"]).reject { |key| Naming.present?(hash, key) }
|
|
321
|
+
@absent = absent.empty? ? nil : to_h.slice(*absent)
|
|
322
|
+
order = hash.keys.map(&:to_s)
|
|
323
|
+
@order = @absent && order.uniq == order ? order : nil
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# @api private
|
|
327
|
+
def self.string_keys(value)
|
|
328
|
+
case value
|
|
329
|
+
when Hash then value.to_h { |k, v| [k.to_s, string_keys(v)] }
|
|
330
|
+
when Array then value.map { |v| string_keys(v) }
|
|
331
|
+
else value
|
|
332
|
+
end
|
|
333
|
+
end
|
|
334
|
+
|
|
244
335
|
def to_h
|
|
245
336
|
out = {
|
|
246
|
-
"type" => type.to_s, "run" => run
|
|
247
|
-
"
|
|
337
|
+
"type" => type.is_a?(Symbol) ? type.to_s : type, "run" => run.is_a?(Run) ? run.to_h : run,
|
|
338
|
+
"details" => details_json, "job" => job,
|
|
339
|
+
"definition" => definition.is_a?(JobDefinition) ? definition.to_h : definition,
|
|
248
340
|
"title" => title, "message" => message, "at" => at,
|
|
249
341
|
}
|
|
342
|
+
@absent&.each { |key, value| out.delete(key) if out[key] == value }
|
|
250
343
|
out["triage"] = triage if triage_tried?
|
|
251
|
-
out
|
|
344
|
+
@extra&.each { |key, value| out[key] = value unless out.key?(key) || Alert::KNOWN.include?(key) }
|
|
345
|
+
return out if @order.nil? || @order == out.keys
|
|
346
|
+
|
|
347
|
+
# The keys in the order they were read, as the object a store held
|
|
348
|
+
# keeps them; any added since go after.
|
|
349
|
+
written = @order.select { |key| out.key?(key) }
|
|
350
|
+
(written + (out.keys - written)).to_h { |key| [key, out[key]] }
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
private
|
|
354
|
+
|
|
355
|
+
def details_json
|
|
356
|
+
return @written_details[1] if @written_details && @written_details[0] == details
|
|
357
|
+
|
|
358
|
+
Naming.to_json_value(details)
|
|
252
359
|
end
|
|
253
360
|
end
|
|
254
361
|
|
|
362
|
+
# The fields this version reads from a stored alert; the rest are kept as they came.
|
|
363
|
+
# @api private
|
|
364
|
+
Alert::KNOWN = %w[type run details job definition title message at triage].freeze
|
|
365
|
+
|
|
255
366
|
# `version` goes up by one on every write, so a store can refuse a write
|
|
256
367
|
# made from a stale read (see Stores::Memory#compare_and_set_state). Absent
|
|
257
368
|
# (nil) counts as 0.
|
|
369
|
+
#
|
|
370
|
+
# from_h reads a state as stored, keeping each field as it came where it
|
|
371
|
+
# has the wrong type (Evaluate.normalize_state then reads it leniently),
|
|
372
|
+
# so a foreign, hand-edited or damaged row is written back unchanged by a
|
|
373
|
+
# read that changes nothing, and affects only its own job. A stored state
|
|
374
|
+
# that is not a JSON object reads as none (nil).
|
|
375
|
+
#
|
|
376
|
+
# `sending` is the outbox (see Evaluate.hold_alerts): entries of
|
|
377
|
+
# { "until" => epoch ms, "alert" => Alert }, read leniently, since an entry
|
|
378
|
+
# a foreign writer left malformed must not make the whole state unreadable:
|
|
379
|
+
# an entry that is not a Hash, or whose alert is not one, is kept as it
|
|
380
|
+
# came. nil when there is none; it is never written empty.
|
|
381
|
+
#
|
|
382
|
+
# `extra` holds the fields this version does not know, written by a newer
|
|
383
|
+
# one, as the JSON they came as (string keys): every write carries them
|
|
384
|
+
# back unchanged, after the known fields, so a process sharing the store
|
|
385
|
+
# with a newer release never erases what that release keeps there. nil
|
|
386
|
+
# when there are none. An open condition this version does not know stays
|
|
387
|
+
# in `open` the same way, under its own name.
|
|
258
388
|
JobState = Struct.new(:job, :open, :consecutive_failures, :silenced_until, :last_alert_at, :pending_recovery,
|
|
259
|
-
:undelivered, :version, keyword_init: true) do
|
|
389
|
+
:undelivered, :version, :sending, :extra, keyword_init: true) do
|
|
260
390
|
include Serializable
|
|
261
391
|
|
|
262
392
|
def self.from_h(hash)
|
|
263
393
|
return hash if hash.is_a?(JobState)
|
|
394
|
+
return nil unless hash.is_a?(Hash)
|
|
264
395
|
|
|
396
|
+
open = Naming.fetch(hash, "open") || {}
|
|
265
397
|
pending = Naming.fetch(hash, "pendingRecovery")
|
|
266
398
|
undelivered = Naming.fetch(hash, "undelivered")
|
|
399
|
+
sending = Naming.fetch(hash, "sending")
|
|
400
|
+
extra = hash.each_with_object({}) { |(k, v), out| out[k.to_s] = v unless JobState::KNOWN.include?(k.to_s) }
|
|
267
401
|
new(
|
|
402
|
+
extra: extra.empty? ? nil : extra,
|
|
268
403
|
job: Naming.fetch(hash, "job"),
|
|
269
|
-
open:
|
|
404
|
+
open: open.is_a?(Hash) ? open.to_h { |k, v| [k.to_sym, v] } : open,
|
|
270
405
|
consecutive_failures: Naming.fetch(hash, "consecutiveFailures", 0),
|
|
271
406
|
silenced_until: Naming.fetch(hash, "silencedUntil"),
|
|
272
407
|
last_alert_at: Naming.fetch(hash, "lastAlertAt"),
|
|
273
|
-
pending_recovery: pending
|
|
274
|
-
undelivered: undelivered
|
|
408
|
+
pending_recovery: pending.is_a?(Array) ? pending.map { |c| c.is_a?(String) ? c.to_sym : c } : pending,
|
|
409
|
+
undelivered: undelivered.is_a?(Array) ? undelivered.map { |a| a.is_a?(Hash) ? Alert.from_h(a) : a } : undelivered,
|
|
275
410
|
version: Naming.fetch(hash, "version"),
|
|
411
|
+
sending: sending.is_a?(Array) ? sending.map { |entry| JobState.sending_entry(entry) } : sending,
|
|
276
412
|
)
|
|
277
413
|
end
|
|
278
414
|
|
|
415
|
+
# One entry of `sending` as read: its alert parsed when it is a Hash
|
|
416
|
+
# that parses, anything else kept as it came. `until` and `alert` come
|
|
417
|
+
# first, in the order the SDK writes them, since a store on Postgres
|
|
418
|
+
# jsonb gives an object's keys back in an order of its own.
|
|
419
|
+
def self.sending_entry(entry)
|
|
420
|
+
return entry unless entry.is_a?(Hash)
|
|
421
|
+
|
|
422
|
+
read = entry.to_h do |key, value|
|
|
423
|
+
next [key.to_s, value] unless key.to_s == "alert" && value.is_a?(Hash)
|
|
424
|
+
|
|
425
|
+
parsed = begin
|
|
426
|
+
Alert.from_h(value)
|
|
427
|
+
rescue StandardError
|
|
428
|
+
value
|
|
429
|
+
end
|
|
430
|
+
["alert", parsed]
|
|
431
|
+
end
|
|
432
|
+
%w[until alert].select { |key| read.key?(key) }.to_h { |key| [key, read[key]] }.merge(read)
|
|
433
|
+
end
|
|
434
|
+
|
|
279
435
|
# pendingRecovery, undelivered and version are left out when unset, as in
|
|
280
|
-
# state written before they existed. The version comes
|
|
281
|
-
# SDK's spread of a normalized state puts it
|
|
436
|
+
# state written before they existed. The version comes after them, where
|
|
437
|
+
# the SDK's spread of a normalized state puts it, and `sending` last, and
|
|
438
|
+
# only while it holds an entry.
|
|
282
439
|
def to_h
|
|
283
440
|
out = {
|
|
284
|
-
"job" => job, "open" => (
|
|
285
|
-
"silencedUntil" => silenced_until, "lastAlertAt" => last_alert_at,
|
|
441
|
+
"job" => job, "open" => open.is_a?(Hash) ? open.transform_keys(&:to_s) : (open || {}),
|
|
442
|
+
"consecutiveFailures" => consecutive_failures, "silencedUntil" => silenced_until, "lastAlertAt" => last_alert_at,
|
|
286
443
|
}
|
|
287
|
-
|
|
288
|
-
|
|
444
|
+
unless pending_recovery.nil?
|
|
445
|
+
out["pendingRecovery"] =
|
|
446
|
+
pending_recovery.is_a?(Array) ? pending_recovery.map { |c| c.is_a?(Symbol) ? c.to_s : c } : pending_recovery
|
|
447
|
+
end
|
|
448
|
+
unless undelivered.nil?
|
|
449
|
+
out["undelivered"] = undelivered.is_a?(Array) ? undelivered.map { |a| a.is_a?(Alert) ? a.to_h : a } : undelivered
|
|
450
|
+
end
|
|
289
451
|
out["version"] = version unless version.nil?
|
|
452
|
+
if sending.is_a?(Array) && !sending.empty?
|
|
453
|
+
out["sending"] = sending.map do |entry|
|
|
454
|
+
entry.is_a?(Hash) ? entry.transform_values { |v| v.is_a?(Alert) ? v.to_h : v } : entry
|
|
455
|
+
end
|
|
456
|
+
end
|
|
457
|
+
extra&.each { |key, value| out[key] = value unless out.key?(key) || JobState::KNOWN.include?(key) }
|
|
290
458
|
out
|
|
291
459
|
end
|
|
292
460
|
end
|
|
293
461
|
|
|
462
|
+
# The fields this version reads from a stored state; the rest are kept in `extra`.
|
|
463
|
+
# @api private
|
|
464
|
+
JobState::KNOWN = %w[job open consecutiveFailures silencedUntil lastAlertAt pendingRecovery undelivered version sending].freeze
|
|
465
|
+
|
|
294
466
|
JobStats = Struct.new(:runs, :ok_rate, :p50_ms, :p95_ms, keyword_init: true) do
|
|
295
467
|
include Serializable
|
|
296
468
|
|
data/lib/cronwatch/version.rb
CHANGED
data/lib/cronwatch/walker.rb
CHANGED
|
@@ -6,6 +6,8 @@ module Cronwatch
|
|
|
6
6
|
# the next match, a field at a time, spilling into the next month or year
|
|
7
7
|
# as croner does (including its habit of stepping into a day the month
|
|
8
8
|
# does not have and letting the date roll over). Month is 0 based.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
9
11
|
class Walker
|
|
10
12
|
DAYS_IN_MONTH = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31].freeze
|
|
11
13
|
# [field, the field above it, offset from a field value to its pattern index]
|
data/lib/cronwatch/web/app.rb
CHANGED
|
@@ -10,19 +10,29 @@ module Cronwatch
|
|
|
10
10
|
# (routes/index.ts) with the same URLs, JSON, auth, CSRF rules and headers,
|
|
11
11
|
# so @cronwatch/mcp works against a Ruby app as it does against a Node one.
|
|
12
12
|
#
|
|
13
|
+
# Made by Client#routes, the one way to mount it:
|
|
14
|
+
#
|
|
13
15
|
# # config/routes.rb
|
|
14
|
-
# mount Cronwatch
|
|
16
|
+
# mount Cronwatch.client.routes => "/cronwatch"
|
|
15
17
|
#
|
|
16
18
|
# # config.ru, standalone
|
|
17
|
-
# run
|
|
19
|
+
# run CW.routes
|
|
20
|
+
#
|
|
21
|
+
# Cronwatch::Web.new(client, **options) makes the same app and is
|
|
22
|
+
# deprecated: it still works through 1.x and goes in 2.0.
|
|
18
23
|
#
|
|
19
24
|
# token: required to reach anything. Send it as `Authorization: Bearer <token>`,
|
|
20
|
-
# or open the dashboard once with `?token=<token>`
|
|
21
|
-
#
|
|
25
|
+
# or open the dashboard once with `?token=<token>` (or sign in with the
|
|
26
|
+
# form, which posts it to <base>/signin) and a cookie is set.
|
|
27
|
+
# Defaults to ENV["CRONWATCH_TOKEN"]; an empty string or one of only
|
|
28
|
+
# whitespace, given here or in the variable, counts as unset, and
|
|
29
|
+
# anything but a String or nil (false, a number, a Symbol) raises TypeError.
|
|
22
30
|
# With no token while Cronwatch::Environment is development or
|
|
23
|
-
# test
|
|
24
|
-
#
|
|
25
|
-
#
|
|
31
|
+
# test (Environment.stated_development?: under Puma, Unicorn,
|
|
32
|
+
# Thin or rackup, which set RACK_ENV=development by default,
|
|
33
|
+
# RACK_ENV alone does not count), the app makes a random one and
|
|
34
|
+
# prints a sign-in link to stdout on its first request; with no
|
|
35
|
+
# token otherwise it answers 503. Pass `token: nil` to opt out and serve it open everywhere,
|
|
26
36
|
# for example behind your own auth. /api/check also accepts the
|
|
27
37
|
# client's cron_secret as a bearer, for a platform cron.
|
|
28
38
|
# base_path: where the app is mounted, so links resolve. Defaults to the
|
|
@@ -64,22 +74,48 @@ module Cronwatch
|
|
|
64
74
|
SECURITY_HEADERS = {
|
|
65
75
|
"x-content-type-options" => "nosniff", "referrer-policy" => "same-origin", "x-robots-tag" => "noindex",
|
|
66
76
|
}.freeze
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
77
|
+
# A bearer is what follows the scheme when the scheme is Bearer (any case)
|
|
78
|
+
# and whitespace follows it. Any other Authorization header (a proxy's
|
|
79
|
+
# Basic auth) is no bearer at all.
|
|
80
|
+
BEARER = Regexp.new("\\ABearer[#{JS::WHITESPACE}]+(.*)\\z", Regexp::IGNORECASE | Regexp::MULTILINE)
|
|
81
|
+
# What GET <base>/api names, so a client such as @cronwatch/mcp can tell
|
|
82
|
+
# what it is talking to. API_VERSION goes up only with a change that is
|
|
83
|
+
# not additive, in a major release.
|
|
84
|
+
LIBRARY = "cronwatch"
|
|
85
|
+
LANGUAGE = "ruby"
|
|
86
|
+
API_VERSION = 1
|
|
87
|
+
private_constant :API_VERSION, :ASSET_CSP, :BEARER, :BOARD_PAGE_RUNS, :COOKIE, :COOKIE_MAX_AGE, :CSP,
|
|
88
|
+
:DEFAULT_RUNS, :LANGUAGE, :LIBRARY, :MAX_RUNS, :SECURITY_HEADERS
|
|
89
|
+
|
|
90
|
+
# Deprecated: use client.routes(**options). Still works through 1.x; a
|
|
91
|
+
# Web made without a client serves Cronwatch.client, read on each request.
|
|
92
|
+
def self.new(client = nil, **options)
|
|
93
|
+
Deprecation.warn("Cronwatch::Web.new(client, ...)", "client.routes(...)")
|
|
94
|
+
build(client, **options)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The app Client#routes returns, without the warning.
|
|
98
|
+
#
|
|
99
|
+
# @api private
|
|
100
|
+
def self.build(client = nil, **options)
|
|
101
|
+
app = allocate
|
|
102
|
+
app.send(:initialize, client, **options)
|
|
103
|
+
app
|
|
104
|
+
end
|
|
70
105
|
|
|
71
106
|
def initialize(client = nil, token: UNSET, base_path: nil, origin: nil)
|
|
72
107
|
@client = client
|
|
73
108
|
@origin = Web.configured_origin(origin)
|
|
74
109
|
@opted_out = token.nil?
|
|
75
|
-
|
|
76
|
-
|
|
110
|
+
# A blank token, given or read, counts as unset; one given that is not a String raises.
|
|
111
|
+
given = token.equal?(UNSET) ? nil : Environment.secret_option(token, "routes: token")
|
|
112
|
+
@token = @opted_out ? nil : given || Environment.secret("CRONWATCH_TOKEN")
|
|
77
113
|
@base_path = base_path&.to_s&.sub(%r{/+\z}, "")
|
|
78
114
|
# A Rack app cannot reliably tell a local caller from a remote one
|
|
79
115
|
# (proxies, tunnels and a server bound to every interface all look
|
|
80
116
|
# alike), so development gets a token too: made here, and shown only in
|
|
81
117
|
# the server log.
|
|
82
|
-
@generated = @token.nil? && !@opted_out &&
|
|
118
|
+
@generated = @token.nil? && !@opted_out && Environment.stated_development?
|
|
83
119
|
@token = Web.development_token if @generated
|
|
84
120
|
@announced = false
|
|
85
121
|
@announce_lock = Mutex.new
|
|
@@ -181,7 +217,7 @@ module Cronwatch
|
|
|
181
217
|
|
|
182
218
|
# No token outside development: fail closed.
|
|
183
219
|
if !@token && !@opted_out
|
|
184
|
-
return wants_html ? html(HTML.message_page("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token: to
|
|
220
|
+
return wants_html ? html(HTML.message_page("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token: to routes), or pass token: nil to serve them open behind your own auth.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503)
|
|
185
221
|
end
|
|
186
222
|
|
|
187
223
|
if method != "GET" && method != "HEAD" && cross_site?(request)
|
|
@@ -189,7 +225,18 @@ module Cronwatch
|
|
|
189
225
|
end
|
|
190
226
|
|
|
191
227
|
authorization = request.header("authorization")
|
|
192
|
-
bearer = authorization&.
|
|
228
|
+
bearer = authorization && BEARER.match(authorization)&.[](1)
|
|
229
|
+
|
|
230
|
+
# The sign-in form posts the token here, in the body, so it stays out of
|
|
231
|
+
# the URL and every access log. Cross-site posts were refused above.
|
|
232
|
+
if @token && method == "POST" && path == "/signin"
|
|
233
|
+
sent = read_body(request)["token"]
|
|
234
|
+
return sign_in_page(base) if sent.nil? || !HTTP.constant_time_equal?(sent, @token)
|
|
235
|
+
|
|
236
|
+
return redirect(sign_in_return(request.header("referer"), public_origin(request), base),
|
|
237
|
+
"set-cookie" => sign_in_cookie(request, base))
|
|
238
|
+
end
|
|
239
|
+
|
|
193
240
|
if @token
|
|
194
241
|
# ?token= is only the sign-in that moves the token into a cookie.
|
|
195
242
|
query = wants_html && method == "GET" ? request.query("token") : nil
|
|
@@ -202,16 +249,18 @@ module Cronwatch
|
|
|
202
249
|
else !cookie.nil? && HTTP.constant_time_equal?(cookie, cookie_value(@token))
|
|
203
250
|
end
|
|
204
251
|
unless cron_secret_ok || token_ok
|
|
252
|
+
return sign_in_page(base) if wants_html
|
|
205
253
|
if @generated
|
|
206
|
-
return
|
|
254
|
+
return api({ ok: false, error: "Unauthorized: CRONWATCH_TOKEN is not set, so this development server made a token; it is in the server log" }, 401)
|
|
207
255
|
end
|
|
208
|
-
|
|
256
|
+
|
|
257
|
+
return api({ ok: false, error: "Unauthorized" }, 401)
|
|
209
258
|
end
|
|
210
259
|
unless query.nil?
|
|
211
|
-
# Move the token from the URL into a cookie so it is not in
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
260
|
+
# Move the token from the URL into a cookie, so it is not left in the
|
|
261
|
+
# browser's history. The request line that carried it may still be in
|
|
262
|
+
# an access log, which is why the sign-in form posts instead.
|
|
263
|
+
return redirect(request.pathname + request.search_without("token"), "set-cookie" => sign_in_cookie(request, base))
|
|
215
264
|
end
|
|
216
265
|
end
|
|
217
266
|
|
|
@@ -257,7 +306,7 @@ module Cronwatch
|
|
|
257
306
|
rescue ArgumentError => e
|
|
258
307
|
return html(HTML.message_page("Not silenced", e.message, base), 400)
|
|
259
308
|
end
|
|
260
|
-
cw.silence(name, duration)
|
|
309
|
+
cw.silence(name, for: duration)
|
|
261
310
|
else
|
|
262
311
|
cw.unsilence(name)
|
|
263
312
|
end
|
|
@@ -271,6 +320,9 @@ module Cronwatch
|
|
|
271
320
|
|
|
272
321
|
def serve_api(request, method, rest, bearer)
|
|
273
322
|
cw = client
|
|
323
|
+
if method == "GET" && rest.empty?
|
|
324
|
+
return api({ ok: true, library: LIBRARY, language: LANGUAGE, version: VERSION, api: API_VERSION })
|
|
325
|
+
end
|
|
274
326
|
return api({ ok: true, jobs: cw.jobs }) if method == "GET" && rest[0] == "jobs" && rest.length == 1
|
|
275
327
|
|
|
276
328
|
if rest[0] == "jobs" && rest.length == 2
|
|
@@ -299,9 +351,13 @@ module Cronwatch
|
|
|
299
351
|
rescue ArgumentError => e
|
|
300
352
|
return api({ ok: false, error: e.message }, 400)
|
|
301
353
|
end
|
|
302
|
-
|
|
354
|
+
cw.silence(name, for: duration)
|
|
355
|
+
return api({ ok: true, job: cw.job_summary(name) })
|
|
356
|
+
end
|
|
357
|
+
if rest[2] == "unsilence"
|
|
358
|
+
cw.unsilence(name)
|
|
359
|
+
return api({ ok: true, job: cw.job_summary(name) })
|
|
303
360
|
end
|
|
304
|
-
return api({ ok: true, state: cw.unsilence(name) }) if rest[2] == "unsilence"
|
|
305
361
|
end
|
|
306
362
|
if rest[0] == "check" && rest.length == 1
|
|
307
363
|
# A page cannot send an Authorization header cross-site, so a GET
|
|
@@ -348,6 +404,37 @@ module Cronwatch
|
|
|
348
404
|
$stdout.flush
|
|
349
405
|
end
|
|
350
406
|
|
|
407
|
+
# The 401 sign-in page, with the form that posts the token to <base>/signin.
|
|
408
|
+
def sign_in_page(base)
|
|
409
|
+
message =
|
|
410
|
+
if @generated
|
|
411
|
+
"CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server " \
|
|
412
|
+
"log: open it once, or enter the token from it below, and this browser stays signed in."
|
|
413
|
+
else
|
|
414
|
+
"Enter your CRONWATCH_TOKEN and this browser stays signed in."
|
|
415
|
+
end
|
|
416
|
+
html(HTML.message_page("Sign in", message, base, sign_in: true), 401)
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
# The cookie a sign-in sets, by ?token= or by the form.
|
|
420
|
+
def sign_in_cookie(request, base)
|
|
421
|
+
secure = public_origin(request).start_with?("https:") ? "; Secure" : ""
|
|
422
|
+
"#{COOKIE}=#{cookie_value(@token)}; Path=#{base.empty? ? "/" : base}; HttpOnly; SameSite=Lax; Max-Age=#{COOKIE_MAX_AGE}#{secure}"
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
# Where a sign-in through the form goes next: the page it was posted from
|
|
426
|
+
# (the Referer) when that is on the public origin and its query has no
|
|
427
|
+
# `token` parameter, else the dashboard. A Referer that starts with the
|
|
428
|
+
# origin and "/" always parses as a URL; its query is read as
|
|
429
|
+
# URLSearchParams reads it, after the tabs and newlines a URL parser drops.
|
|
430
|
+
def sign_in_return(referer, origin, base)
|
|
431
|
+
return "#{base}/" if referer.nil? || !referer.start_with?("#{origin}/")
|
|
432
|
+
|
|
433
|
+
url = referer.delete("\t\n\r").split("#", 2).first.to_s
|
|
434
|
+
query = url.include?("?") ? url.split("?", 2).last : ""
|
|
435
|
+
Request.parse_query(query).any? { |key, _| key == "token" } ? "#{base}/" : referer
|
|
436
|
+
end
|
|
437
|
+
|
|
351
438
|
# The cookie holds a digest of the token, so a leaked cookie does not reveal the bearer token itself.
|
|
352
439
|
def cookie_value(token)
|
|
353
440
|
Digest::SHA256.hexdigest("cronwatch-cookie:#{token}")
|
|
@@ -446,19 +533,7 @@ module Cronwatch
|
|
|
446
533
|
|
|
447
534
|
# Number(string): decimal, 0x/0o/0b, Infinity, or NaN.
|
|
448
535
|
def js_number(value)
|
|
449
|
-
|
|
450
|
-
return 0 if text.empty?
|
|
451
|
-
# Ruby's Float() wants a digit on both sides of the point; JavaScript does not.
|
|
452
|
-
return Float(text.sub(/\A([+-]?)\./, "\\10.").sub(/\.(?=[eE]|\z)/, ".0")) if DECIMAL.match?(text)
|
|
453
|
-
return text.start_with?("-") ? -Float::INFINITY : Float::INFINITY if /\A[+-]?Infinity\z/.match?(text)
|
|
454
|
-
|
|
455
|
-
if (m = /\A0([xXoObB])([0-9a-fA-F]+)\z/.match(text))
|
|
456
|
-
return Integer(m[2], RADIX.fetch(m[1].downcase))
|
|
457
|
-
end
|
|
458
|
-
|
|
459
|
-
Float::NAN
|
|
460
|
-
rescue ArgumentError
|
|
461
|
-
Float::NAN
|
|
536
|
+
JS.to_number(value)
|
|
462
537
|
end
|
|
463
538
|
|
|
464
539
|
def whole(number)
|
|
@@ -506,6 +581,8 @@ module Cronwatch
|
|
|
506
581
|
end
|
|
507
582
|
|
|
508
583
|
# What the routes need from a Rack env, read the way the SDK reads a fetch Request.
|
|
584
|
+
#
|
|
585
|
+
# @api private
|
|
509
586
|
class Request
|
|
510
587
|
attr_reader :env
|
|
511
588
|
|