cronwatch 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +18 -9
- data/lib/cronwatch/alerts/datadog.rb +1 -0
- data/lib/cronwatch/alerts/discord.rb +22 -3
- data/lib/cronwatch/alerts/email.rb +2 -0
- data/lib/cronwatch/alerts/honeybadger.rb +1 -0
- data/lib/cronwatch/alerts/postmark.rb +1 -0
- data/lib/cronwatch/alerts/provider.rb +2 -0
- data/lib/cronwatch/alerts/resend.rb +1 -0
- data/lib/cronwatch/alerts/rollbar.rb +1 -0
- data/lib/cronwatch/alerts/sentry.rb +1 -0
- data/lib/cronwatch/alerts/sigv4.rb +2 -0
- data/lib/cronwatch/alerts/slack.rb +1 -0
- data/lib/cronwatch/alerts/twilio.rb +1 -0
- data/lib/cronwatch/alerts/webhook.rb +18 -5
- data/lib/cronwatch/client.rb +66 -17
- data/lib/cronwatch/cron_pattern.rb +17 -8
- data/lib/cronwatch/deprecation.rb +18 -0
- data/lib/cronwatch/duration.rb +2 -0
- data/lib/cronwatch/environment.rb +93 -17
- data/lib/cronwatch/evaluate.rb +41 -10
- data/lib/cronwatch/flight.rb +2 -0
- data/lib/cronwatch/format.rb +1 -0
- data/lib/cronwatch/http.rb +21 -5
- data/lib/cronwatch/job.rb +2 -0
- data/lib/cronwatch/js.rb +74 -1
- data/lib/cronwatch/monitored.rb +6 -1
- data/lib/cronwatch/output.rb +3 -0
- data/lib/cronwatch/pg_cron.rb +22 -3
- data/lib/cronwatch/rails/active_job.rb +2 -2
- data/lib/cronwatch/schedule.rb +25 -1
- data/lib/cronwatch/scheduler.rb +6 -1
- data/lib/cronwatch/serialize.rb +1 -0
- data/lib/cronwatch/stats.rb +1 -0
- data/lib/cronwatch/stores/active_record.rb +36 -18
- data/lib/cronwatch/ticker.rb +2 -0
- data/lib/cronwatch/triage/anthropic.rb +2 -0
- data/lib/cronwatch/types.rb +148 -31
- data/lib/cronwatch/version.rb +1 -1
- data/lib/cronwatch/walker.rb +2 -0
- data/lib/cronwatch/web/app.rb +108 -33
- data/lib/cronwatch/web/html.rb +10 -6
- data/lib/cronwatch/web/origin.rb +2 -0
- data/lib/cronwatch/web/pwa.rb +20 -2
- data/lib/cronwatch/web/timeline.rb +2 -0
- data/lib/cronwatch/web.rb +1 -2
- data/lib/cronwatch/zone.rb +2 -0
- data/lib/cronwatch.rb +2 -0
- data/lib/generators/cronwatch/install/install_generator.rb +1 -1
- metadata +2 -1
|
@@ -1,14 +1,91 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Cronwatch
|
|
4
|
-
# The app's environment, read
|
|
5
|
-
#
|
|
4
|
+
# The app's environment, read as every CronWatch library reads it: the
|
|
5
|
+
# first of CRONWATCH_ENV, APP_ENV, then the app's own (Rails.env when Rails
|
|
6
|
+
# is loaded, else RAILS_ENV, then RACK_ENV) that holds more than spaces,
|
|
7
|
+
# trimmed and lowercased, with "prod" read as "production" and "dev",
|
|
8
|
+
# "local", "test" and "testing" as "development". None set is neither,
|
|
9
|
+
# which is the safe reading. The SDK reads NODE_ENV where this reads the
|
|
10
|
+
# app's own.
|
|
11
|
+
#
|
|
12
|
+
# @api private
|
|
6
13
|
module Environment
|
|
7
|
-
|
|
14
|
+
# The variables read before the app's own, in order.
|
|
15
|
+
SHARED = %w[CRONWATCH_ENV APP_ENV].freeze
|
|
16
|
+
ALIASES = {
|
|
17
|
+
"prod" => "production", "dev" => "development", "local" => "development", "test" => "development",
|
|
18
|
+
"testing" => "development",
|
|
19
|
+
}.freeze
|
|
20
|
+
private_constant :SHARED, :ALIASES
|
|
8
21
|
|
|
9
22
|
module_function
|
|
10
23
|
|
|
11
|
-
# "production", "development", ... or nil when nothing names one.
|
|
24
|
+
# "production", "development", "staging", ... or nil when nothing names one.
|
|
25
|
+
def environment
|
|
26
|
+
_, value = source
|
|
27
|
+
return nil if value.nil?
|
|
28
|
+
|
|
29
|
+
value = JS.trim(value).downcase
|
|
30
|
+
ALIASES.fetch(value, value)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Where the environment was read from and its value as set:
|
|
34
|
+
# ["CRONWATCH_ENV", " dev"], ["Rails.env", "test"], or nil.
|
|
35
|
+
def source
|
|
36
|
+
SHARED.each do |variable|
|
|
37
|
+
value = ENV.fetch(variable, nil)
|
|
38
|
+
return [variable, value] if present?(value)
|
|
39
|
+
end
|
|
40
|
+
if defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
41
|
+
env = ::Rails.env.to_s
|
|
42
|
+
return ["Rails.env", env] if present?(env)
|
|
43
|
+
end
|
|
44
|
+
%w[RAILS_ENV RACK_ENV].each do |variable|
|
|
45
|
+
value = ENV.fetch(variable, nil)
|
|
46
|
+
return [variable, value] if present?(value)
|
|
47
|
+
end
|
|
48
|
+
nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def present?(value)
|
|
52
|
+
!value.nil? && !blank?(value)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Whether a String is empty or only whitespace, as JavaScript's
|
|
56
|
+
# String.prototype.trim sees it (JS::WHITESPACE). Bytes that are not
|
|
57
|
+
# UTF-8 are not whitespace.
|
|
58
|
+
def blank?(value)
|
|
59
|
+
JS.trim(value.dup.force_encoding(Encoding::UTF_8).scrub).empty?
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# A secret from the environment (CRONWATCH_TOKEN, CRON_SECRET): nil when
|
|
63
|
+
# the variable is unset, empty or only whitespace, so a blank value counts
|
|
64
|
+
# as not set and the routes fail closed. Any other value is used as it
|
|
65
|
+
# is, untrimmed.
|
|
66
|
+
def secret(variable)
|
|
67
|
+
value = ENV.fetch(variable, nil)
|
|
68
|
+
present?(value) ? value : nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# A token or secret passed in code: a String, or nil. A String that is
|
|
72
|
+
# empty or only whitespace counts as not given (nil here; the caller
|
|
73
|
+
# tells that from nil the opt-out). Anything else (false, a number, a
|
|
74
|
+
# Symbol) raises TypeError naming the option, so it never becomes a
|
|
75
|
+
# password.
|
|
76
|
+
def secret_option(value, what)
|
|
77
|
+
return nil if value.nil?
|
|
78
|
+
unless value.is_a?(String)
|
|
79
|
+
kind = [true, false].include?(value) ? value.to_s : value.class.name
|
|
80
|
+
raise TypeError, "#{what} must be a String, or nil to opt out, not #{kind}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
blank?(value) ? nil : value
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# The framework's own name for the environment, as it sets it: Rails.env
|
|
87
|
+
# when Rails is loaded, otherwise RAILS_ENV, then RACK_ENV, or nil. The
|
|
88
|
+
# section of a Solid Queue file is chosen by it, as Solid Queue does.
|
|
12
89
|
def name
|
|
13
90
|
if defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
14
91
|
env = ::Rails.env.to_s
|
|
@@ -17,30 +94,29 @@ module Cronwatch
|
|
|
17
94
|
[ENV.fetch("RAILS_ENV", nil), ENV.fetch("RACK_ENV", nil)].find { |value| value && !value.empty? }
|
|
18
95
|
end
|
|
19
96
|
|
|
20
|
-
# Development
|
|
97
|
+
# Development (including test): Cronwatch::Web without a token serves only then.
|
|
21
98
|
def development?
|
|
22
|
-
|
|
99
|
+
environment == "development"
|
|
23
100
|
end
|
|
24
101
|
|
|
25
102
|
def production?
|
|
26
|
-
|
|
103
|
+
environment == "production"
|
|
27
104
|
end
|
|
28
105
|
|
|
29
|
-
# Whether development
|
|
30
|
-
#
|
|
31
|
-
#
|
|
106
|
+
# Whether development was named by the app rather than by its server,
|
|
107
|
+
# for the dashboard's own token (Cronwatch::Web). Puma, Unicorn, Thin
|
|
108
|
+
# and rackup set RACK_ENV to "development" when nothing names an
|
|
32
109
|
# environment, so under one of them, in production too, RACK_ENV alone
|
|
33
|
-
# reads "development". There it takes Rails.env
|
|
34
|
-
#
|
|
35
|
-
# server sets it.
|
|
110
|
+
# reads "development". There it takes CRONWATCH_ENV, APP_ENV, Rails.env
|
|
111
|
+
# or RAILS_ENV to say development; any other value of RACK_ENV that reads
|
|
112
|
+
# as development ("test", "dev") still counts, as no server sets it.
|
|
36
113
|
def stated_development?
|
|
37
114
|
return false unless development?
|
|
38
|
-
return true if defined?(::Rails) && ::Rails.respond_to?(:env) && !::Rails.env.to_s.empty?
|
|
39
115
|
|
|
40
|
-
|
|
41
|
-
return
|
|
116
|
+
variable, value = source
|
|
117
|
+
return true unless variable == "RACK_ENV" && JS.trim(value).downcase == "development"
|
|
42
118
|
|
|
43
|
-
|
|
119
|
+
!defaulting_server?
|
|
44
120
|
end
|
|
45
121
|
|
|
46
122
|
# A server that sets RACK_ENV when it is unset is running this process.
|
data/lib/cronwatch/evaluate.rb
CHANGED
|
@@ -4,6 +4,8 @@ module Cronwatch
|
|
|
4
4
|
# Pure decisions about a job's health. Each function takes the current state
|
|
5
5
|
# and returns the new state plus the alerts that should go out. Nothing here
|
|
6
6
|
# touches a store or a network, which is what makes it testable.
|
|
7
|
+
#
|
|
8
|
+
# @api private
|
|
7
9
|
module Evaluate
|
|
8
10
|
DEFAULT_GRACE_MS = 10 * 60_000
|
|
9
11
|
DEFAULT_TIMEOUT_MS = 60 * 60_000
|
|
@@ -69,6 +71,12 @@ module Cronwatch
|
|
|
69
71
|
[count, MAX_DURATION_MS].min.to_i
|
|
70
72
|
end
|
|
71
73
|
|
|
74
|
+
# A JSON number, as JavaScript's typeof value === "number" sees one
|
|
75
|
+
# read from JSON: an Integer or a Float (never true or false).
|
|
76
|
+
def json_number?(value)
|
|
77
|
+
value.is_a?(Integer) || value.is_a?(Float)
|
|
78
|
+
end
|
|
79
|
+
|
|
72
80
|
def empty_state(job)
|
|
73
81
|
JobState.new(job: job, open: {}, consecutive_failures: 0, silenced_until: nil, last_alert_at: nil,
|
|
74
82
|
pending_recovery: [], undelivered: [])
|
|
@@ -77,21 +85,35 @@ module Cronwatch
|
|
|
77
85
|
# A stored state with every field present, or a fresh one. State written
|
|
78
86
|
# by an older version lacks the newer fields. `sending` is the exception:
|
|
79
87
|
# it is there only while it holds an alert (see hold_alerts).
|
|
88
|
+
#
|
|
89
|
+
# Read leniently, since a foreign, hand-edited or damaged row must affect
|
|
90
|
+
# only its own job, and the next write puts it right: a state that is not
|
|
91
|
+
# an object reads as none; `open` keeps only its entries whose value is a
|
|
92
|
+
# number (anything but an object reads as {}); `silenced_until` and
|
|
93
|
+
# `last_alert_at` that are not numbers read as nil; `pending_recovery`
|
|
94
|
+
# keeps only its conditions (strings as stored), and `undelivered` only
|
|
95
|
+
# its entries that are objects (a list of neither shape reads as []).
|
|
96
|
+
# Unknown fields are kept as written.
|
|
80
97
|
def normalize_state(state, job)
|
|
98
|
+
state = JobState.from_h(state)
|
|
81
99
|
return empty_state(job) if state.nil?
|
|
82
100
|
|
|
83
|
-
state = JobState.from_h(state)
|
|
84
101
|
sending = state.sending
|
|
102
|
+
open = state.open.is_a?(Hash) ? state.open.select { |_, at| json_number?(at) } : {}
|
|
103
|
+
pending = state.pending_recovery.is_a?(Array) ? state.pending_recovery.grep(Symbol) : []
|
|
104
|
+
undelivered = state.undelivered.is_a?(Array) ? state.undelivered.grep(Alert) : []
|
|
85
105
|
JobState.new(
|
|
86
106
|
job: state.job.nil? ? job : state.job,
|
|
87
|
-
open:
|
|
107
|
+
open: open,
|
|
88
108
|
consecutive_failures: failure_count(state),
|
|
89
|
-
silenced_until: state.silenced_until,
|
|
90
|
-
last_alert_at: state.last_alert_at,
|
|
91
|
-
pending_recovery:
|
|
92
|
-
undelivered:
|
|
109
|
+
silenced_until: json_number?(state.silenced_until) ? state.silenced_until : nil,
|
|
110
|
+
last_alert_at: json_number?(state.last_alert_at) ? state.last_alert_at : nil,
|
|
111
|
+
pending_recovery: pending,
|
|
112
|
+
undelivered: undelivered,
|
|
93
113
|
version: state.version,
|
|
94
114
|
sending: sending.is_a?(Array) && !sending.empty? ? sending.dup : nil,
|
|
115
|
+
# Fields a newer release wrote, carried through every write.
|
|
116
|
+
extra: state.extra,
|
|
95
117
|
)
|
|
96
118
|
end
|
|
97
119
|
|
|
@@ -114,7 +136,8 @@ module Cronwatch
|
|
|
114
136
|
# as JavaScript prints a number.
|
|
115
137
|
def alert_key(alert)
|
|
116
138
|
at = alert.at
|
|
117
|
-
|
|
139
|
+
run = alert.run
|
|
140
|
+
"#{alert.type}|#{at.is_a?(Numeric) ? JS.number(at) : at}|#{run.id if run.is_a?(Run)}"
|
|
118
141
|
end
|
|
119
142
|
|
|
120
143
|
# `alerts` added to the undelivered queue: one with the same key as a
|
|
@@ -444,11 +467,19 @@ module Cronwatch
|
|
|
444
467
|
# is dropped rather than sent late. An alert for a condition is stale once
|
|
445
468
|
# that condition has closed, or has closed and opened again (it opened at a
|
|
446
469
|
# time other than the alert's). A recovery is stale when any condition it
|
|
447
|
-
# names is open again; while they all stay closed it is kept.
|
|
470
|
+
# names is open again; while they all stay closed it is kept. From a
|
|
471
|
+
# foreign or damaged row: an alert whose `at` is not a number, and a
|
|
472
|
+
# recovery whose `details.after` is not a list of conditions, are stale.
|
|
448
473
|
def stale_alert?(alert, state)
|
|
449
|
-
|
|
474
|
+
if alert.type == :recovered
|
|
475
|
+
# One whose details say nothing of what it recovers from cannot be judged, and goes.
|
|
476
|
+
after = alert.details.is_a?(Hash) ? alert.details[:after] : nil
|
|
477
|
+
return true unless after.is_a?(Array)
|
|
450
478
|
|
|
451
|
-
|
|
479
|
+
return after.any? { |condition| !condition.is_a?(Symbol) || state.open.key?(condition) }
|
|
480
|
+
end
|
|
481
|
+
# One with no time cannot match an open condition.
|
|
482
|
+
!json_number?(alert.at) || state.open[alert.type] != alert.at
|
|
452
483
|
end
|
|
453
484
|
|
|
454
485
|
# How a job looks at a glance. Silence wins, then stuck, failing and late.
|
data/lib/cronwatch/flight.rb
CHANGED
data/lib/cronwatch/format.rb
CHANGED
data/lib/cronwatch/http.rb
CHANGED
|
@@ -7,8 +7,15 @@ require "uri"
|
|
|
7
7
|
module Cronwatch
|
|
8
8
|
# POSTs for the alert channels. Anything with `post(url, body, headers)`
|
|
9
9
|
# returning a Response can stand in for it, which is how the tests run.
|
|
10
|
+
#
|
|
11
|
+
# @api private
|
|
10
12
|
module HTTP
|
|
11
13
|
TIMEOUT = 10
|
|
14
|
+
# The most of an answer's body read (1 MiB): a channel reads 200
|
|
15
|
+
# characters of a refusal, and a hostile or broken endpoint could
|
|
16
|
+
# otherwise stream gigabytes into the worker within the deadline.
|
|
17
|
+
# Reading stops there. A 2xx answer's body is not read at all.
|
|
18
|
+
MAX_BODY = 1_048_576
|
|
12
19
|
|
|
13
20
|
Response = Struct.new(:status, :body, keyword_init: true) do
|
|
14
21
|
def ok? = status >= 200 && status < 300
|
|
@@ -33,6 +40,8 @@ module Cronwatch
|
|
|
33
40
|
# sends. Past the deadline before an answer, raises HTTP::TimeoutError;
|
|
34
41
|
# past it while the body is still arriving, returns the answer with an
|
|
35
42
|
# empty body, as the SDK's channels treat a body they could not read.
|
|
43
|
+
# A 2xx answer comes back with an empty body, unread, since no channel
|
|
44
|
+
# needs it; any other with at most MAX_BODY bytes of it.
|
|
36
45
|
def post(url, body, headers)
|
|
37
46
|
uri = HTTP.postable(url)
|
|
38
47
|
deadline = HTTP.monotonic + @timeout
|
|
@@ -46,7 +55,11 @@ module Cronwatch
|
|
|
46
55
|
conn.read_timeout = remaining
|
|
47
56
|
conn.write_timeout = remaining
|
|
48
57
|
conn.request(request) do |response|
|
|
49
|
-
|
|
58
|
+
status = response.code.to_i
|
|
59
|
+
# Returning from inside the block leaves the rest unread; the connection closes after.
|
|
60
|
+
return Response.new(status: status, body: "") if status >= 200 && status < 300
|
|
61
|
+
|
|
62
|
+
return Response.new(status: status, body: read_body(conn, response, deadline))
|
|
50
63
|
end
|
|
51
64
|
end
|
|
52
65
|
rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
|
|
@@ -65,17 +78,20 @@ module Cronwatch
|
|
|
65
78
|
|
|
66
79
|
private
|
|
67
80
|
|
|
68
|
-
# The body, read in chunks until
|
|
81
|
+
# The body, read in chunks until it ends or MAX_BODY bytes have come,
|
|
82
|
+
# whichever is first; "" when the deadline passes first.
|
|
69
83
|
def read_body(conn, response, deadline)
|
|
70
|
-
|
|
84
|
+
body = "".b
|
|
71
85
|
conn.read_timeout = HTTP.remaining(deadline)
|
|
72
86
|
response.read_body do |chunk|
|
|
73
87
|
raise TimeoutError if HTTP.monotonic >= deadline
|
|
74
88
|
|
|
75
|
-
|
|
89
|
+
body << chunk.b.byteslice(0, MAX_BODY - body.bytesize)
|
|
90
|
+
break if body.bytesize >= MAX_BODY
|
|
91
|
+
|
|
76
92
|
conn.read_timeout = HTTP.remaining(deadline)
|
|
77
93
|
end
|
|
78
|
-
|
|
94
|
+
body
|
|
79
95
|
rescue Net::ReadTimeout, TimeoutError
|
|
80
96
|
""
|
|
81
97
|
end
|
data/lib/cronwatch/job.rb
CHANGED
|
@@ -95,6 +95,8 @@ module Cronwatch
|
|
|
95
95
|
end
|
|
96
96
|
|
|
97
97
|
# Collects a run's output and metrics while its block runs.
|
|
98
|
+
#
|
|
99
|
+
# @api private
|
|
98
100
|
class RunRecorder
|
|
99
101
|
# Lines are dropped from the front once the output is well past the cap;
|
|
100
102
|
# Output.redact_and_cap trims it exactly at the end.
|
data/lib/cronwatch/js.rb
CHANGED
|
@@ -6,6 +6,8 @@ module Cronwatch
|
|
|
6
6
|
# webhook bodies, so the gem reproduces them byte for byte: Math.round,
|
|
7
7
|
# String(number), JSON.stringify, String.prototype.trim and lengths counted
|
|
8
8
|
# in UTF-16 code units.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
9
11
|
module JS
|
|
10
12
|
# What JavaScript's \s and trim() treat as whitespace.
|
|
11
13
|
WHITESPACE = "\\t\\n\\v\\f\\r \\u00a0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff"
|
|
@@ -40,8 +42,10 @@ module Cronwatch
|
|
|
40
42
|
value - floor >= 0.5 ? floor + 1 : floor
|
|
41
43
|
end
|
|
42
44
|
|
|
45
|
+
# Number.isFinite of the number as JavaScript would hold it: an Integer
|
|
46
|
+
# past a double's range is Infinity there, so it is not finite here.
|
|
43
47
|
def finite?(value)
|
|
44
|
-
value.is_a?(
|
|
48
|
+
value.is_a?(Numeric) && value.real? && value.to_f.finite?
|
|
45
49
|
end
|
|
46
50
|
|
|
47
51
|
# Number.isInteger.
|
|
@@ -159,12 +163,44 @@ module Cronwatch
|
|
|
159
163
|
end
|
|
160
164
|
|
|
161
165
|
def quote(text)
|
|
166
|
+
if text.encoding == Encoding::UTF_8 && !text.valid_encoding? && text.b.match?(LONE)
|
|
167
|
+
# A lone surrogate (see lone_surrogate), written as JSON.stringify writes one.
|
|
168
|
+
parts = text.b.split(LONE_SPLIT).each_with_index.map do |part, i|
|
|
169
|
+
next quote(part.force_encoding(Encoding::UTF_8))[1...-1] if i.even?
|
|
170
|
+
|
|
171
|
+
bytes = part.bytes
|
|
172
|
+
format("\\u%04x", ((bytes[0] & 0x0F) << 12) | ((bytes[1] & 0x3F) << 6) | (bytes[2] & 0x3F))
|
|
173
|
+
end
|
|
174
|
+
return "\"#{parts.join}\""
|
|
175
|
+
end
|
|
162
176
|
text = text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) unless text.encoding == Encoding::UTF_8 && text.valid_encoding?
|
|
163
177
|
text = text.scrub unless text.valid_encoding?
|
|
164
178
|
escaped = text.gsub(/["\\\u0000-\u001f]/) { |c| ESCAPES[c] || format("\\u%04x", c.ord) }
|
|
165
179
|
"\"#{escaped}\""
|
|
166
180
|
end
|
|
167
181
|
|
|
182
|
+
# A surrogate code unit on its own, in the bytes UTF-8 would give it.
|
|
183
|
+
# They make the string invalid UTF-8, which Ruby's own methods refuse, so
|
|
184
|
+
# one is only ever the last thing added to text about to be written:
|
|
185
|
+
# quote writes it as \udXXX, as JSON.stringify does.
|
|
186
|
+
LONE = /\xED[\xA0-\xBF][\x80-\xBF]/n
|
|
187
|
+
LONE_SPLIT = /(\xED[\xA0-\xBF][\x80-\xBF])/n
|
|
188
|
+
|
|
189
|
+
def lone_surrogate(unit)
|
|
190
|
+
[0xED, 0x80 | ((unit >> 6) & 0x3F), 0x80 | (unit & 0x3F)].pack("C*").force_encoding(Encoding::UTF_8)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# String.prototype.slice(0, units) as JavaScript gives it: the first
|
|
194
|
+
# `units` UTF-16 code units, and the high surrogate of a pair the cut
|
|
195
|
+
# splits. Returns the head Ruby can hold (head16) and that surrogate's
|
|
196
|
+
# code unit, or nil when the cut splits no pair.
|
|
197
|
+
def slice16(text, units)
|
|
198
|
+
head = head16(text, units)
|
|
199
|
+
return [head, nil] if text.ascii_only? || head.length == text.length || length16(head) == units
|
|
200
|
+
|
|
201
|
+
[head, 0xD800 + ((text[head.length].ord - 0x10000) >> 10)]
|
|
202
|
+
end
|
|
203
|
+
|
|
168
204
|
# Property order: array-index keys ascending, then the rest as inserted.
|
|
169
205
|
def object_keys(hash)
|
|
170
206
|
keys = hash.keys
|
|
@@ -184,5 +220,42 @@ module Cronwatch
|
|
|
184
220
|
def parse(text)
|
|
185
221
|
::JSON.parse(text)
|
|
186
222
|
end
|
|
223
|
+
|
|
224
|
+
DECIMAL = /\A[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?\z/
|
|
225
|
+
RADIX = { "x" => 16, "o" => 8, "b" => 2 }.freeze
|
|
226
|
+
|
|
227
|
+
# Number(string): decimal, 0x/0o/0b, Infinity, or NaN; 0 for a string
|
|
228
|
+
# of only whitespace.
|
|
229
|
+
def to_number(value)
|
|
230
|
+
text = trim(value.dup.force_encoding(Encoding::UTF_8).scrub)
|
|
231
|
+
return 0 if text.empty?
|
|
232
|
+
# Ruby's Float() wants a digit on both sides of the point; JavaScript does not.
|
|
233
|
+
return Float(text.sub(/\A([+-]?)\./, "\\10.").sub(/\.(?=[eE]|\z)/, ".0")) if DECIMAL.match?(text)
|
|
234
|
+
return text.start_with?("-") ? -Float::INFINITY : Float::INFINITY if /\A[+-]?Infinity\z/.match?(text)
|
|
235
|
+
|
|
236
|
+
if (m = /\A0([xXoObB])([0-9a-fA-F]+)\z/.match(text))
|
|
237
|
+
return Integer(m[2], RADIX.fetch(m[1].downcase))
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
Float::NAN
|
|
241
|
+
rescue ArgumentError
|
|
242
|
+
Float::NAN
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# A time or a count of milliseconds as a SQL column holds it: an Integer
|
|
246
|
+
# or a Float kept, a String that is not blank read as Number() reads it
|
|
247
|
+
# (Postgres hands BIGINT over as text), and nil when the result is not a
|
|
248
|
+
# finite number. A whole number in a Float comes back an Integer.
|
|
249
|
+
def finite_number(value)
|
|
250
|
+
n =
|
|
251
|
+
case value
|
|
252
|
+
when Integer, Float then value
|
|
253
|
+
when String then trim(value.dup.force_encoding(Encoding::UTF_8).scrub).empty? ? nil : to_number(value)
|
|
254
|
+
end
|
|
255
|
+
return nil unless finite?(n)
|
|
256
|
+
return n.to_i if n.is_a?(Float) && n == n.floor && n.abs <= MAX_SAFE_INTEGER
|
|
257
|
+
|
|
258
|
+
n
|
|
259
|
+
end
|
|
187
260
|
end
|
|
188
261
|
end
|
data/lib/cronwatch/monitored.rb
CHANGED
|
@@ -6,6 +6,8 @@ module Cronwatch
|
|
|
6
6
|
# declared on Cronwatch.client. Needs nothing beyond the core, so Sidekiq
|
|
7
7
|
# without Rails can use it. Loaded by cronwatch/scheduler, which schedule:
|
|
8
8
|
# :from_scheduler and declare_from_scheduler! need.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
9
11
|
module Monitored
|
|
10
12
|
# What `cronwatch` is outside a monitored run: it takes log and metric
|
|
11
13
|
# calls and drops them, so the job's code runs the same either way.
|
|
@@ -51,7 +53,10 @@ module Cronwatch
|
|
|
51
53
|
client = Cronwatch.client
|
|
52
54
|
@lock.synchronize do
|
|
53
55
|
current = @registration
|
|
54
|
-
|
|
56
|
+
# Kept while the client still holds the job: one forgotten since
|
|
57
|
+
# (the dashboard's forget) is declared again, with its schedule,
|
|
58
|
+
# as the other ports' bridges declare it again before a check.
|
|
59
|
+
return current if current && current[0].equal?(client) && client.defined_jobs.any? { |d| d.name == @name }
|
|
55
60
|
|
|
56
61
|
handle = client.job(@name, **resolve_locked)
|
|
57
62
|
@registration = [client, handle].freeze
|
data/lib/cronwatch/output.rb
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Cronwatch
|
|
4
|
+
# A run's output and error on their way to the store: redaction, the cap,
|
|
5
|
+
# NUL and encoding. `redact_secrets` is public, the default `redact:` and
|
|
6
|
+
# the one to call from a redact of your own; the rest is @api private.
|
|
4
7
|
module Output
|
|
5
8
|
# Output is capped so a chatty job cannot fill the store. The tail is kept.
|
|
6
9
|
# Counted in UTF-16 code units, as the SDK counts it.
|
data/lib/cronwatch/pg_cron.rb
CHANGED
|
@@ -26,6 +26,8 @@ module Cronwatch
|
|
|
26
26
|
# A job that is renamed, unscheduled or no longer picked keeps its old
|
|
27
27
|
# name's runs and history, and that name is declared again without a
|
|
28
28
|
# schedule, so it is never reported missed. Its description says why.
|
|
29
|
+
# Forgetting that old name from the dashboard while a run of it is still
|
|
30
|
+
# open lets the run go: it is not recorded, and no error is reported.
|
|
29
31
|
#
|
|
30
32
|
# require "cronwatch/pg_cron"
|
|
31
33
|
# Cronwatch.configure do |c|
|
|
@@ -86,6 +88,8 @@ module Cronwatch
|
|
|
86
88
|
# The options of a definition that are declared again, without its schedule, for a name no longer in use.
|
|
87
89
|
UNSCHEDULED = %i[description tags grace timeout max_duration budget failures_before_alert].freeze
|
|
88
90
|
DESCRIBED = /\Apg_cron job (\d+) in /
|
|
91
|
+
private_constant :BACKFILL, :COLUMNS, :DESCRIBED, :JOBS_SQL, :MAX_PAGES, :NEWEST_SQL, :PAGE, :REBOOT, :RUNS_SQL,
|
|
92
|
+
:SCHEDULE_ONLY, :SECONDS, :SETTING_SQL, :UNSCHEDULED, :UTC
|
|
89
93
|
|
|
90
94
|
attr_reader :name
|
|
91
95
|
|
|
@@ -111,6 +115,8 @@ module Cronwatch
|
|
|
111
115
|
@declared = {}
|
|
112
116
|
# Names declared again without a schedule by retire, whose open runs are still read.
|
|
113
117
|
@retired = Set.new
|
|
118
|
+
# The names the host declares, read on each sync after the retires (nil for a host without defined_jobs).
|
|
119
|
+
@declared_now = nil
|
|
114
120
|
@scanned = false
|
|
115
121
|
@warned = Set.new
|
|
116
122
|
# Jobids whose callback failed, reported once until it works again.
|
|
@@ -204,6 +210,8 @@ module Cronwatch
|
|
|
204
210
|
end
|
|
205
211
|
|
|
206
212
|
# Queries through the pg gem's PG::Connection#exec_params.
|
|
213
|
+
#
|
|
214
|
+
# @api private
|
|
207
215
|
class PGConnection
|
|
208
216
|
def initialize(connection)
|
|
209
217
|
@connection = connection
|
|
@@ -217,6 +225,8 @@ module Cronwatch
|
|
|
217
225
|
|
|
218
226
|
# Queries through an ActiveRecord connection's exec_query, checking one
|
|
219
227
|
# out of the pool for each query when given a class or a pool.
|
|
228
|
+
#
|
|
229
|
+
# @api private
|
|
220
230
|
class ActiveRecordConnection
|
|
221
231
|
def initialize(source)
|
|
222
232
|
@source = source
|
|
@@ -278,6 +288,10 @@ module Cronwatch
|
|
|
278
288
|
retire_unused(host, names, definitions, all)
|
|
279
289
|
return [] if !recording || names.empty?
|
|
280
290
|
|
|
291
|
+
# The names declared now, after the retires above. A run copied under
|
|
292
|
+
# a retired name that was then forgotten (the dashboard's forget) has
|
|
293
|
+
# no job to go to: it is let go, never recorded and never read again.
|
|
294
|
+
@declared_now = host.respond_to?(:defined_jobs) ? host.defined_jobs.to_set(&:name) : nil
|
|
281
295
|
alerts = []
|
|
282
296
|
start_cursors(host, names, alerts, now)
|
|
283
297
|
read_new(host, names, alerts, now)
|
|
@@ -451,10 +465,13 @@ module Cronwatch
|
|
|
451
465
|
visible = all.map(&:jobid).to_set
|
|
452
466
|
host.store.list_jobs.each do |stored|
|
|
453
467
|
definition = stored.definition
|
|
468
|
+
# A foreign or damaged definition (not an object, tags not a list) is not one of ours.
|
|
469
|
+
next if !definition.is_a?(JobDefinition) || definition.unreadable? || !definition.tags.is_a?(Array)
|
|
454
470
|
next if !stored.name.start_with?(@prefix) || in_use.include?(stored.name)
|
|
455
|
-
next if definition.schedule.nil? || definition.schedule.to_s.empty? || !
|
|
471
|
+
next if definition.schedule.nil? || definition.schedule.to_s.empty? || !definition.tags.include?("pg_cron")
|
|
456
472
|
|
|
457
|
-
|
|
473
|
+
description = definition.description
|
|
474
|
+
match = DESCRIBED.match(description.is_a?(String) ? description : "")
|
|
458
475
|
next unless match
|
|
459
476
|
|
|
460
477
|
jobid = match[1].to_i
|
|
@@ -477,8 +494,10 @@ module Cronwatch
|
|
|
477
494
|
runid = Integer(row["runid"])
|
|
478
495
|
jobid = Integer(row["jobid"])
|
|
479
496
|
name = @pending[runid] || names[jobid]
|
|
480
|
-
|
|
497
|
+
if name.nil? || (@declared_now && !names.value?(name) && !@declared_now.include?(name))
|
|
498
|
+
@pending.delete(runid)
|
|
481
499
|
@held.delete(runid)
|
|
500
|
+
@retired.delete(name) if name
|
|
482
501
|
return
|
|
483
502
|
end
|
|
484
503
|
if row["start_time"].nil? && !PgCron.finished_status?(row["status"])
|
|
@@ -7,7 +7,7 @@ require_relative "../scheduler"
|
|
|
7
7
|
|
|
8
8
|
module Cronwatch
|
|
9
9
|
# Monitors an ActiveJob class. Each perform is a recorded run with the
|
|
10
|
-
# trigger "
|
|
10
|
+
# trigger "active-job"; `cronwatch` in the job is the run's context, for
|
|
11
11
|
# log and metric. A perform that raises is recorded as failed and then
|
|
12
12
|
# raises as before, so retry_on, discard_on and error reporters see it
|
|
13
13
|
# unchanged.
|
|
@@ -34,7 +34,7 @@ module Cronwatch
|
|
|
34
34
|
extend ActiveSupport::Concern
|
|
35
35
|
include Cronwatch::Monitored
|
|
36
36
|
|
|
37
|
-
TRIGGER =
|
|
37
|
+
TRIGGER = Scheduler::ACTIVE_JOB_TRIGGER
|
|
38
38
|
|
|
39
39
|
included do
|
|
40
40
|
around_perform :cronwatch_perform
|
data/lib/cronwatch/schedule.rb
CHANGED
|
@@ -11,6 +11,8 @@ module Cronwatch
|
|
|
11
11
|
# is accepted and how a fire lands on a clock that jumps (daylight saving),
|
|
12
12
|
# so both are ported here and a Node and a Ruby process sharing one store
|
|
13
13
|
# agree on every due time.
|
|
14
|
+
#
|
|
15
|
+
# @api private
|
|
14
16
|
module Schedule
|
|
15
17
|
# How early a run may start and still count for the fire it was meant for.
|
|
16
18
|
EARLY_SLACK_MS = 60_000
|
|
@@ -29,6 +31,8 @@ module Cronwatch
|
|
|
29
31
|
|
|
30
32
|
# The compiled cron fields, kept out of the public shape.
|
|
31
33
|
attr_accessor :pattern
|
|
34
|
+
# Whether the cron names a date no month has ("0 0 30 2 *"), and so never fires.
|
|
35
|
+
attr_accessor :never
|
|
32
36
|
|
|
33
37
|
def cron? = kind == :cron
|
|
34
38
|
def interval? = kind == :interval
|
|
@@ -47,12 +51,21 @@ module Cronwatch
|
|
|
47
51
|
def to_h = { "dueAt" => due_at, "deadline" => deadline }
|
|
48
52
|
end
|
|
49
53
|
|
|
54
|
+
# Where a cron is asked once whether it ever fires: more than a 400-year
|
|
55
|
+
# cycle of the calendar before the year 3000, as in schedule.ts.
|
|
56
|
+
PROBE_FROM_MS = 946_684_800_000 # 2000-01-01T00:00:00Z
|
|
57
|
+
# Text that starts like an ISO date and time, which croner would take for a one-time date.
|
|
58
|
+
ONE_TIME_DATE = /\A\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/
|
|
59
|
+
|
|
50
60
|
@cache = {}
|
|
51
61
|
@lock = Mutex.new
|
|
52
62
|
|
|
53
63
|
module_function
|
|
54
64
|
|
|
55
|
-
#
|
|
65
|
+
# Text with a colon after its first character, which croner would take
|
|
66
|
+
# for a one-time date, is refused. A cron that names a date no month has
|
|
67
|
+
# ("0 0 30 2 *") is read, and never fires: nothing is ever due, so it is
|
|
68
|
+
# never missed. Parsed once per (schedule, timezone) pair and cached. Without a timezone
|
|
56
69
|
# the expression is read in the process timezone, like crontab. Vercel and
|
|
57
70
|
# GitHub Actions run their crons in UTC, so pass timezone: "UTC" for those.
|
|
58
71
|
def parse(schedule, timezone = nil)
|
|
@@ -68,6 +81,13 @@ module Cronwatch
|
|
|
68
81
|
|
|
69
82
|
parsed = Parsed.new(kind: :interval, source: text, every_ms: every_ms)
|
|
70
83
|
else
|
|
84
|
+
# croner reads text with a colon after its first character as a
|
|
85
|
+
# one-time date to fire at, not as a cron expression. A cron monitor
|
|
86
|
+
# has no use for one, so every such text is refused, as in the SDK.
|
|
87
|
+
if text[1..].to_s.include?(":")
|
|
88
|
+
reason = ONE_TIME_DATE.match?(text) ? "CronPattern: a one-time date is not supported" : "Invalid ISO8601 passed to timezone parser."
|
|
89
|
+
raise ArgumentError, "schedule \"#{schedule}\" is not a cron expression or \"every <duration>\": #{reason}"
|
|
90
|
+
end
|
|
71
91
|
begin
|
|
72
92
|
pattern = CronPattern.compile(text)
|
|
73
93
|
rescue ArgumentError => e
|
|
@@ -75,6 +95,8 @@ module Cronwatch
|
|
|
75
95
|
end
|
|
76
96
|
parsed = Parsed.new(kind: :cron, source: text, timezone: timezone.nil? || timezone == "" ? nil : timezone)
|
|
77
97
|
parsed.pattern = pattern
|
|
98
|
+
# The calendar alone decides it, so it is asked in UTC.
|
|
99
|
+
parsed.never = pattern.next_after(PROBE_FROM_MS, "UTC").nil?
|
|
78
100
|
end
|
|
79
101
|
parsed.freeze
|
|
80
102
|
@lock.synchronize { @cache[key] = parsed }
|
|
@@ -88,6 +110,7 @@ module Cronwatch
|
|
|
88
110
|
# stepped over an hour at a time. Ported from fireAfter in schedule.ts.
|
|
89
111
|
def fire_after(parsed, from)
|
|
90
112
|
raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
|
|
113
|
+
return nil if parsed.never
|
|
91
114
|
|
|
92
115
|
start = count_from(from)
|
|
93
116
|
return nil if start.nil?
|
|
@@ -111,6 +134,7 @@ module Cronwatch
|
|
|
111
134
|
# any that do not move forward (see fire_after).
|
|
112
135
|
def fires_between(parsed, from, to, limit)
|
|
113
136
|
raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
|
|
137
|
+
return [] if parsed.never
|
|
114
138
|
|
|
115
139
|
out = []
|
|
116
140
|
start = count_from(from)
|
data/lib/cronwatch/scheduler.rb
CHANGED
|
@@ -40,6 +40,11 @@ module Cronwatch
|
|
|
40
40
|
end
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
+
# The trigger of a run an ActiveJob perform records, through
|
|
44
|
+
# Cronwatch::ActiveJob or a scheduler's entry. Runs recorded before 1.0
|
|
45
|
+
# carry "active_job".
|
|
46
|
+
ACTIVE_JOB_TRIGGER = "active-job"
|
|
47
|
+
|
|
43
48
|
# How far ahead the daylight saving check looks, and how far either side
|
|
44
49
|
# of each clock change it compares the scheduler's runs with CronWatch's.
|
|
45
50
|
HORIZON_YEARS = 5
|
|
@@ -622,7 +627,7 @@ module Cronwatch
|
|
|
622
627
|
::ActiveJob::Base.around_perform do |job, block|
|
|
623
628
|
declaration = Cronwatch::Scheduler.declaration_for_active_job(job)
|
|
624
629
|
if declaration
|
|
625
|
-
Cronwatch::Monitored.record(declaration,
|
|
630
|
+
Cronwatch::Monitored.record(declaration, ACTIVE_JOB_TRIGGER, job) { block.call }
|
|
626
631
|
else
|
|
627
632
|
block.call
|
|
628
633
|
end
|