cronwatch 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +20 -11
- data/lib/cronwatch/alerts/datadog.rb +1 -0
- data/lib/cronwatch/alerts/discord.rb +34 -2
- data/lib/cronwatch/alerts/email.rb +2 -0
- data/lib/cronwatch/alerts/honeybadger.rb +1 -0
- data/lib/cronwatch/alerts/postmark.rb +1 -0
- data/lib/cronwatch/alerts/provider.rb +2 -0
- data/lib/cronwatch/alerts/resend.rb +1 -0
- data/lib/cronwatch/alerts/rollbar.rb +1 -0
- data/lib/cronwatch/alerts/sentry.rb +1 -0
- data/lib/cronwatch/alerts/sigv4.rb +2 -0
- data/lib/cronwatch/alerts/slack.rb +1 -0
- data/lib/cronwatch/alerts/twilio.rb +1 -0
- data/lib/cronwatch/alerts/webhook.rb +18 -5
- data/lib/cronwatch/client.rb +275 -110
- data/lib/cronwatch/cron_pattern.rb +17 -8
- data/lib/cronwatch/deprecation.rb +18 -0
- data/lib/cronwatch/duration.rb +2 -0
- data/lib/cronwatch/environment.rb +109 -7
- data/lib/cronwatch/evaluate.rb +145 -10
- data/lib/cronwatch/flight.rb +6 -0
- data/lib/cronwatch/format.rb +1 -0
- data/lib/cronwatch/http.rb +21 -5
- data/lib/cronwatch/job.rb +7 -2
- data/lib/cronwatch/js.rb +74 -1
- data/lib/cronwatch/monitored.rb +6 -1
- data/lib/cronwatch/output.rb +28 -1
- data/lib/cronwatch/pg_cron.rb +29 -6
- data/lib/cronwatch/rails/active_job.rb +2 -2
- data/lib/cronwatch/schedule.rb +25 -1
- data/lib/cronwatch/scheduler.rb +79 -11
- data/lib/cronwatch/serialize.rb +1 -0
- data/lib/cronwatch/stats.rb +1 -0
- data/lib/cronwatch/stores/active_record.rb +37 -11
- data/lib/cronwatch/ticker.rb +14 -0
- data/lib/cronwatch/triage/anthropic.rb +2 -0
- data/lib/cronwatch/types.rb +199 -27
- data/lib/cronwatch/version.rb +1 -1
- data/lib/cronwatch/walker.rb +2 -0
- data/lib/cronwatch/web/app.rb +114 -37
- data/lib/cronwatch/web/html.rb +9 -5
- data/lib/cronwatch/web/origin.rb +2 -0
- data/lib/cronwatch/web/pwa.rb +2 -0
- data/lib/cronwatch/web/timeline.rb +2 -0
- data/lib/cronwatch/web.rb +1 -2
- data/lib/cronwatch/zone.rb +2 -0
- data/lib/cronwatch.rb +2 -0
- data/lib/generators/cronwatch/install/install_generator.rb +52 -8
- metadata +2 -1
|
@@ -7,6 +7,8 @@ require_relative "walker"
|
|
|
7
7
|
module Cronwatch
|
|
8
8
|
# A cron expression's fields, and the next time they match. What is
|
|
9
9
|
# accepted, and which times match, follow croner; Fugit parses the values.
|
|
10
|
+
#
|
|
11
|
+
# @api private
|
|
10
12
|
class CronPattern
|
|
11
13
|
NICKNAMES = {
|
|
12
14
|
"@yearly" => "0 0 1 1 *", "@annually" => "0 0 1 1 *", "@monthly" => "0 0 1 * *", "@weekly" => "0 0 * * 0",
|
|
@@ -27,14 +29,15 @@ module Cronwatch
|
|
|
27
29
|
|
|
28
30
|
def initialize(text)
|
|
29
31
|
fields, dom, dow = croner_fields(text)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
# Fugit refuses a date no month has ("30 2"), and drops a month the
|
|
33
|
+
# days cannot fall in ("31 2,3" reads as March alone). Croner keeps
|
|
34
|
+
# both, so the days are read with every month and the months with
|
|
35
|
+
# every day, and a date no month has is a cron that never fires.
|
|
36
|
+
cron = fugit(fields.each_with_index.map { |f, i| i == 3 ? dom : (i == 4 ? "*" : (i == 5 ? dow : f)) })
|
|
37
|
+
months = fugit(["0", "0", "0", "*", fields[4], "*"])
|
|
35
38
|
# Croner accepts a few forms Fugit does not read (a range with #, a day of
|
|
36
|
-
# the month with L after it
|
|
37
|
-
raise ArgumentError, "CronPattern: '#{text}' uses a form the Ruby port does not read" unless cron
|
|
39
|
+
# the month with L after it). Those are refused.
|
|
40
|
+
raise ArgumentError, "CronPattern: '#{text}' uses a form the Ruby port does not read" unless cron && months
|
|
38
41
|
|
|
39
42
|
@seconds = flags(cron.seconds, 60, 0)
|
|
40
43
|
@minutes = flags(cron.minutes, 60, 0)
|
|
@@ -45,7 +48,7 @@ module Cronwatch
|
|
|
45
48
|
else
|
|
46
49
|
cron.monthdays.each { |d| @days[d - 1] = true if d.positive? }
|
|
47
50
|
end
|
|
48
|
-
@months = flags(
|
|
51
|
+
@months = flags(months.months, 12, 1)
|
|
49
52
|
@weekdays = Array.new(7, 0)
|
|
50
53
|
if cron.weekdays.nil?
|
|
51
54
|
@weekdays.fill(ANY_NTH)
|
|
@@ -75,6 +78,12 @@ module Cronwatch
|
|
|
75
78
|
|
|
76
79
|
private
|
|
77
80
|
|
|
81
|
+
def fugit(fields)
|
|
82
|
+
Fugit::Cron.parse(fields.join(" "))
|
|
83
|
+
rescue StandardError
|
|
84
|
+
nil
|
|
85
|
+
end
|
|
86
|
+
|
|
78
87
|
def flags(values, size, base)
|
|
79
88
|
return Array.new(size, true) if values.nil?
|
|
80
89
|
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cronwatch
|
|
4
|
+
# Warns that a name is deprecated: it still works through 1.x and goes in
|
|
5
|
+
# 2.0. The warning is Ruby's own deprecation category, so it shows where
|
|
6
|
+
# Ruby's deprecation warnings show (`ruby -w`, `-W:deprecated`, or
|
|
7
|
+
# `Warning[:deprecated] = true`) and points at the caller's line.
|
|
8
|
+
#
|
|
9
|
+
# @api private
|
|
10
|
+
module Deprecation
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def warn(old, replacement, uplevel: 2)
|
|
14
|
+
Kernel.warn("[cronwatch] #{old} is deprecated and goes in 2.0; use #{replacement}",
|
|
15
|
+
uplevel: uplevel, category: :deprecated)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
data/lib/cronwatch/duration.rb
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Cronwatch
|
|
4
4
|
# "15m", "1h30m", "90s", "2d", or a number of milliseconds.
|
|
5
|
+
#
|
|
6
|
+
# @api private
|
|
5
7
|
module Duration
|
|
6
8
|
UNIT_MS = { "ms" => 1, "s" => 1000, "m" => 60_000, "h" => 3_600_000, "d" => 86_400_000, "w" => 604_800_000 }.freeze
|
|
7
9
|
PART = Regexp.new("(\\d+(?:\\.\\d+)?)[#{JS::WHITESPACE}]*(ms|s|m|h|d|w)")
|
|
@@ -1,14 +1,91 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Cronwatch
|
|
4
|
-
# The app's environment, read
|
|
5
|
-
#
|
|
4
|
+
# The app's environment, read as every CronWatch library reads it: the
|
|
5
|
+
# first of CRONWATCH_ENV, APP_ENV, then the app's own (Rails.env when Rails
|
|
6
|
+
# is loaded, else RAILS_ENV, then RACK_ENV) that holds more than spaces,
|
|
7
|
+
# trimmed and lowercased, with "prod" read as "production" and "dev",
|
|
8
|
+
# "local", "test" and "testing" as "development". None set is neither,
|
|
9
|
+
# which is the safe reading. The SDK reads NODE_ENV where this reads the
|
|
10
|
+
# app's own.
|
|
11
|
+
#
|
|
12
|
+
# @api private
|
|
6
13
|
module Environment
|
|
7
|
-
|
|
14
|
+
# The variables read before the app's own, in order.
|
|
15
|
+
SHARED = %w[CRONWATCH_ENV APP_ENV].freeze
|
|
16
|
+
ALIASES = {
|
|
17
|
+
"prod" => "production", "dev" => "development", "local" => "development", "test" => "development",
|
|
18
|
+
"testing" => "development",
|
|
19
|
+
}.freeze
|
|
20
|
+
private_constant :SHARED, :ALIASES
|
|
8
21
|
|
|
9
22
|
module_function
|
|
10
23
|
|
|
11
|
-
# "production", "development", ... or nil when nothing names one.
|
|
24
|
+
# "production", "development", "staging", ... or nil when nothing names one.
|
|
25
|
+
def environment
|
|
26
|
+
_, value = source
|
|
27
|
+
return nil if value.nil?
|
|
28
|
+
|
|
29
|
+
value = JS.trim(value).downcase
|
|
30
|
+
ALIASES.fetch(value, value)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Where the environment was read from and its value as set:
|
|
34
|
+
# ["CRONWATCH_ENV", " dev"], ["Rails.env", "test"], or nil.
|
|
35
|
+
def source
|
|
36
|
+
SHARED.each do |variable|
|
|
37
|
+
value = ENV.fetch(variable, nil)
|
|
38
|
+
return [variable, value] if present?(value)
|
|
39
|
+
end
|
|
40
|
+
if defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
41
|
+
env = ::Rails.env.to_s
|
|
42
|
+
return ["Rails.env", env] if present?(env)
|
|
43
|
+
end
|
|
44
|
+
%w[RAILS_ENV RACK_ENV].each do |variable|
|
|
45
|
+
value = ENV.fetch(variable, nil)
|
|
46
|
+
return [variable, value] if present?(value)
|
|
47
|
+
end
|
|
48
|
+
nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def present?(value)
|
|
52
|
+
!value.nil? && !blank?(value)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Whether a String is empty or only whitespace, as JavaScript's
|
|
56
|
+
# String.prototype.trim sees it (JS::WHITESPACE). Bytes that are not
|
|
57
|
+
# UTF-8 are not whitespace.
|
|
58
|
+
def blank?(value)
|
|
59
|
+
JS.trim(value.dup.force_encoding(Encoding::UTF_8).scrub).empty?
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# A secret from the environment (CRONWATCH_TOKEN, CRON_SECRET): nil when
|
|
63
|
+
# the variable is unset, empty or only whitespace, so a blank value counts
|
|
64
|
+
# as not set and the routes fail closed. Any other value is used as it
|
|
65
|
+
# is, untrimmed.
|
|
66
|
+
def secret(variable)
|
|
67
|
+
value = ENV.fetch(variable, nil)
|
|
68
|
+
present?(value) ? value : nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# A token or secret passed in code: a String, or nil. A String that is
|
|
72
|
+
# empty or only whitespace counts as not given (nil here; the caller
|
|
73
|
+
# tells that from nil the opt-out). Anything else (false, a number, a
|
|
74
|
+
# Symbol) raises TypeError naming the option, so it never becomes a
|
|
75
|
+
# password.
|
|
76
|
+
def secret_option(value, what)
|
|
77
|
+
return nil if value.nil?
|
|
78
|
+
unless value.is_a?(String)
|
|
79
|
+
kind = [true, false].include?(value) ? value.to_s : value.class.name
|
|
80
|
+
raise TypeError, "#{what} must be a String, or nil to opt out, not #{kind}"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
blank?(value) ? nil : value
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# The framework's own name for the environment, as it sets it: Rails.env
|
|
87
|
+
# when Rails is loaded, otherwise RAILS_ENV, then RACK_ENV, or nil. The
|
|
88
|
+
# section of a Solid Queue file is chosen by it, as Solid Queue does.
|
|
12
89
|
def name
|
|
13
90
|
if defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
14
91
|
env = ::Rails.env.to_s
|
|
@@ -17,13 +94,38 @@ module Cronwatch
|
|
|
17
94
|
[ENV.fetch("RAILS_ENV", nil), ENV.fetch("RACK_ENV", nil)].find { |value| value && !value.empty? }
|
|
18
95
|
end
|
|
19
96
|
|
|
20
|
-
# Development
|
|
97
|
+
# Development (including test): Cronwatch::Web without a token serves only then.
|
|
21
98
|
def development?
|
|
22
|
-
|
|
99
|
+
environment == "development"
|
|
23
100
|
end
|
|
24
101
|
|
|
25
102
|
def production?
|
|
26
|
-
|
|
103
|
+
environment == "production"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Whether development was named by the app rather than by its server,
|
|
107
|
+
# for the dashboard's own token (Cronwatch::Web). Puma, Unicorn, Thin
|
|
108
|
+
# and rackup set RACK_ENV to "development" when nothing names an
|
|
109
|
+
# environment, so under one of them, in production too, RACK_ENV alone
|
|
110
|
+
# reads "development". There it takes CRONWATCH_ENV, APP_ENV, Rails.env
|
|
111
|
+
# or RAILS_ENV to say development; any other value of RACK_ENV that reads
|
|
112
|
+
# as development ("test", "dev") still counts, as no server sets it.
|
|
113
|
+
def stated_development?
|
|
114
|
+
return false unless development?
|
|
115
|
+
|
|
116
|
+
variable, value = source
|
|
117
|
+
return true unless variable == "RACK_ENV" && JS.trim(value).downcase == "development"
|
|
118
|
+
|
|
119
|
+
!defaulting_server?
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# A server that sets RACK_ENV when it is unset is running this process.
|
|
123
|
+
def defaulting_server?
|
|
124
|
+
return true if defined?(::Puma::Server) || defined?(::Puma::Launcher) || defined?(::Unicorn::HttpServer) ||
|
|
125
|
+
defined?(::Thin::Server) || defined?(::Rackup::Server)
|
|
126
|
+
|
|
127
|
+
# Rack 2 autoloads its server: counted only once it has loaded.
|
|
128
|
+
defined?(::Rack) && ::Rack.const_defined?(:Server, false) && ::Rack.autoload?(:Server).nil?
|
|
27
129
|
end
|
|
28
130
|
end
|
|
29
131
|
end
|
data/lib/cronwatch/evaluate.rb
CHANGED
|
@@ -4,6 +4,8 @@ module Cronwatch
|
|
|
4
4
|
# Pure decisions about a job's health. Each function takes the current state
|
|
5
5
|
# and returns the new state plus the alerts that should go out. Nothing here
|
|
6
6
|
# touches a store or a network, which is what makes it testable.
|
|
7
|
+
#
|
|
8
|
+
# @api private
|
|
7
9
|
module Evaluate
|
|
8
10
|
DEFAULT_GRACE_MS = 10 * 60_000
|
|
9
11
|
DEFAULT_TIMEOUT_MS = 60 * 60_000
|
|
@@ -35,6 +37,14 @@ module Cronwatch
|
|
|
35
37
|
[ms, MAX_DURATION_MS].min
|
|
36
38
|
end
|
|
37
39
|
|
|
40
|
+
# When a silence of `ms` from `now` ends: a whole millisecond, never past
|
|
41
|
+
# MAX_DURATION_MS (2**53 - 1), however long the silence asked for. Every
|
|
42
|
+
# port sharing the store reads it back unchanged, where a larger number
|
|
43
|
+
# could wrap to a time long past and send alerts during the silence.
|
|
44
|
+
def silence_end(now, ms)
|
|
45
|
+
[now + (ms >= MAX_DURATION_MS ? MAX_DURATION_MS : ms.floor), MAX_DURATION_MS].min
|
|
46
|
+
end
|
|
47
|
+
|
|
38
48
|
# The version a stored state counts as for compare_and_set_state: its
|
|
39
49
|
# `version` when that is a whole number from 0 to MAX_DURATION_MS
|
|
40
50
|
# (2**53 - 1), else 0, as when it is absent. The SQL stores read it the
|
|
@@ -61,28 +71,145 @@ module Cronwatch
|
|
|
61
71
|
[count, MAX_DURATION_MS].min.to_i
|
|
62
72
|
end
|
|
63
73
|
|
|
74
|
+
# A JSON number, as JavaScript's typeof value === "number" sees one
|
|
75
|
+
# read from JSON: an Integer or a Float (never true or false).
|
|
76
|
+
def json_number?(value)
|
|
77
|
+
value.is_a?(Integer) || value.is_a?(Float)
|
|
78
|
+
end
|
|
79
|
+
|
|
64
80
|
def empty_state(job)
|
|
65
81
|
JobState.new(job: job, open: {}, consecutive_failures: 0, silenced_until: nil, last_alert_at: nil,
|
|
66
82
|
pending_recovery: [], undelivered: [])
|
|
67
83
|
end
|
|
68
84
|
|
|
69
|
-
# A stored state with every field present, or a fresh one. State written
|
|
85
|
+
# A stored state with every field present, or a fresh one. State written
|
|
86
|
+
# by an older version lacks the newer fields. `sending` is the exception:
|
|
87
|
+
# it is there only while it holds an alert (see hold_alerts).
|
|
88
|
+
#
|
|
89
|
+
# Read leniently, since a foreign, hand-edited or damaged row must affect
|
|
90
|
+
# only its own job, and the next write puts it right: a state that is not
|
|
91
|
+
# an object reads as none; `open` keeps only its entries whose value is a
|
|
92
|
+
# number (anything but an object reads as {}); `silenced_until` and
|
|
93
|
+
# `last_alert_at` that are not numbers read as nil; `pending_recovery`
|
|
94
|
+
# keeps only its conditions (strings as stored), and `undelivered` only
|
|
95
|
+
# its entries that are objects (a list of neither shape reads as []).
|
|
96
|
+
# Unknown fields are kept as written.
|
|
70
97
|
def normalize_state(state, job)
|
|
98
|
+
state = JobState.from_h(state)
|
|
71
99
|
return empty_state(job) if state.nil?
|
|
72
100
|
|
|
73
|
-
|
|
101
|
+
sending = state.sending
|
|
102
|
+
open = state.open.is_a?(Hash) ? state.open.select { |_, at| json_number?(at) } : {}
|
|
103
|
+
pending = state.pending_recovery.is_a?(Array) ? state.pending_recovery.grep(Symbol) : []
|
|
104
|
+
undelivered = state.undelivered.is_a?(Array) ? state.undelivered.grep(Alert) : []
|
|
74
105
|
JobState.new(
|
|
75
106
|
job: state.job.nil? ? job : state.job,
|
|
76
|
-
open:
|
|
107
|
+
open: open,
|
|
77
108
|
consecutive_failures: failure_count(state),
|
|
78
|
-
silenced_until: state.silenced_until,
|
|
79
|
-
last_alert_at: state.last_alert_at,
|
|
80
|
-
pending_recovery:
|
|
81
|
-
undelivered:
|
|
109
|
+
silenced_until: json_number?(state.silenced_until) ? state.silenced_until : nil,
|
|
110
|
+
last_alert_at: json_number?(state.last_alert_at) ? state.last_alert_at : nil,
|
|
111
|
+
pending_recovery: pending,
|
|
112
|
+
undelivered: undelivered,
|
|
82
113
|
version: state.version,
|
|
114
|
+
sending: sending.is_a?(Array) && !sending.empty? ? sending.dup : nil,
|
|
115
|
+
# Fields a newer release wrote, carried through every write.
|
|
116
|
+
extra: state.extra,
|
|
83
117
|
)
|
|
84
118
|
end
|
|
85
119
|
|
|
120
|
+
# ---------------------------------------------------------------- delivery
|
|
121
|
+
|
|
122
|
+
# Alerts kept per job for retry, and per job being sent; past it the oldest go.
|
|
123
|
+
MAX_UNDELIVERED = 20
|
|
124
|
+
|
|
125
|
+
# How long an alert in `sending` is left to the process sending it.
|
|
126
|
+
# Longer than any send takes: at most three alerts go out together, each
|
|
127
|
+
# with 25 seconds of triage and 15 of channels.
|
|
128
|
+
SEND_LEASE_MS = 5 * 60_000
|
|
129
|
+
|
|
130
|
+
# A state and how many alerts the queue let go to make it.
|
|
131
|
+
Held = Struct.new(:state, :dropped, keyword_init: true) do
|
|
132
|
+
def to_h = { "state" => state.to_h, "dropped" => dropped }
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Identifies an alert across retries, and in `sending`. `at` is printed
|
|
136
|
+
# as JavaScript prints a number.
|
|
137
|
+
def alert_key(alert)
|
|
138
|
+
at = alert.at
|
|
139
|
+
run = alert.run
|
|
140
|
+
"#{alert.type}|#{at.is_a?(Numeric) ? JS.number(at) : at}|#{run.id if run.is_a?(Run)}"
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# `alerts` added to the undelivered queue: one with the same key as a
|
|
144
|
+
# queued alert replaces it where it stands, the rest go at the end, and
|
|
145
|
+
# only the newest MAX_UNDELIVERED stay. `dropped` counts those let go.
|
|
146
|
+
def queue_undelivered(state, alerts)
|
|
147
|
+
following = clone_state(state)
|
|
148
|
+
by_key = alerts.to_h { |alert| [alert_key(alert), alert] }
|
|
149
|
+
queue = following.undelivered.map { |alert| by_key.fetch(alert_key(alert), alert) }
|
|
150
|
+
known = queue.to_set { |alert| alert_key(alert) }
|
|
151
|
+
queue.concat(alerts.reject { |alert| known.include?(alert_key(alert)) })
|
|
152
|
+
following.undelivered = queue.last(MAX_UNDELIVERED)
|
|
153
|
+
Held.new(state: following, dropped: [0, queue.length - MAX_UNDELIVERED].max)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The outbox. Alerts just composed are written with the state that opens
|
|
157
|
+
# their condition, before any is sent, so a process that stops part way
|
|
158
|
+
# does not lose them: into `sending`, each with its lease ending at
|
|
159
|
+
# `until_at`, when this process sends them, or (deliver: :check) straight
|
|
160
|
+
# into the undelivered queue for a check elsewhere. `dropped` counts
|
|
161
|
+
# alerts let go past MAX_UNDELIVERED.
|
|
162
|
+
def hold_alerts(state, alerts, until_at, deferred)
|
|
163
|
+
return Held.new(state: state, dropped: 0) if alerts.empty?
|
|
164
|
+
return queue_undelivered(state, alerts) if deferred
|
|
165
|
+
|
|
166
|
+
following = clone_state(state)
|
|
167
|
+
entries = (following.sending || []) + alerts.map { |alert| { "until" => until_at, "alert" => alert } }
|
|
168
|
+
following.sending = entries.last(MAX_UNDELIVERED)
|
|
169
|
+
Held.new(state: following, dropped: [0, entries.length - MAX_UNDELIVERED].max)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Alerts in `sending` whose lease ran out by `now`: the process sending
|
|
173
|
+
# them stopped before it recorded how the send went. They go to the
|
|
174
|
+
# undelivered queue, where the retry sends them (with triage, which is
|
|
175
|
+
# never stored with them here) or drops them as stale. An entry that is
|
|
176
|
+
# not a Hash with an alert is dropped; one without a numeric `until`
|
|
177
|
+
# counts as run out.
|
|
178
|
+
def release_sending(state, now)
|
|
179
|
+
sending = state.sending.is_a?(Array) ? state.sending : []
|
|
180
|
+
lapsed, held = sending.partition { |entry| !(entry.is_a?(Hash) && entry["until"].is_a?(Numeric) && entry["until"] > now) }
|
|
181
|
+
return Held.new(state: state, dropped: 0) if lapsed.empty?
|
|
182
|
+
|
|
183
|
+
kept = state.dup
|
|
184
|
+
kept.sending = held
|
|
185
|
+
queue_undelivered(kept, lapsed.filter_map { |entry| sending_alert(entry) })
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# How a send went. Delivered and stale alerts leave the queue; failed
|
|
189
|
+
# ones replace their queued copy, so a triage made on this attempt is
|
|
190
|
+
# kept, or join the queue. Every one of them leaves `sending`.
|
|
191
|
+
# last_alert_at moves only on a delivery. `dropped` counts alerts let go
|
|
192
|
+
# past MAX_UNDELIVERED.
|
|
193
|
+
def record_sent(state, delivered, failed, stale, now)
|
|
194
|
+
following = clone_state(state)
|
|
195
|
+
done = (delivered + stale).to_set { |alert| alert_key(alert) }
|
|
196
|
+
following.undelivered = following.undelivered.reject { |alert| done.include?(alert_key(alert)) }
|
|
197
|
+
sent = (delivered + failed + stale).to_set { |alert| alert_key(alert) }
|
|
198
|
+
held = (following.sending || []).reject do |entry|
|
|
199
|
+
alert = sending_alert(entry)
|
|
200
|
+
alert && sent.include?(alert_key(alert))
|
|
201
|
+
end
|
|
202
|
+
following.sending = held.empty? ? nil : held
|
|
203
|
+
following.last_alert_at = now if delivered.any?
|
|
204
|
+
queue_undelivered(following, failed)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# The alert of an entry of `sending`, or nil when it has none, or one
|
|
208
|
+
# that did not parse as an alert (which could not be sent either).
|
|
209
|
+
def sending_alert(entry)
|
|
210
|
+
entry.is_a?(Hash) && entry["alert"].is_a?(Alert) ? entry["alert"] : nil
|
|
211
|
+
end
|
|
212
|
+
|
|
86
213
|
def clone_state(state)
|
|
87
214
|
normalize_state(state, state.job)
|
|
88
215
|
end
|
|
@@ -340,11 +467,19 @@ module Cronwatch
|
|
|
340
467
|
# is dropped rather than sent late. An alert for a condition is stale once
|
|
341
468
|
# that condition has closed, or has closed and opened again (it opened at a
|
|
342
469
|
# time other than the alert's). A recovery is stale when any condition it
|
|
343
|
-
# names is open again; while they all stay closed it is kept.
|
|
470
|
+
# names is open again; while they all stay closed it is kept. From a
|
|
471
|
+
# foreign or damaged row: an alert whose `at` is not a number, and a
|
|
472
|
+
# recovery whose `details.after` is not a list of conditions, are stale.
|
|
344
473
|
def stale_alert?(alert, state)
|
|
345
|
-
|
|
474
|
+
if alert.type == :recovered
|
|
475
|
+
# One whose details say nothing of what it recovers from cannot be judged, and goes.
|
|
476
|
+
after = alert.details.is_a?(Hash) ? alert.details[:after] : nil
|
|
477
|
+
return true unless after.is_a?(Array)
|
|
346
478
|
|
|
347
|
-
|
|
479
|
+
return after.any? { |condition| !condition.is_a?(Symbol) || state.open.key?(condition) }
|
|
480
|
+
end
|
|
481
|
+
# One with no time cannot match an open condition.
|
|
482
|
+
!json_number?(alert.at) || state.open[alert.type] != alert.at
|
|
348
483
|
end
|
|
349
484
|
|
|
350
485
|
# How a job looks at a glance. Silence wins, then stuck, failing and late.
|
data/lib/cronwatch/flight.rb
CHANGED
|
@@ -3,8 +3,14 @@
|
|
|
3
3
|
module Cronwatch
|
|
4
4
|
class Client
|
|
5
5
|
# One shared check: the first caller runs it, the others wait for its result.
|
|
6
|
+
#
|
|
7
|
+
# @api private
|
|
6
8
|
class Flight
|
|
9
|
+
# The thread running the check.
|
|
10
|
+
attr_reader :owner
|
|
11
|
+
|
|
7
12
|
def initialize
|
|
13
|
+
@owner = Thread.current
|
|
8
14
|
@lock = Mutex.new
|
|
9
15
|
@done = ConditionVariable.new
|
|
10
16
|
@finished = false
|
data/lib/cronwatch/format.rb
CHANGED
data/lib/cronwatch/http.rb
CHANGED
|
@@ -7,8 +7,15 @@ require "uri"
|
|
|
7
7
|
module Cronwatch
|
|
8
8
|
# POSTs for the alert channels. Anything with `post(url, body, headers)`
|
|
9
9
|
# returning a Response can stand in for it, which is how the tests run.
|
|
10
|
+
#
|
|
11
|
+
# @api private
|
|
10
12
|
module HTTP
|
|
11
13
|
TIMEOUT = 10
|
|
14
|
+
# The most of an answer's body read (1 MiB): a channel reads 200
|
|
15
|
+
# characters of a refusal, and a hostile or broken endpoint could
|
|
16
|
+
# otherwise stream gigabytes into the worker within the deadline.
|
|
17
|
+
# Reading stops there. A 2xx answer's body is not read at all.
|
|
18
|
+
MAX_BODY = 1_048_576
|
|
12
19
|
|
|
13
20
|
Response = Struct.new(:status, :body, keyword_init: true) do
|
|
14
21
|
def ok? = status >= 200 && status < 300
|
|
@@ -33,6 +40,8 @@ module Cronwatch
|
|
|
33
40
|
# sends. Past the deadline before an answer, raises HTTP::TimeoutError;
|
|
34
41
|
# past it while the body is still arriving, returns the answer with an
|
|
35
42
|
# empty body, as the SDK's channels treat a body they could not read.
|
|
43
|
+
# A 2xx answer comes back with an empty body, unread, since no channel
|
|
44
|
+
# needs it; any other with at most MAX_BODY bytes of it.
|
|
36
45
|
def post(url, body, headers)
|
|
37
46
|
uri = HTTP.postable(url)
|
|
38
47
|
deadline = HTTP.monotonic + @timeout
|
|
@@ -46,7 +55,11 @@ module Cronwatch
|
|
|
46
55
|
conn.read_timeout = remaining
|
|
47
56
|
conn.write_timeout = remaining
|
|
48
57
|
conn.request(request) do |response|
|
|
49
|
-
|
|
58
|
+
status = response.code.to_i
|
|
59
|
+
# Returning from inside the block leaves the rest unread; the connection closes after.
|
|
60
|
+
return Response.new(status: status, body: "") if status >= 200 && status < 300
|
|
61
|
+
|
|
62
|
+
return Response.new(status: status, body: read_body(conn, response, deadline))
|
|
50
63
|
end
|
|
51
64
|
end
|
|
52
65
|
rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
|
|
@@ -65,17 +78,20 @@ module Cronwatch
|
|
|
65
78
|
|
|
66
79
|
private
|
|
67
80
|
|
|
68
|
-
# The body, read in chunks until
|
|
81
|
+
# The body, read in chunks until it ends or MAX_BODY bytes have come,
|
|
82
|
+
# whichever is first; "" when the deadline passes first.
|
|
69
83
|
def read_body(conn, response, deadline)
|
|
70
|
-
|
|
84
|
+
body = "".b
|
|
71
85
|
conn.read_timeout = HTTP.remaining(deadline)
|
|
72
86
|
response.read_body do |chunk|
|
|
73
87
|
raise TimeoutError if HTTP.monotonic >= deadline
|
|
74
88
|
|
|
75
|
-
|
|
89
|
+
body << chunk.b.byteslice(0, MAX_BODY - body.bytesize)
|
|
90
|
+
break if body.bytesize >= MAX_BODY
|
|
91
|
+
|
|
76
92
|
conn.read_timeout = HTTP.remaining(deadline)
|
|
77
93
|
end
|
|
78
|
-
|
|
94
|
+
body
|
|
79
95
|
rescue Net::ReadTimeout, TimeoutError
|
|
80
96
|
""
|
|
81
97
|
end
|
data/lib/cronwatch/job.rb
CHANGED
|
@@ -95,9 +95,11 @@ module Cronwatch
|
|
|
95
95
|
end
|
|
96
96
|
|
|
97
97
|
# Collects a run's output and metrics while its block runs.
|
|
98
|
+
#
|
|
99
|
+
# @api private
|
|
98
100
|
class RunRecorder
|
|
99
101
|
# Lines are dropped from the front once the output is well past the cap;
|
|
100
|
-
# Output.
|
|
102
|
+
# Output.redact_and_cap trims it exactly at the end.
|
|
101
103
|
KEEP = 64 * 1024
|
|
102
104
|
|
|
103
105
|
attr_reader :context, :signal
|
|
@@ -136,8 +138,11 @@ module Cronwatch
|
|
|
136
138
|
@lock.synchronize { @metrics[name] = value }
|
|
137
139
|
end
|
|
138
140
|
|
|
141
|
+
# The lines still held (past 64 KB the oldest are let go), joined and not
|
|
142
|
+
# yet capped: the client redacts them first, then caps them
|
|
143
|
+
# (Output.redact_and_cap).
|
|
139
144
|
def output
|
|
140
|
-
@lock.synchronize { @lines.empty? ? nil :
|
|
145
|
+
@lock.synchronize { @lines.empty? ? nil : @lines.join("\n") }
|
|
141
146
|
end
|
|
142
147
|
|
|
143
148
|
# What an expect rule is checked against: everything logged, or when that
|
data/lib/cronwatch/js.rb
CHANGED
|
@@ -6,6 +6,8 @@ module Cronwatch
|
|
|
6
6
|
# webhook bodies, so the gem reproduces them byte for byte: Math.round,
|
|
7
7
|
# String(number), JSON.stringify, String.prototype.trim and lengths counted
|
|
8
8
|
# in UTF-16 code units.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
9
11
|
module JS
|
|
10
12
|
# What JavaScript's \s and trim() treat as whitespace.
|
|
11
13
|
WHITESPACE = "\\t\\n\\v\\f\\r \\u00a0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff"
|
|
@@ -40,8 +42,10 @@ module Cronwatch
|
|
|
40
42
|
value - floor >= 0.5 ? floor + 1 : floor
|
|
41
43
|
end
|
|
42
44
|
|
|
45
|
+
# Number.isFinite of the number as JavaScript would hold it: an Integer
|
|
46
|
+
# past a double's range is Infinity there, so it is not finite here.
|
|
43
47
|
def finite?(value)
|
|
44
|
-
value.is_a?(
|
|
48
|
+
value.is_a?(Numeric) && value.real? && value.to_f.finite?
|
|
45
49
|
end
|
|
46
50
|
|
|
47
51
|
# Number.isInteger.
|
|
@@ -159,12 +163,44 @@ module Cronwatch
|
|
|
159
163
|
end
|
|
160
164
|
|
|
161
165
|
def quote(text)
|
|
166
|
+
if text.encoding == Encoding::UTF_8 && !text.valid_encoding? && text.b.match?(LONE)
|
|
167
|
+
# A lone surrogate (see lone_surrogate), written as JSON.stringify writes one.
|
|
168
|
+
parts = text.b.split(LONE_SPLIT).each_with_index.map do |part, i|
|
|
169
|
+
next quote(part.force_encoding(Encoding::UTF_8))[1...-1] if i.even?
|
|
170
|
+
|
|
171
|
+
bytes = part.bytes
|
|
172
|
+
format("\\u%04x", ((bytes[0] & 0x0F) << 12) | ((bytes[1] & 0x3F) << 6) | (bytes[2] & 0x3F))
|
|
173
|
+
end
|
|
174
|
+
return "\"#{parts.join}\""
|
|
175
|
+
end
|
|
162
176
|
text = text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) unless text.encoding == Encoding::UTF_8 && text.valid_encoding?
|
|
163
177
|
text = text.scrub unless text.valid_encoding?
|
|
164
178
|
escaped = text.gsub(/["\\\u0000-\u001f]/) { |c| ESCAPES[c] || format("\\u%04x", c.ord) }
|
|
165
179
|
"\"#{escaped}\""
|
|
166
180
|
end
|
|
167
181
|
|
|
182
|
+
# A surrogate code unit on its own, in the bytes UTF-8 would give it.
|
|
183
|
+
# They make the string invalid UTF-8, which Ruby's own methods refuse, so
|
|
184
|
+
# one is only ever the last thing added to text about to be written:
|
|
185
|
+
# quote writes it as \udXXX, as JSON.stringify does.
|
|
186
|
+
LONE = /\xED[\xA0-\xBF][\x80-\xBF]/n
|
|
187
|
+
LONE_SPLIT = /(\xED[\xA0-\xBF][\x80-\xBF])/n
|
|
188
|
+
|
|
189
|
+
def lone_surrogate(unit)
|
|
190
|
+
[0xED, 0x80 | ((unit >> 6) & 0x3F), 0x80 | (unit & 0x3F)].pack("C*").force_encoding(Encoding::UTF_8)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# String.prototype.slice(0, units) as JavaScript gives it: the first
|
|
194
|
+
# `units` UTF-16 code units, and the high surrogate of a pair the cut
|
|
195
|
+
# splits. Returns the head Ruby can hold (head16) and that surrogate's
|
|
196
|
+
# code unit, or nil when the cut splits no pair.
|
|
197
|
+
def slice16(text, units)
|
|
198
|
+
head = head16(text, units)
|
|
199
|
+
return [head, nil] if text.ascii_only? || head.length == text.length || length16(head) == units
|
|
200
|
+
|
|
201
|
+
[head, 0xD800 + ((text[head.length].ord - 0x10000) >> 10)]
|
|
202
|
+
end
|
|
203
|
+
|
|
168
204
|
# Property order: array-index keys ascending, then the rest as inserted.
|
|
169
205
|
def object_keys(hash)
|
|
170
206
|
keys = hash.keys
|
|
@@ -184,5 +220,42 @@ module Cronwatch
|
|
|
184
220
|
def parse(text)
|
|
185
221
|
::JSON.parse(text)
|
|
186
222
|
end
|
|
223
|
+
|
|
224
|
+
DECIMAL = /\A[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?\z/
|
|
225
|
+
RADIX = { "x" => 16, "o" => 8, "b" => 2 }.freeze
|
|
226
|
+
|
|
227
|
+
# Number(string): decimal, 0x/0o/0b, Infinity, or NaN; 0 for a string
|
|
228
|
+
# of only whitespace.
|
|
229
|
+
def to_number(value)
|
|
230
|
+
text = trim(value.dup.force_encoding(Encoding::UTF_8).scrub)
|
|
231
|
+
return 0 if text.empty?
|
|
232
|
+
# Ruby's Float() wants a digit on both sides of the point; JavaScript does not.
|
|
233
|
+
return Float(text.sub(/\A([+-]?)\./, "\\10.").sub(/\.(?=[eE]|\z)/, ".0")) if DECIMAL.match?(text)
|
|
234
|
+
return text.start_with?("-") ? -Float::INFINITY : Float::INFINITY if /\A[+-]?Infinity\z/.match?(text)
|
|
235
|
+
|
|
236
|
+
if (m = /\A0([xXoObB])([0-9a-fA-F]+)\z/.match(text))
|
|
237
|
+
return Integer(m[2], RADIX.fetch(m[1].downcase))
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
Float::NAN
|
|
241
|
+
rescue ArgumentError
|
|
242
|
+
Float::NAN
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# A time or a count of milliseconds as a SQL column holds it: an Integer
|
|
246
|
+
# or a Float kept, a String that is not blank read as Number() reads it
|
|
247
|
+
# (Postgres hands BIGINT over as text), and nil when the result is not a
|
|
248
|
+
# finite number. A whole number in a Float comes back an Integer.
|
|
249
|
+
def finite_number(value)
|
|
250
|
+
n =
|
|
251
|
+
case value
|
|
252
|
+
when Integer, Float then value
|
|
253
|
+
when String then trim(value.dup.force_encoding(Encoding::UTF_8).scrub).empty? ? nil : to_number(value)
|
|
254
|
+
end
|
|
255
|
+
return nil unless finite?(n)
|
|
256
|
+
return n.to_i if n.is_a?(Float) && n == n.floor && n.abs <= MAX_SAFE_INTEGER
|
|
257
|
+
|
|
258
|
+
n
|
|
259
|
+
end
|
|
187
260
|
end
|
|
188
261
|
end
|
data/lib/cronwatch/monitored.rb
CHANGED
|
@@ -6,6 +6,8 @@ module Cronwatch
|
|
|
6
6
|
# declared on Cronwatch.client. Needs nothing beyond the core, so Sidekiq
|
|
7
7
|
# without Rails can use it. Loaded by cronwatch/scheduler, which schedule:
|
|
8
8
|
# :from_scheduler and declare_from_scheduler! need.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
9
11
|
module Monitored
|
|
10
12
|
# What `cronwatch` is outside a monitored run: it takes log and metric
|
|
11
13
|
# calls and drops them, so the job's code runs the same either way.
|
|
@@ -51,7 +53,10 @@ module Cronwatch
|
|
|
51
53
|
client = Cronwatch.client
|
|
52
54
|
@lock.synchronize do
|
|
53
55
|
current = @registration
|
|
54
|
-
|
|
56
|
+
# Kept while the client still holds the job: one forgotten since
|
|
57
|
+
# (the dashboard's forget) is declared again, with its schedule,
|
|
58
|
+
# as the other ports' bridges declare it again before a check.
|
|
59
|
+
return current if current && current[0].equal?(client) && client.defined_jobs.any? { |d| d.name == @name }
|
|
55
60
|
|
|
56
61
|
handle = client.job(@name, **resolve_locked)
|
|
57
62
|
@registration = [client, handle].freeze
|