solana-ruby-watchtower 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +14 -0
- data/LICENSE +21 -0
- data/README.md +181 -0
- data/examples/solana-watchtower.service +28 -0
- data/examples/watchtower.yml +56 -0
- data/exe/solana-watchtower +9 -0
- data/lib/solana/ruby/watchtower/check_state.rb +98 -0
- data/lib/solana/ruby/watchtower/cli.rb +133 -0
- data/lib/solana/ruby/watchtower/config.rb +134 -0
- data/lib/solana/ruby/watchtower/heartbeat.rb +27 -0
- data/lib/solana/ruby/watchtower/monitor.rb +189 -0
- data/lib/solana/ruby/watchtower/notifiers.rb +100 -0
- data/lib/solana/ruby/watchtower/observe.rb +46 -0
- data/lib/solana/ruby/watchtower/rpc.rb +111 -0
- data/lib/solana/ruby/watchtower/runner.rb +51 -0
- data/lib/solana/ruby/watchtower/stores/active_record_store.rb +35 -0
- data/lib/solana/ruby/watchtower/stores/file_store.rb +56 -0
- data/lib/solana/ruby/watchtower/stores/memory_store.rb +29 -0
- data/lib/solana/ruby/watchtower/version.rb +9 -0
- data/lib/solana/ruby/watchtower.rb +34 -0
- data/solana-ruby-watchtower.gemspec +43 -0
- metadata +127 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'net/http'
|
|
4
|
+
require 'uri'
|
|
5
|
+
|
|
6
|
+
module Solana::Ruby::Watchtower
|
|
7
|
+
# A dead-man's switch: GETs a URL (healthchecks.io, Uptime Kuma, Better Stack
|
|
8
|
+
# and the like) after every completed vote check. When the monitor dies, hangs
|
|
9
|
+
# or loses its network, the pings stop and that service raises the alarm,
|
|
10
|
+
# which nothing inside the monitor can do for itself.
|
|
11
|
+
class Heartbeat
|
|
12
|
+
def initialize(url, logger:)
|
|
13
|
+
@uri = URI.parse(url)
|
|
14
|
+
@logger = logger
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def ping
|
|
18
|
+
Net::HTTP.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https', open_timeout: 5, read_timeout: 5) do |http|
|
|
19
|
+
http.get(@uri.request_uri)
|
|
20
|
+
end
|
|
21
|
+
true
|
|
22
|
+
rescue StandardError => e
|
|
23
|
+
@logger.error("heartbeat ping to #{@uri.host} failed: #{e.class}")
|
|
24
|
+
false
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
5
|
+
module Solana::Ruby::Watchtower
|
|
6
|
+
# Runs the checks for one validator and turns their results into alerts.
|
|
7
|
+
#
|
|
8
|
+
# monitor.check_votes # every 30s: is the staked identity voting?
|
|
9
|
+
# monitor.check_gossip # every 2 min: are the nodes up, is only one staked?
|
|
10
|
+
#
|
|
11
|
+
# Each check key keeps a CheckState in the store, and posts when it goes bad,
|
|
12
|
+
# every realert_minutes while it stays bad, and when it recovers:
|
|
13
|
+
#
|
|
14
|
+
# rpc:votes, rpc:gossip every RPC endpoint failed
|
|
15
|
+
# voting last vote too far behind the tip, or delinquent
|
|
16
|
+
# vote_identity the vote account points at another identity
|
|
17
|
+
# staked:gossip staked identity absent from gossip, or on an unknown host
|
|
18
|
+
# staked:location staked identity's address flipping between hosts
|
|
19
|
+
# node:<name> a node absent from gossip, or running a key it should not
|
|
20
|
+
class Monitor
|
|
21
|
+
CHECKS = %w[votes gossip].freeze
|
|
22
|
+
|
|
23
|
+
attr_reader :config, :store, :notifier
|
|
24
|
+
|
|
25
|
+
def initialize(config, store: nil, notifier: nil, rpc: nil, heartbeat: :from_config,
|
|
26
|
+
logger: Logger.new($stdout), clock: -> { Time.now.utc })
|
|
27
|
+
@config = config
|
|
28
|
+
@logger = logger
|
|
29
|
+
@store = store || Stores::FileStore.new(config.state_file)
|
|
30
|
+
@notifier = notifier || Notifiers.build(config.notify, logger: logger)
|
|
31
|
+
@rpc = rpc || RpcPool.from_urls(config.rpc_urls, timeout: config.rpc_timeout_seconds)
|
|
32
|
+
@heartbeat = heartbeat == :from_config ? (config.heartbeat_url && Heartbeat.new(config.heartbeat_url, logger: logger)) : heartbeat
|
|
33
|
+
@clock = clock
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def run(check)
|
|
37
|
+
case check.to_s
|
|
38
|
+
when 'votes' then check_votes
|
|
39
|
+
when 'gossip' then check_gossip
|
|
40
|
+
else raise ArgumentError, "unknown check #{check.inspect} (#{CHECKS.join(' or ')})"
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def check_votes
|
|
45
|
+
answer = rpc_answer('rpc:votes', config.rpc_failures_to_alert) { |rpc| Observe.vote_account(rpc, config.vote_account) }
|
|
46
|
+
# Unknown is not bad: the other checks keep their state until RPC is back.
|
|
47
|
+
if answer
|
|
48
|
+
evaluate_voting(answer.value)
|
|
49
|
+
evaluate_vote_identity(answer.value) if answer.value.found
|
|
50
|
+
end
|
|
51
|
+
# After every completed run, whatever it found: the heartbeat proves the
|
|
52
|
+
# monitor is alive, the alerts above say what it saw.
|
|
53
|
+
@heartbeat&.ping
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def check_gossip
|
|
57
|
+
pubkeys = [config.identity, *config.nodes.map(&:unstaked_identity).compact]
|
|
58
|
+
answer = rpc_answer('rpc:gossip', config.failures_to_alert) do |rpc|
|
|
59
|
+
Observe.gossip(rpc, pubkeys: pubkeys, ips: config.nodes.map(&:ip))
|
|
60
|
+
end
|
|
61
|
+
return unless answer
|
|
62
|
+
|
|
63
|
+
entries = answer.value
|
|
64
|
+
staked = entries.find { |e| e.pubkey == config.identity }
|
|
65
|
+
evaluate_staked_gossip(staked)
|
|
66
|
+
evaluate_staked_location(staked.ip) if staked
|
|
67
|
+
config.nodes.each { |node| evaluate_node(node, entries.select { |e| e.ip == node.ip }) }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
def rpc_answer(key, failures_to_alert)
|
|
73
|
+
answer = @rpc.first_answer { |rpc| yield rpc }
|
|
74
|
+
evaluate(key, ok: true, message: "answering via #{answer.label}", failures_to_alert: failures_to_alert)
|
|
75
|
+
answer
|
|
76
|
+
rescue RpcPool::AllFailed => e
|
|
77
|
+
evaluate(key, ok: false, message: "every RPC endpoint failed: #{e.message}", failures_to_alert: failures_to_alert)
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def evaluate_voting(vote)
|
|
82
|
+
unless vote.found
|
|
83
|
+
return evaluate('voting', ok: false,
|
|
84
|
+
message: "vote account #{short(config.vote_account)} is in neither the current nor the delinquent list")
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
lag = [vote.slot - vote.last_vote, 0].max
|
|
88
|
+
ok, message =
|
|
89
|
+
if vote.delinquent
|
|
90
|
+
[false, "DELINQUENT: last vote at slot #{vote.last_vote}, #{lag} slots behind"]
|
|
91
|
+
elsif lag > config.max_vote_lag_slots
|
|
92
|
+
[false, "last vote #{lag} slots behind the tip (limit #{config.max_vote_lag_slots})"]
|
|
93
|
+
else
|
|
94
|
+
[true, "voting, #{lag} slots behind the tip"]
|
|
95
|
+
end
|
|
96
|
+
evaluate('voting', ok: ok, message: message)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def evaluate_vote_identity(vote)
|
|
100
|
+
ok = vote.node_pubkey == config.identity
|
|
101
|
+
evaluate('vote_identity', ok: ok,
|
|
102
|
+
message: if ok
|
|
103
|
+
"vote account identity is #{short(config.identity)}"
|
|
104
|
+
else
|
|
105
|
+
"vote account identity is #{vote.node_pubkey}, expected #{config.identity}"
|
|
106
|
+
end)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Gossip runs every couple of minutes, so a single miss is reported.
|
|
110
|
+
def evaluate_staked_gossip(staked)
|
|
111
|
+
# An address that is neither node means the staked key is running somewhere
|
|
112
|
+
# else: a leaked key, or a node added without updating the config.
|
|
113
|
+
ok = staked && config.nodes.any? { |n| n.ip == staked.ip }
|
|
114
|
+
message = staked ? "staked identity in gossip on #{where(staked.ip)}" : "staked identity #{short(config.identity)} is not in gossip"
|
|
115
|
+
evaluate('staked:gossip', ok: ok ? true : false, message: message, failures_to_alert: 1)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Gossip keeps one address per identity, whichever node pushed last. One node
|
|
119
|
+
# holding the staked identity gives a steady address; two nodes holding it make
|
|
120
|
+
# the address flip between them. A single change is a role switch, reported
|
|
121
|
+
# but not alarming.
|
|
122
|
+
def evaluate_staked_location(ip)
|
|
123
|
+
state = store.fetch('staked:location')
|
|
124
|
+
now = @clock.call
|
|
125
|
+
window = config.double_active_window_minutes * 60
|
|
126
|
+
previous = state.data['ip']
|
|
127
|
+
moves = Array(state.data['moves']).map { |t| Time.iso8601(t) }.select { |t| t > now - window }
|
|
128
|
+
|
|
129
|
+
if previous && previous != ip
|
|
130
|
+
moves << now
|
|
131
|
+
@notifier.deliver(tag("ℹ️ staked identity moved from #{where(previous)} to #{where(ip)}")) if moves.size < config.double_active_moves
|
|
132
|
+
end
|
|
133
|
+
state.data = { 'ip' => ip, 'moves' => moves.map { |t| t.utc.iso8601 } }
|
|
134
|
+
|
|
135
|
+
double = moves.size >= config.double_active_moves
|
|
136
|
+
message = if double
|
|
137
|
+
"staked identity's gossip address changed #{moves.size} times in #{config.double_active_window_minutes} min, " \
|
|
138
|
+
"now #{where(ip)}: BOTH NODES MAY BE RUNNING THE STAKED IDENTITY"
|
|
139
|
+
else
|
|
140
|
+
"staked identity steady on #{where(ip)}"
|
|
141
|
+
end
|
|
142
|
+
evaluate_state(state, ok: !double, message: message, failures_to_alert: 1)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def evaluate_node(node, here)
|
|
146
|
+
expected = { config.identity => 'staked' }
|
|
147
|
+
expected[node.unstaked_identity] = 'standby' if node.unstaked_identity
|
|
148
|
+
known = here.filter_map { |e| expected[e.pubkey] }
|
|
149
|
+
|
|
150
|
+
ok, message =
|
|
151
|
+
if here.empty?
|
|
152
|
+
[false, "#{node.name} (#{node.ip}) is not in gossip: the node is down or cut off"]
|
|
153
|
+
elsif known.empty?
|
|
154
|
+
[false, "#{node.name} is in gossip as #{here.map { |e| short(e.pubkey) }.join(', ')}, " \
|
|
155
|
+
'neither the staked identity nor its own standby key']
|
|
156
|
+
else
|
|
157
|
+
[true, "#{node.name} is in gossip as #{known.uniq.join(' + ')}"]
|
|
158
|
+
end
|
|
159
|
+
evaluate("node:#{node.name}", ok: ok, message: message, failures_to_alert: 1)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def evaluate(key, ok:, message:, failures_to_alert: config.failures_to_alert)
|
|
163
|
+
evaluate_state(store.fetch(key), ok: ok, message: message, failures_to_alert: failures_to_alert)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def evaluate_state(state, ok:, message:, failures_to_alert:)
|
|
167
|
+
now = @clock.call
|
|
168
|
+
alert = state.observe(ok: ok, message: message, now: now, failures_to_alert: failures_to_alert,
|
|
169
|
+
realert_after: config.realert_minutes * 60)
|
|
170
|
+
@logger.debug("#{state.key}: #{ok ? 'ok' : 'FAIL'}: #{message}")
|
|
171
|
+
if alert
|
|
172
|
+
@logger.warn(alert.text)
|
|
173
|
+
state.acknowledge(alert, now) if @notifier.deliver(tag(alert.text))
|
|
174
|
+
end
|
|
175
|
+
store.save(state)
|
|
176
|
+
state
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# Names the validator, so one chat can take alerts for several.
|
|
180
|
+
def tag(text) = "[#{config.name}] #{text}"
|
|
181
|
+
|
|
182
|
+
def where(ip)
|
|
183
|
+
node = config.nodes.find { |n| n.ip == ip }
|
|
184
|
+
node ? "#{node.name} (#{ip})" : "UNKNOWN HOST #{ip}"
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def short(pubkey) = "#{pubkey.to_s[0, 8]}…"
|
|
188
|
+
end
|
|
189
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'net/http'
|
|
5
|
+
require 'uri'
|
|
6
|
+
|
|
7
|
+
module Solana::Ruby::Watchtower
|
|
8
|
+
# Where alerts go. A notifier answers #deliver(text) with true when the text
|
|
9
|
+
# was delivered and false when it was not, in which case the monitor offers
|
|
10
|
+
# the alert again on the next run. Notifiers never raise.
|
|
11
|
+
module Notifiers
|
|
12
|
+
# Builds the notifiers named under `notify:` in the config. With none
|
|
13
|
+
# configured, alerts are only logged.
|
|
14
|
+
def self.build(settings, logger:)
|
|
15
|
+
notifiers = settings.map do |kind, options|
|
|
16
|
+
options ||= {}
|
|
17
|
+
case kind.to_s
|
|
18
|
+
when 'telegram' then Telegram.new(bot_token: options['bot_token'], chat_id: options['chat_id'], logger: logger)
|
|
19
|
+
when 'webhook' then Webhook.new(url: options['url'], logger: logger)
|
|
20
|
+
else raise ConfigError, "notify.#{kind}: unknown notifier (telegram or webhook)"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
return Log.new(logger) if notifiers.empty?
|
|
24
|
+
|
|
25
|
+
notifiers.size == 1 ? notifiers.first : Fanout.new(notifiers)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Logs instead of sending. The fallback when nothing is configured.
|
|
29
|
+
class Log
|
|
30
|
+
def initialize(logger)
|
|
31
|
+
@logger = logger
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def deliver(text)
|
|
35
|
+
@logger.warn("alert (no notifier configured): #{text}")
|
|
36
|
+
true
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
class Telegram
|
|
41
|
+
def initialize(bot_token:, chat_id:, logger:)
|
|
42
|
+
raise ConfigError, 'notify.telegram.bot_token is empty' if bot_token.to_s.strip.empty?
|
|
43
|
+
raise ConfigError, 'notify.telegram.chat_id is empty' if chat_id.to_s.strip.empty?
|
|
44
|
+
|
|
45
|
+
@uri = URI("https://api.telegram.org/bot#{bot_token}/sendMessage")
|
|
46
|
+
@chat_id = chat_id.to_s
|
|
47
|
+
@logger = logger
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def deliver(text)
|
|
51
|
+
response = Net::HTTP.start(@uri.host, @uri.port, use_ssl: true, open_timeout: 5, read_timeout: 10) do |http|
|
|
52
|
+
http.post(@uri.path, URI.encode_www_form(chat_id: @chat_id, text: text, disable_web_page_preview: true))
|
|
53
|
+
end
|
|
54
|
+
return true if response.is_a?(Net::HTTPSuccess)
|
|
55
|
+
|
|
56
|
+
@logger.error("Telegram answered #{response.code}: #{response.body.to_s[0, 200]}")
|
|
57
|
+
false
|
|
58
|
+
rescue StandardError => e
|
|
59
|
+
# The class only: a connection error can quote the URL, and the bot token is in it.
|
|
60
|
+
@logger.error("Telegram send failed: #{e.class}")
|
|
61
|
+
false
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# POSTs JSON to a webhook. The text is sent as both `text` (Slack) and
|
|
66
|
+
# `content` (Discord); each ignores the other's field.
|
|
67
|
+
class Webhook
|
|
68
|
+
def initialize(url:, logger:)
|
|
69
|
+
@uri = URI.parse(url.to_s)
|
|
70
|
+
raise ConfigError, 'notify.webhook.url is not an http(s) URL' unless @uri.is_a?(URI::HTTP) && @uri.host
|
|
71
|
+
|
|
72
|
+
@logger = logger
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def deliver(text)
|
|
76
|
+
response = Net::HTTP.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https', open_timeout: 5, read_timeout: 10) do |http|
|
|
77
|
+
http.post(@uri.request_uri, JSON.generate(text: text, content: text), 'Content-Type' => 'application/json')
|
|
78
|
+
end
|
|
79
|
+
return true if response.is_a?(Net::HTTPSuccess)
|
|
80
|
+
|
|
81
|
+
@logger.error("webhook #{@uri.host} answered #{response.code}")
|
|
82
|
+
false
|
|
83
|
+
rescue StandardError => e
|
|
84
|
+
@logger.error("webhook #{@uri.host} send failed: #{e.class}")
|
|
85
|
+
false
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Sends to every notifier. Delivered only when all of them took it, so a
|
|
90
|
+
# channel that failed gets the alert again, at the cost of a repeat on the
|
|
91
|
+
# channels that did not fail: a duplicate is cheaper than a missed alert.
|
|
92
|
+
class Fanout
|
|
93
|
+
def initialize(notifiers)
|
|
94
|
+
@notifiers = notifiers
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def deliver(text) = @notifiers.map { |n| n.deliver(text) }.all?
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Solana::Ruby::Watchtower
|
|
4
|
+
# Reads what the cluster says about a validator. Each method takes an RPC
|
|
5
|
+
# client (anything with #call(method, params)) and returns plain data.
|
|
6
|
+
module Observe
|
|
7
|
+
extend self
|
|
8
|
+
|
|
9
|
+
# found: false when the vote account is in neither list.
|
|
10
|
+
VoteAccount = Data.define(:slot, :found, :delinquent, :last_vote, :node_pubkey)
|
|
11
|
+
|
|
12
|
+
GossipEntry = Data.define(:pubkey, :ip)
|
|
13
|
+
|
|
14
|
+
def vote_account(rpc, vote_pubkey)
|
|
15
|
+
# Both at processed: lastVote is reported at the commitment asked for, so
|
|
16
|
+
# comparing it to a confirmed slot would add a steady false lag.
|
|
17
|
+
slot = Integer(rpc.call('getSlot', [{ 'commitment' => 'processed' }]))
|
|
18
|
+
accounts = rpc.call('getVoteAccounts', [{ 'commitment' => 'processed', 'votePubkey' => vote_pubkey }])
|
|
19
|
+
|
|
20
|
+
current = Array(accounts['current']).find { |a| a['votePubkey'] == vote_pubkey }
|
|
21
|
+
delinquent = Array(accounts['delinquent']).find { |a| a['votePubkey'] == vote_pubkey }
|
|
22
|
+
account = current || delinquent
|
|
23
|
+
|
|
24
|
+
VoteAccount.new(slot: slot, found: !account.nil?, delinquent: !delinquent.nil?,
|
|
25
|
+
last_vote: account && Integer(account['lastVote']), node_pubkey: account&.fetch('nodePubkey'))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# The gossip table narrowed to the given identities and anything advertised
|
|
29
|
+
# from the given IPs; the full table is thousands of entries.
|
|
30
|
+
def gossip(rpc, pubkeys:, ips:)
|
|
31
|
+
rpc.call('getClusterNodes', []).filter_map do |node|
|
|
32
|
+
entry = GossipEntry.new(pubkey: node['pubkey'], ip: host(node['gossip']))
|
|
33
|
+
entry if pubkeys.include?(entry.pubkey) || ips.include?(entry.ip)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# "1.2.3.4:8000" or "[2001:db8::1]:8000" -> the address alone.
|
|
40
|
+
def host(addr)
|
|
41
|
+
return nil if addr.nil? || addr.empty?
|
|
42
|
+
|
|
43
|
+
addr.rpartition(':').first.delete_prefix('[').delete_suffix(']')
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'net/http'
|
|
5
|
+
require 'openssl'
|
|
6
|
+
require 'uri'
|
|
7
|
+
|
|
8
|
+
module Solana::Ruby::Watchtower
|
|
9
|
+
# A minimal JSON-RPC client for one endpoint.
|
|
10
|
+
#
|
|
11
|
+
# Short timeouts and a single retry: a check runs every 30 seconds, so a slow
|
|
12
|
+
# endpoint is better skipped in favour of the next one than waited on.
|
|
13
|
+
class Rpc
|
|
14
|
+
class Error < Solana::Ruby::Watchtower::Error; end
|
|
15
|
+
|
|
16
|
+
TRANSIENT = [Timeout::Error, IOError, EOFError, SystemCallError, SocketError, OpenSSL::SSL::SSLError].freeze
|
|
17
|
+
|
|
18
|
+
attr_reader :label
|
|
19
|
+
|
|
20
|
+
def initialize(url, timeout: 10, open_timeout: 5, attempts: 2, sleeper: ->(s) { sleep(s) })
|
|
21
|
+
@uri = URI.parse(url)
|
|
22
|
+
@timeout = timeout
|
|
23
|
+
@open_timeout = open_timeout
|
|
24
|
+
@attempts = attempts
|
|
25
|
+
@sleeper = sleeper
|
|
26
|
+
@id = 0
|
|
27
|
+
# Host only: providers put API keys in the path or query string, and the
|
|
28
|
+
# label ends up in alerts and logs.
|
|
29
|
+
@label = @uri.host || 'rpc'
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def call(method, params = [])
|
|
33
|
+
attempt = 0
|
|
34
|
+
begin
|
|
35
|
+
attempt += 1
|
|
36
|
+
post(method, params)
|
|
37
|
+
rescue *TRANSIENT, RetryableHttp => e
|
|
38
|
+
if attempt < @attempts
|
|
39
|
+
@sleeper.call(1)
|
|
40
|
+
retry
|
|
41
|
+
end
|
|
42
|
+
raise Error, describe(e)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
# 429 and 5xx are worth a second try; any other HTTP status is not.
|
|
49
|
+
class RetryableHttp < StandardError; end
|
|
50
|
+
|
|
51
|
+
def post(method, params)
|
|
52
|
+
@id += 1
|
|
53
|
+
body = JSON.generate(jsonrpc: '2.0', id: @id, method: method, params: params)
|
|
54
|
+
response = Net::HTTP.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https',
|
|
55
|
+
open_timeout: @open_timeout, read_timeout: @timeout) do |http|
|
|
56
|
+
http.post(@uri.request_uri, body, 'Content-Type' => 'application/json')
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
code = response.code.to_i
|
|
60
|
+
raise RetryableHttp, "HTTP #{code}" if code == 429 || code >= 500
|
|
61
|
+
raise Error, "HTTP #{code}" unless response.is_a?(Net::HTTPSuccess)
|
|
62
|
+
|
|
63
|
+
parsed = JSON.parse(response.body.to_s)
|
|
64
|
+
if parsed['error']
|
|
65
|
+
raise Error, "RPC error #{parsed['error']['code']}: #{parsed['error']['message']}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
parsed.fetch('result') { raise Error, 'response has no result' }
|
|
69
|
+
rescue JSON::ParserError
|
|
70
|
+
raise Error, 'response is not JSON'
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# The class and message, never a URL: some exceptions quote the request
|
|
74
|
+
# line, and the API key may be in it.
|
|
75
|
+
def describe(error)
|
|
76
|
+
return error.message if error.is_a?(RetryableHttp)
|
|
77
|
+
|
|
78
|
+
"#{error.class.name.split('::').last}: #{error.message.gsub(@uri.to_s, @label)}"
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Endpoints tried in order; the first to answer wins.
|
|
83
|
+
class RpcPool
|
|
84
|
+
class AllFailed < Solana::Ruby::Watchtower::Error; end
|
|
85
|
+
|
|
86
|
+
Answer = Data.define(:value, :label)
|
|
87
|
+
|
|
88
|
+
def self.from_urls(urls, timeout: 10)
|
|
89
|
+
new(urls.map { |url| Rpc.new(url, timeout: timeout) })
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# clients respond to #label and #call(method, params).
|
|
93
|
+
def initialize(clients)
|
|
94
|
+
raise ArgumentError, 'no RPC endpoints' if clients.empty?
|
|
95
|
+
|
|
96
|
+
@clients = clients
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Yields each client until one gets through the block without raising, and
|
|
100
|
+
# returns its result. Raises AllFailed naming every endpoint's error.
|
|
101
|
+
def first_answer
|
|
102
|
+
errors = []
|
|
103
|
+
@clients.each do |client|
|
|
104
|
+
return Answer.new(value: yield(client), label: client.label)
|
|
105
|
+
rescue StandardError => e
|
|
106
|
+
errors << "#{client.label}: #{e.message}"
|
|
107
|
+
end
|
|
108
|
+
raise AllFailed, errors.join('; ')
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Solana::Ruby::Watchtower
|
|
4
|
+
# Runs a Monitor's checks on their intervals until told to stop.
|
|
5
|
+
#
|
|
6
|
+
# Single-threaded on purpose: a check never overlaps the previous run of
|
|
7
|
+
# itself, and a slow run delays the next one instead of piling up behind it.
|
|
8
|
+
# A check that raises is logged and tried again on schedule; the vote check
|
|
9
|
+
# then skips its heartbeat ping, which is how a crash loop gets noticed.
|
|
10
|
+
class Runner
|
|
11
|
+
def initialize(monitor, logger:, votes_every:, gossip_every:,
|
|
12
|
+
now: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }, sleeper: ->(s) { sleep(s) })
|
|
13
|
+
@monitor = monitor
|
|
14
|
+
@logger = logger
|
|
15
|
+
@schedule = { 'votes' => votes_every, 'gossip' => gossip_every }
|
|
16
|
+
@now = now
|
|
17
|
+
@sleeper = sleeper
|
|
18
|
+
@stop = false
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def stop! = @stop = true
|
|
22
|
+
|
|
23
|
+
def stopped? = @stop
|
|
24
|
+
|
|
25
|
+
def run
|
|
26
|
+
due = @schedule.keys.to_h { |check| [check, @now.call] }
|
|
27
|
+
until @stop
|
|
28
|
+
due.each do |check, at|
|
|
29
|
+
break if @stop
|
|
30
|
+
next if @now.call < at
|
|
31
|
+
|
|
32
|
+
run_check(check)
|
|
33
|
+
# From when it was due, so the cadence does not drift by each run's
|
|
34
|
+
# duration; but never into the past after a long stall.
|
|
35
|
+
due[check] = [at + @schedule[check], @now.call].max
|
|
36
|
+
end
|
|
37
|
+
# Short naps, so a stop is honoured within a second.
|
|
38
|
+
wait = due.values.min - @now.call
|
|
39
|
+
@sleeper.call([[wait, 1].min, 0.05].max) if wait.positive? && !@stop
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def run_check(check)
|
|
46
|
+
@monitor.run(check)
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
@logger.error("#{check} check crashed: #{e.class}: #{e.message}")
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Solana::Ruby::Watchtower
|
|
4
|
+
module Stores
|
|
5
|
+
# States as rows of an ActiveRecord model, for running the monitor inside a
|
|
6
|
+
# Rails app. The gem does not depend on Rails; pass in a model whose table
|
|
7
|
+
# has these columns (README has the migration):
|
|
8
|
+
#
|
|
9
|
+
# key string (unique), alerting boolean, consecutive_failures integer,
|
|
10
|
+
# failing_since / last_alerted_at / last_checked_at datetime,
|
|
11
|
+
# last_message text, data json/jsonb
|
|
12
|
+
class ActiveRecordStore
|
|
13
|
+
def initialize(model)
|
|
14
|
+
@model = model
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def fetch(key)
|
|
18
|
+
record = @model.find_by(key: key.to_s)
|
|
19
|
+
return CheckState.new(key: key) unless record
|
|
20
|
+
|
|
21
|
+
CheckState.new(**CheckState::ATTRIBUTES.to_h { |attr| [attr, record.public_send(attr)] })
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def save(state)
|
|
25
|
+
record = @model.find_or_initialize_by(key: state.key)
|
|
26
|
+
record.assign_attributes(
|
|
27
|
+
(CheckState::ATTRIBUTES - [:key]).to_h { |attr| [attr, state.public_send(attr)] }
|
|
28
|
+
)
|
|
29
|
+
record.save!
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def all = @model.order(:key).map { |record| fetch(record.key) }
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
module Solana::Ruby::Watchtower
|
|
7
|
+
module Stores
|
|
8
|
+
# All states in one JSON file, rewritten atomically on every save, so a
|
|
9
|
+
# restart neither repeats an alert already sent nor forgets one that is
|
|
10
|
+
# firing. Meant for a single monitor process.
|
|
11
|
+
class FileStore
|
|
12
|
+
def initialize(path)
|
|
13
|
+
@path = path
|
|
14
|
+
@lock = Mutex.new
|
|
15
|
+
@states = read
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
attr_reader :path
|
|
19
|
+
|
|
20
|
+
def fetch(key)
|
|
21
|
+
@lock.synchronize do
|
|
22
|
+
hash = @states[key.to_s]
|
|
23
|
+
hash ? CheckState.from_h(hash) : CheckState.new(key: key)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def save(state)
|
|
28
|
+
@lock.synchronize do
|
|
29
|
+
@states[state.key] = JSON.parse(JSON.generate(state.to_h))
|
|
30
|
+
write
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def all = @lock.synchronize { @states.values.map { |h| CheckState.from_h(h) } }
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def read
|
|
39
|
+
return {} unless File.exist?(@path)
|
|
40
|
+
|
|
41
|
+
JSON.parse(File.read(@path))
|
|
42
|
+
rescue JSON::ParserError => e
|
|
43
|
+
# Starting over costs a repeated alert; refusing to start costs the monitor.
|
|
44
|
+
warn "solana-watchtower: #{@path} is not valid JSON (#{e.message}); starting with empty state"
|
|
45
|
+
{}
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def write
|
|
49
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
|
50
|
+
tmp = "#{@path}.tmp"
|
|
51
|
+
File.write(tmp, JSON.pretty_generate(@states))
|
|
52
|
+
File.rename(tmp, @path)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Solana::Ruby::Watchtower
|
|
4
|
+
# Where CheckStates live between runs. A store answers #fetch(key), returning a
|
|
5
|
+
# CheckState (a fresh one for a key it has never seen), and #save(state).
|
|
6
|
+
module Stores
|
|
7
|
+
# Lost on restart, so a restart forgets an alert that is firing and fires it
|
|
8
|
+
# again. For tests and one-off runs; the daemon uses FileStore.
|
|
9
|
+
class MemoryStore
|
|
10
|
+
def initialize
|
|
11
|
+
@states = {}
|
|
12
|
+
@lock = Mutex.new
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def fetch(key)
|
|
16
|
+
@lock.synchronize do
|
|
17
|
+
hash = @states[key.to_s]
|
|
18
|
+
hash ? CheckState.from_h(hash) : CheckState.new(key: key)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def save(state)
|
|
23
|
+
@lock.synchronize { @states[state.key] = Marshal.load(Marshal.dump(state.to_h)) }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def all = @lock.synchronize { @states.values.map { |h| CheckState.from_h(h) } }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|