tlm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +36 -0
- data/LICENSE +21 -0
- data/README.md +159 -0
- data/lib/tlm/active_job.rb +30 -0
- data/lib/tlm/backtrace.rb +118 -0
- data/lib/tlm/breadcrumbs.rb +22 -0
- data/lib/tlm/config.rb +121 -0
- data/lib/tlm/context.rb +53 -0
- data/lib/tlm/event.rb +88 -0
- data/lib/tlm/heartbeat.rb +74 -0
- data/lib/tlm/http.rb +43 -0
- data/lib/tlm/job_args.rb +36 -0
- data/lib/tlm/rack.rb +103 -0
- data/lib/tlm/railtie.rb +98 -0
- data/lib/tlm/sender.rb +298 -0
- data/lib/tlm/sidekiq.rb +58 -0
- data/lib/tlm/source.rb +44 -0
- data/lib/tlm/tasks.rb +10 -0
- data/lib/tlm/test_command.rb +56 -0
- data/lib/tlm/text.rb +21 -0
- data/lib/tlm/version.rb +3 -0
- data/lib/tlm.rb +209 -0
- metadata +64 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
module Tlm
|
|
2
|
+
# Tells Tlm the app is up: one heartbeat when it starts, then one an
|
|
3
|
+
# hour, so a healthy app with no errors still shows as set up, along
|
|
4
|
+
# with its release, environment and SDK version.
|
|
5
|
+
class Heartbeat
|
|
6
|
+
INTERVAL = 60 * 60
|
|
7
|
+
|
|
8
|
+
def self.payload(config)
|
|
9
|
+
{
|
|
10
|
+
'v' => 1,
|
|
11
|
+
'sdk' => { 'name' => 'tlm', 'version' => VERSION },
|
|
12
|
+
'environment' => config.environment,
|
|
13
|
+
'release' => config.release,
|
|
14
|
+
'server' => Event.server,
|
|
15
|
+
'runtime' => runtime
|
|
16
|
+
}
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.runtime
|
|
20
|
+
runtime = { 'name' => 'ruby', 'version' => RUBY_VERSION, 'engine' => RUBY_ENGINE }
|
|
21
|
+
runtime['rails'] = ::Rails.version.to_s if defined?(::Rails) && ::Rails.respond_to?(:version)
|
|
22
|
+
runtime
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def initialize(sender:, config:, interval: INTERVAL)
|
|
26
|
+
@sender = sender
|
|
27
|
+
@config = config
|
|
28
|
+
@interval = interval
|
|
29
|
+
@thread = nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# The first heartbeat is queued right here, so it is on its way by the
|
|
33
|
+
# time `start` returns.
|
|
34
|
+
def start
|
|
35
|
+
return self if @thread&.alive?
|
|
36
|
+
|
|
37
|
+
safe_beat
|
|
38
|
+
@thread = Thread.new { run }.tap do |thread|
|
|
39
|
+
thread.name = 'tlm-heartbeat'
|
|
40
|
+
thread.report_on_exception = false
|
|
41
|
+
end
|
|
42
|
+
self
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def stop
|
|
46
|
+
@thread&.kill&.join(1)
|
|
47
|
+
@thread = nil
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def after_fork
|
|
51
|
+
running = !@thread.nil?
|
|
52
|
+
@thread = nil
|
|
53
|
+
start if running
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def beat = @sender.heartbeat(self.class.payload(@config))
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def run
|
|
61
|
+
loop do
|
|
62
|
+
sleep @interval
|
|
63
|
+
safe_beat
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# A heartbeat that can't be built must not end the next ones.
|
|
68
|
+
def safe_beat
|
|
69
|
+
beat
|
|
70
|
+
rescue StandardError
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
data/lib/tlm/http.rb
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
require 'net/http'
|
|
2
|
+
|
|
3
|
+
module Tlm
|
|
4
|
+
# One request to Tlm, as every send makes it.
|
|
5
|
+
class Http
|
|
6
|
+
Response = Data.define(:status, :headers, :body) do
|
|
7
|
+
def header(name) = headers[name.downcase]
|
|
8
|
+
def success? = status.between?(200, 299)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def initialize(key:, endpoint:, open_timeout: 5, read_timeout: 10)
|
|
12
|
+
@key = key
|
|
13
|
+
@endpoint = endpoint
|
|
14
|
+
@open_timeout = open_timeout
|
|
15
|
+
@read_timeout = read_timeout
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def post(path, body)
|
|
19
|
+
uri = URI("#{@endpoint}#{path}")
|
|
20
|
+
response = connect(uri) { |http| http.request(request(uri, body)) }
|
|
21
|
+
headers = response.each_header.to_h { |name, value| [ name.downcase, value ] }
|
|
22
|
+
|
|
23
|
+
Response.new(status: response.code.to_i, headers:, body: response.body.to_s)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def connect(uri, &)
|
|
29
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https',
|
|
30
|
+
open_timeout: @open_timeout, read_timeout: @read_timeout,
|
|
31
|
+
write_timeout: @read_timeout, &)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def request(uri, body)
|
|
35
|
+
Net::HTTP::Post.new(uri).tap do |request|
|
|
36
|
+
request['authorization'] = "Bearer #{@key}"
|
|
37
|
+
request['content-type'] = 'application/json'
|
|
38
|
+
request['user-agent'] = "tlm-ruby/#{VERSION}"
|
|
39
|
+
request.body = body
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
data/lib/tlm/job_args.rb
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
module Tlm
|
|
2
|
+
# The arguments a failed job ran with, as its `args` tag, so the issue
|
|
3
|
+
# shows the input that broke it. Values under keys that look secret are
|
|
4
|
+
# sent as "[FILTERED]", at any depth; records as their global id; past
|
|
5
|
+
# 2 KB of JSON, the first 2 KB of it, ending in "…[cut]".
|
|
6
|
+
module JobArgs
|
|
7
|
+
BYTES = 2048
|
|
8
|
+
FILTERED = '[FILTERED]'.freeze
|
|
9
|
+
CUT = '…[cut]'.freeze
|
|
10
|
+
SECRET = /password|passwd|secret|token|api_?key|authorization|cookie|session|dsn|credential|private|(^|[_.-])(key|auth)($|[_.-])/i
|
|
11
|
+
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# The tag for `args`, or nil when job args are turned off or can't be
|
|
15
|
+
# walked.
|
|
16
|
+
def tag(args)
|
|
17
|
+
return unless Tlm.config.job_args && !args.nil?
|
|
18
|
+
|
|
19
|
+
filtered = Event.safe(filter(args))
|
|
20
|
+
json = JSON.generate(filtered)
|
|
21
|
+
json.bytesize <= BYTES ? filtered : "#{Text.cut(json, BYTES)}#{CUT}"
|
|
22
|
+
rescue StandardError, SystemStackError
|
|
23
|
+
nil
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def filter(value)
|
|
27
|
+
case value
|
|
28
|
+
when Hash then value.to_h { |key, item| [ key, secret?(key) ? FILTERED : filter(item) ] }
|
|
29
|
+
when Array then value.map { |item| filter(item) }
|
|
30
|
+
else value.respond_to?(:to_global_id) ? value.to_global_id.to_s : value
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def secret?(key) = (key.is_a?(String) || key.is_a?(Symbol)) && SECRET.match?(key.to_s)
|
|
35
|
+
end
|
|
36
|
+
end
|
data/lib/tlm/rack.rb
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
require 'rack'
|
|
2
|
+
|
|
3
|
+
module Tlm
|
|
4
|
+
# Attaches the current request to errors reported while handling it, and
|
|
5
|
+
# reports the exceptions that escape the app.
|
|
6
|
+
#
|
|
7
|
+
# use Tlm::Rack
|
|
8
|
+
#
|
|
9
|
+
# Credentials (cookies, authorization and the like) are never sent.
|
|
10
|
+
# Exceptions that mean the client's mistake (a 4xx) aren't reported.
|
|
11
|
+
class Rack
|
|
12
|
+
SECRET_HEADERS = %w[ cookie authorization set-cookie proxy-authorization x-api-key ].freeze
|
|
13
|
+
SECRET_HEADER = /token|secret|password|api-key|apikey/
|
|
14
|
+
# Where frameworks that turn an exception into a 500 leave it.
|
|
15
|
+
RESCUED = %w[ rack.exception sinatra.error ].freeze
|
|
16
|
+
|
|
17
|
+
def self.request(env)
|
|
18
|
+
request = ::Rack::Request.new(env)
|
|
19
|
+
{ 'method' => request.request_method, 'url' => request.url, 'headers' => headers(env) }
|
|
20
|
+
rescue StandardError
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.headers(env)
|
|
25
|
+
env.each_with_object({}) do |(name, value), headers|
|
|
26
|
+
next unless name.start_with?('HTTP_') || %w[ CONTENT_TYPE CONTENT_LENGTH ].include?(name)
|
|
27
|
+
|
|
28
|
+
header = name.delete_prefix('HTTP_').downcase.tr('_', '-')
|
|
29
|
+
headers[header] = value.to_s unless secret?(header)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.secret?(header) = SECRET_HEADERS.include?(header) || SECRET_HEADER.match?(header)
|
|
34
|
+
|
|
35
|
+
# Whether `exception` stands for a 4xx answer.
|
|
36
|
+
def self.client_error?(exception)
|
|
37
|
+
status = status_of(exception)
|
|
38
|
+
status.is_a?(Integer) && status.between?(400, 499)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def self.status_of(exception)
|
|
42
|
+
if exception.respond_to?(:http_status)
|
|
43
|
+
exception.http_status
|
|
44
|
+
elsif defined?(::ActionDispatch::ExceptionWrapper)
|
|
45
|
+
::ActionDispatch::ExceptionWrapper.status_code_for_exception(exception.class.name)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Starts and ends each request with a clean context, from outside other
|
|
50
|
+
# middleware that log around the app.
|
|
51
|
+
class Reset
|
|
52
|
+
def initialize(app)
|
|
53
|
+
@app = app
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def call(env)
|
|
57
|
+
Context.reset
|
|
58
|
+
@app.call(env)
|
|
59
|
+
ensure
|
|
60
|
+
Context.reset
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def initialize(app)
|
|
65
|
+
@app = app
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# On a keep-alive connection one thread serves many requests: what the
|
|
69
|
+
# last one set must not show up in this one's errors.
|
|
70
|
+
def call(env)
|
|
71
|
+
start(env)
|
|
72
|
+
response = @app.call(env)
|
|
73
|
+
report_rescued(env, response)
|
|
74
|
+
response
|
|
75
|
+
rescue Exception => exception
|
|
76
|
+
report(exception)
|
|
77
|
+
raise
|
|
78
|
+
ensure
|
|
79
|
+
Context.reset
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
def start(env)
|
|
85
|
+
Context.user = nil
|
|
86
|
+
Context.request = self.class.request(env)
|
|
87
|
+
Context.request_id = env['action_dispatch.request_id'] || env['HTTP_X_REQUEST_ID']
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def report(exception)
|
|
91
|
+
return unless Tlm.reportable?(exception) && !self.class.client_error?(exception)
|
|
92
|
+
|
|
93
|
+
Tlm.capture_exception(exception)
|
|
94
|
+
rescue StandardError
|
|
95
|
+
nil
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def report_rescued(env, response)
|
|
99
|
+
exception = RESCUED.lazy.filter_map { |key| env[key] }.first
|
|
100
|
+
report(exception) if exception.is_a?(Exception) && response[0].to_i >= 500
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
data/lib/tlm/railtie.rb
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
require 'logger'
|
|
2
|
+
require 'rails/railtie'
|
|
3
|
+
|
|
4
|
+
module Tlm
|
|
5
|
+
# Sets Tlm up in a Rails app: the Rack middleware, ActiveJob and Sidekiq
|
|
6
|
+
# failures, the Rails error reporter, log lines and queries as
|
|
7
|
+
# breadcrumbs, heartbeats, and `bin/rails tlm:test`.
|
|
8
|
+
class Railtie < ::Rails::Railtie
|
|
9
|
+
# Reports what reaches `Rails.error`: errors in jobs, runners, and
|
|
10
|
+
# whatever the app reports itself.
|
|
11
|
+
class ErrorSubscriber
|
|
12
|
+
# Rails reports every exception it answers a request with, a 404's too.
|
|
13
|
+
REQUEST_SOURCE = 'application.action_dispatch'.freeze
|
|
14
|
+
|
|
15
|
+
def report(error, handled:, severity:, context:, source: nil)
|
|
16
|
+
return if source == REQUEST_SOURCE && Tlm::Rack.client_error?(error)
|
|
17
|
+
|
|
18
|
+
tags = context.filter_map { |key, value| [ key.to_s, value.to_s ] if scalar?(value) }.to_h
|
|
19
|
+
tags['source'] = source if source
|
|
20
|
+
tags['handled'] = handled.to_s
|
|
21
|
+
Tlm.capture_exception(error, level: severity, tags:)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def scalar?(value) = value.is_a?(String) || value.is_a?(Symbol) || value.is_a?(Numeric) || value == true || value == false
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The app's log lines, info and above, as breadcrumbs.
|
|
30
|
+
class BreadcrumbLogger < ::Logger
|
|
31
|
+
LEVELS = %w[ debug info warning error fatal error ].freeze
|
|
32
|
+
COLOR = /\e\[[\d;]*m/
|
|
33
|
+
|
|
34
|
+
def initialize = super(nil)
|
|
35
|
+
|
|
36
|
+
# Fixed: Rails sets the level of every logger it broadcasts to, and
|
|
37
|
+
# debug lines would push the request's own off the list.
|
|
38
|
+
def level = ::Logger::INFO
|
|
39
|
+
def level=(_level); end
|
|
40
|
+
|
|
41
|
+
def add(severity, message = nil, progname = nil)
|
|
42
|
+
severity ||= ::Logger::UNKNOWN
|
|
43
|
+
return true if severity < level
|
|
44
|
+
|
|
45
|
+
message = block_given? ? yield : progname if message.nil?
|
|
46
|
+
Breadcrumbs.add('log', LEVELS.fetch(severity, 'error'), text(message)) unless message.nil?
|
|
47
|
+
true
|
|
48
|
+
end
|
|
49
|
+
alias log add
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def text(message) = (message.is_a?(String) ? message : message.inspect).gsub(COLOR, '').strip
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# "QUERY User Load 3.2 ms" for each database query.
|
|
57
|
+
class QueryBreadcrumbs
|
|
58
|
+
SKIPPED = %w[ SCHEMA ].freeze
|
|
59
|
+
|
|
60
|
+
def call(event)
|
|
61
|
+
payload = event.payload
|
|
62
|
+
return if SKIPPED.include?(payload[:name])
|
|
63
|
+
|
|
64
|
+
kind = payload[:cached] ? 'CACHE' : 'QUERY'
|
|
65
|
+
message = [ kind, payload[:name], format('%.1f ms', event.duration) ].compact.join(' ')
|
|
66
|
+
Breadcrumbs.add('active_record', :debug, message)
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Tlm::Rack goes inside DebugExceptions, to see exceptions before Rails
|
|
71
|
+
# turns them into error pages; the context is reset around the whole
|
|
72
|
+
# stack, since Rails logs before and after it.
|
|
73
|
+
initializer 'tlm.middleware' do |app|
|
|
74
|
+
app.config.middleware.insert 0, Tlm::Rack::Reset
|
|
75
|
+
app.config.middleware.insert_after ::ActionDispatch::DebugExceptions, Tlm::Rack
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
initializer 'tlm.active_job' do
|
|
79
|
+
ActiveSupport.on_load(:active_job) { prepend Tlm::ActiveJob }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
config.after_initialize do
|
|
83
|
+
Tlm::Railtie.capture if Tlm.start
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
rake_tasks do
|
|
87
|
+
require 'tlm/tasks'
|
|
88
|
+
task 'tlm:test' => :environment
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def self.capture
|
|
92
|
+
::Rails.error.subscribe(ErrorSubscriber.new)
|
|
93
|
+
::Rails.logger.broadcast_to(BreadcrumbLogger.new) if ::Rails.logger.respond_to?(:broadcast_to)
|
|
94
|
+
ActiveSupport::Notifications.subscribe('sql.active_record', QueryBreadcrumbs.new)
|
|
95
|
+
Tlm::Sidekiq.install if defined?(::Sidekiq)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
data/lib/tlm/sender.rb
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
require 'json'
|
|
2
|
+
require 'monitor'
|
|
3
|
+
|
|
4
|
+
module Tlm
|
|
5
|
+
# Sends events to Tlm from a background thread, one at a time, so that a
|
|
6
|
+
# slow or unreachable Tlm never slows the app down.
|
|
7
|
+
#
|
|
8
|
+
# * The queue is bounded (1000 by default); when full, the oldest item is
|
|
9
|
+
# dropped.
|
|
10
|
+
# * A 5xx, a 425 or a network error resends the same event (same uuid, so
|
|
11
|
+
# it is counted once) after a backoff of 1s, 2s, 4s, … up to a minute,
|
|
12
|
+
# at most 8 times.
|
|
13
|
+
# * An event that can't be encoded as JSON is dropped.
|
|
14
|
+
# * A 429 drops that event (it was already counted) and pauses sending
|
|
15
|
+
# for `Retry-After` seconds.
|
|
16
|
+
# * 400, 404, 413 and 422 drop the event; resending can't help.
|
|
17
|
+
# * 401 or 403 mean the key isn't accepted: nothing more is sent, events
|
|
18
|
+
# or heartbeats, until the process restarts with another.
|
|
19
|
+
#
|
|
20
|
+
# Heartbeats share the queue, with their own rules, all silent: at most
|
|
21
|
+
# one waits (a newer one replaces it); a 429 drops it and holds later
|
|
22
|
+
# ones back for `Retry-After` without pausing events; 401 or 403 stop
|
|
23
|
+
# heartbeats; any other 4xx but 425 drops it.
|
|
24
|
+
class Sender
|
|
25
|
+
MAX_QUEUE = 1000
|
|
26
|
+
FIRST_BACKOFF = 1
|
|
27
|
+
MAX_BACKOFF = 60
|
|
28
|
+
MAX_ATTEMPTS = 8
|
|
29
|
+
DEFAULT_RETRY_AFTER = 60
|
|
30
|
+
WARNING_INTERVAL = 60
|
|
31
|
+
PATHS = { event: '/api/events', heartbeat: '/api/heartbeats' }.freeze
|
|
32
|
+
REFUSED = [ 401, 403 ].freeze
|
|
33
|
+
REJECTED = [ 400, 404, 413, 422 ].freeze
|
|
34
|
+
|
|
35
|
+
Item = Data.define(:kind, :payload) do
|
|
36
|
+
def heartbeat? = kind == :heartbeat
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
MONOTONIC = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
40
|
+
|
|
41
|
+
def initialize(transport:, logger:, max_queue: MAX_QUEUE, clock: MONOTONIC)
|
|
42
|
+
@transport = transport
|
|
43
|
+
@logger = logger
|
|
44
|
+
@max_queue = max_queue
|
|
45
|
+
@clock = clock
|
|
46
|
+
@queue = []
|
|
47
|
+
@in_flight = nil
|
|
48
|
+
@resume_at = nil
|
|
49
|
+
@backoff = FIRST_BACKOFF
|
|
50
|
+
@attempts = 0
|
|
51
|
+
@stopped = false
|
|
52
|
+
@heartbeats_after = nil
|
|
53
|
+
@last_warning = nil
|
|
54
|
+
@thread = nil
|
|
55
|
+
setup_monitor
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Queues `event`. Never blocks.
|
|
59
|
+
def send_event(event) = enqueue(Item.new(:event, event))
|
|
60
|
+
|
|
61
|
+
# Queues a heartbeat, replacing one still waiting. Never blocks.
|
|
62
|
+
def heartbeat(payload) = enqueue(Item.new(:heartbeat, payload))
|
|
63
|
+
|
|
64
|
+
def start
|
|
65
|
+
synchronize do
|
|
66
|
+
@thread = nil unless @thread&.alive?
|
|
67
|
+
@thread ||= Thread.new { run }.tap do |thread|
|
|
68
|
+
thread.name = 'tlm-sender'
|
|
69
|
+
thread.report_on_exception = false
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
self
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def stop
|
|
76
|
+
thread = synchronize { @thread.tap { @thread = nil } }
|
|
77
|
+
thread&.kill&.join(1)
|
|
78
|
+
synchronize do
|
|
79
|
+
@queue.unshift(@in_flight) if @in_flight
|
|
80
|
+
@in_flight = nil
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Waits up to `timeout` seconds for everything queued to be sent, or
|
|
85
|
+
# only the events. False when it couldn't be.
|
|
86
|
+
def flush(timeout, heartbeats: true)
|
|
87
|
+
deadline = now + timeout
|
|
88
|
+
synchronize do
|
|
89
|
+
until idle?(heartbeats)
|
|
90
|
+
left = deadline - now
|
|
91
|
+
return false unless left.positive?
|
|
92
|
+
|
|
93
|
+
@wakeup.wait(left)
|
|
94
|
+
end
|
|
95
|
+
true
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Sends the next item right here, if one is due. False when none is.
|
|
100
|
+
def deliver_next
|
|
101
|
+
item = synchronize { take }
|
|
102
|
+
deliver(item) if item
|
|
103
|
+
!item.nil?
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def queued = synchronize { @queue.dup }
|
|
107
|
+
|
|
108
|
+
# Seconds until sending resumes, nil when it isn't paused.
|
|
109
|
+
def paused_for = synchronize { pause_left }
|
|
110
|
+
|
|
111
|
+
def stopped? = synchronize { @stopped }
|
|
112
|
+
|
|
113
|
+
# Threads don't survive a fork, and neither should the parent's queue,
|
|
114
|
+
# which the parent sends itself.
|
|
115
|
+
def after_fork
|
|
116
|
+
running = !@thread.nil?
|
|
117
|
+
setup_monitor
|
|
118
|
+
@thread = nil
|
|
119
|
+
@queue = []
|
|
120
|
+
@in_flight = nil
|
|
121
|
+
start if running
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
private
|
|
125
|
+
|
|
126
|
+
def setup_monitor
|
|
127
|
+
@monitor = Monitor.new
|
|
128
|
+
@wakeup = @monitor.new_cond
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def synchronize(&) = @monitor.synchronize(&)
|
|
132
|
+
|
|
133
|
+
def now = @clock.call
|
|
134
|
+
|
|
135
|
+
def run
|
|
136
|
+
loop { deliver(synchronize { wait_for_item }) }
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def wait_for_item
|
|
140
|
+
loop do
|
|
141
|
+
item = take
|
|
142
|
+
return item if item
|
|
143
|
+
|
|
144
|
+
@wakeup.wait(pause_left)
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def take
|
|
149
|
+
return if @in_flight || pause_left || @queue.empty?
|
|
150
|
+
|
|
151
|
+
@in_flight = @queue.shift
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def idle?(heartbeats)
|
|
155
|
+
pending = [ *@queue, *@in_flight ]
|
|
156
|
+
heartbeats ? pending.empty? : pending.all?(&:heartbeat?)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def pause_left
|
|
160
|
+
return unless @resume_at
|
|
161
|
+
|
|
162
|
+
left = @resume_at - now
|
|
163
|
+
return left if left.positive?
|
|
164
|
+
|
|
165
|
+
@resume_at = nil
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def enqueue(item)
|
|
169
|
+
synchronize do
|
|
170
|
+
next if @stopped
|
|
171
|
+
next if item.heartbeat? && !heartbeats_allowed?
|
|
172
|
+
|
|
173
|
+
@queue.reject!(&:heartbeat?) if item.heartbeat?
|
|
174
|
+
@queue.shift if @queue.size >= @max_queue
|
|
175
|
+
@queue << item
|
|
176
|
+
@wakeup.broadcast
|
|
177
|
+
end
|
|
178
|
+
nil
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def heartbeats_allowed?
|
|
182
|
+
case @heartbeats_after
|
|
183
|
+
when nil then true
|
|
184
|
+
when :never then false
|
|
185
|
+
else now >= @heartbeats_after
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def deliver(item)
|
|
190
|
+
result = post(item)
|
|
191
|
+
synchronize do
|
|
192
|
+
@in_flight = nil
|
|
193
|
+
handle(item, result)
|
|
194
|
+
@wakeup.broadcast
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def post(item)
|
|
199
|
+
body = encode(item.payload)
|
|
200
|
+
body ? @transport.post(PATHS.fetch(item.kind), body) : :unencodable
|
|
201
|
+
rescue StandardError => error
|
|
202
|
+
error
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def encode(payload)
|
|
206
|
+
JSON.generate(payload)
|
|
207
|
+
rescue JSON::GeneratorError, EncodingError, SystemStackError
|
|
208
|
+
nil
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def handle(item, result)
|
|
212
|
+
status = result.status if result.respond_to?(:status)
|
|
213
|
+
|
|
214
|
+
if result == :unencodable
|
|
215
|
+
warn("Tlm dropped an event that couldn't be encoded as JSON")
|
|
216
|
+
elsif status&.between?(200, 299)
|
|
217
|
+
reset_backoff
|
|
218
|
+
elsif item.heartbeat?
|
|
219
|
+
handle_heartbeat(item, status, result)
|
|
220
|
+
else
|
|
221
|
+
handle_event(item, status, result)
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def handle_event(item, status, result)
|
|
226
|
+
if status == 429
|
|
227
|
+
@resume_at = now + retry_after(result)
|
|
228
|
+
reset_backoff
|
|
229
|
+
elsif REFUSED.include?(status)
|
|
230
|
+
refuse(status)
|
|
231
|
+
elsif REJECTED.include?(status)
|
|
232
|
+
warn("Tlm rejected an event (HTTP #{status}#{detail(status, result)}); check the key and the endpoint")
|
|
233
|
+
@attempts = 0
|
|
234
|
+
else
|
|
235
|
+
retry_later(item)
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def handle_heartbeat(item, status, result)
|
|
240
|
+
case status
|
|
241
|
+
when 429
|
|
242
|
+
@heartbeats_after = now + retry_after(result)
|
|
243
|
+
reset_backoff
|
|
244
|
+
when *REFUSED
|
|
245
|
+
@heartbeats_after = :never
|
|
246
|
+
@attempts = 0
|
|
247
|
+
when 425, nil, 500.. then retry_later(item)
|
|
248
|
+
else @attempts = 0
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def refuse(status)
|
|
253
|
+
@logger.warn("Tlm refused the key (HTTP #{status}); nothing more will be sent: check the DSN")
|
|
254
|
+
@stopped = true
|
|
255
|
+
@queue.clear
|
|
256
|
+
@attempts = 0
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# An item that fails every time must not stand in front of every other
|
|
260
|
+
# one forever.
|
|
261
|
+
def retry_later(item)
|
|
262
|
+
@resume_at = now + @backoff
|
|
263
|
+
@backoff = [ @backoff * 2, MAX_BACKOFF ].min
|
|
264
|
+
@attempts += 1
|
|
265
|
+
|
|
266
|
+
if @attempts >= MAX_ATTEMPTS
|
|
267
|
+
warn("Tlm gave up on an event after #{@attempts} tries") unless item.heartbeat?
|
|
268
|
+
@attempts = 0
|
|
269
|
+
elsif item.heartbeat? && @queue.any?(&:heartbeat?)
|
|
270
|
+
@attempts = 0
|
|
271
|
+
else
|
|
272
|
+
@queue.unshift(item)
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def reset_backoff
|
|
277
|
+
@backoff = FIRST_BACKOFF
|
|
278
|
+
@attempts = 0
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def retry_after(response)
|
|
282
|
+
seconds = response.header('retry-after').to_s[/\A\s*(\d+)/, 1].to_i
|
|
283
|
+
seconds.positive? ? seconds : DEFAULT_RETRY_AFTER
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def detail(status, response)
|
|
287
|
+
status == 422 && !response.body.empty? ? ": #{Text.cut(response.body, 500)}" : ''
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# At most once a minute, so a bad key doesn't flood the app's logs.
|
|
291
|
+
def warn(message)
|
|
292
|
+
return if @last_warning && now - @last_warning < WARNING_INTERVAL
|
|
293
|
+
|
|
294
|
+
@last_warning = now
|
|
295
|
+
@logger.warn(message)
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
end
|
data/lib/tlm/sidekiq.rb
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
require 'sidekiq'
|
|
2
|
+
|
|
3
|
+
module Tlm
|
|
4
|
+
# Reports Sidekiq job failures, tagged with the job, queue, job id,
|
|
5
|
+
# attempt, max attempts and arguments (see Tlm::JobArgs). The Railtie
|
|
6
|
+
# installs it; elsewhere, call `Tlm::Sidekiq.install` where Sidekiq is
|
|
7
|
+
# configured.
|
|
8
|
+
#
|
|
9
|
+
# A job that raises is reported on every attempt. ActiveJob jobs run by
|
|
10
|
+
# Sidekiq are reported once, by Tlm::ActiveJob.
|
|
11
|
+
module Sidekiq
|
|
12
|
+
DEFAULT_MAX_RETRIES = 25
|
|
13
|
+
|
|
14
|
+
class Middleware
|
|
15
|
+
include ::Sidekiq::ServerMiddleware
|
|
16
|
+
|
|
17
|
+
def call(_instance, job, queue)
|
|
18
|
+
Context.isolate do
|
|
19
|
+
yield
|
|
20
|
+
rescue Exception => exception
|
|
21
|
+
Tlm.capture_exception(exception, tags: Tlm::Sidekiq.tags(job, queue)) if Tlm.reportable?(exception)
|
|
22
|
+
raise
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
module_function
|
|
28
|
+
|
|
29
|
+
def install = ::Sidekiq.configure_server { |config| add(config) }
|
|
30
|
+
|
|
31
|
+
def add(config)
|
|
32
|
+
config.server_middleware { |chain| chain.add(Middleware) unless chain.exists?(Middleware) }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def tags(job, queue)
|
|
36
|
+
{
|
|
37
|
+
'job' => (job['wrapped'] || job['class']).to_s,
|
|
38
|
+
'queue' => queue.to_s,
|
|
39
|
+
'job_id' => job['jid'].to_s,
|
|
40
|
+
'attempt' => attempt(job).to_s,
|
|
41
|
+
'max_attempts' => max_attempts(job['retry'])&.to_s,
|
|
42
|
+
'args' => JobArgs.tag(job['args'])
|
|
43
|
+
}.compact
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# `retry_count` is set on the first failure: 0 while the second attempt
|
|
47
|
+
# runs.
|
|
48
|
+
def attempt(job) = job['retry_count'].is_a?(Integer) ? job['retry_count'] + 2 : 1
|
|
49
|
+
|
|
50
|
+
def max_attempts(retry_option)
|
|
51
|
+
case retry_option
|
|
52
|
+
when true then DEFAULT_MAX_RETRIES + 1
|
|
53
|
+
when Integer then retry_option + 1
|
|
54
|
+
when false then 1
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|