zharp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +33 -0
- data/LICENSE +21 -0
- data/README.md +170 -0
- data/lib/generators/zharp/install/install_generator.rb +39 -0
- data/lib/generators/zharp/install/templates/zharp.rb.tt +15 -0
- data/lib/zharp/bodies.rb +386 -0
- data/lib/zharp/capture.rb +361 -0
- data/lib/zharp/config.rb +129 -0
- data/lib/zharp/context.rb +133 -0
- data/lib/zharp/db_errors.rb +229 -0
- data/lib/zharp/errors.rb +158 -0
- data/lib/zharp/fork.rb +70 -0
- data/lib/zharp/headers.rb +63 -0
- data/lib/zharp/instrumentations.rb +17 -0
- data/lib/zharp/logs.rb +128 -0
- data/lib/zharp/rack.rb +148 -0
- data/lib/zharp/railtie.rb +47 -0
- data/lib/zharp/setup.rb +206 -0
- data/lib/zharp/sidekiq.rb +13 -0
- data/lib/zharp/source.rb +118 -0
- data/lib/zharp/stamp.rb +124 -0
- data/lib/zharp/version.rb +5 -0
- data/lib/zharp.rb +218 -0
- metadata +341 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zharp
|
|
4
|
+
# Failed database queries as errors that say where they came from.
|
|
5
|
+
#
|
|
6
|
+
# The driver's instrumentation (pg, mysql2, trilogy) records the exception on the query's span
|
|
7
|
+
# first, before it has travelled anywhere. So:
|
|
8
|
+
# - that record_exception is held, not reported, and the app's call site is taken there;
|
|
9
|
+
# - if the exception then escapes (Action Pack, Rails.error, the Rack middleware, a job runner),
|
|
10
|
+
# that one report carries the query: statement, system, database, code;
|
|
11
|
+
# - if the app caught it, it is reported as handled when the request or job ends, with the call
|
|
12
|
+
# site as its stack.
|
|
13
|
+
# One report per failure either way. ActiveRecord::StatementInvalid counts as its driver's error
|
|
14
|
+
# (its cause).
|
|
15
|
+
module DbErrors
|
|
16
|
+
DB_SYSTEM = %w[db.system db.system.name].freeze
|
|
17
|
+
STATEMENT = %w[db.query.text db.statement].freeze
|
|
18
|
+
DATABASE = %w[db.namespace db.name].freeze
|
|
19
|
+
# How long a failure outside any request waits for a handler before it is reported as caught.
|
|
20
|
+
LONE_WAIT = 1.5
|
|
21
|
+
MAX_PENDING = 200
|
|
22
|
+
MAX_FRAMES = 40
|
|
23
|
+
|
|
24
|
+
Failure = Struct.new(:error, :frames, :attributes, :span_context, :done)
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
def first(attributes, names)
|
|
29
|
+
return nil unless attributes
|
|
30
|
+
|
|
31
|
+
names.each do |name|
|
|
32
|
+
value = attributes[name]
|
|
33
|
+
return value.to_s unless value.nil? || value.to_s.empty?
|
|
34
|
+
end
|
|
35
|
+
nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The app's part of a call site, innermost first: from its own first frame on (Active Record
|
|
39
|
+
# and the driver between the app and the query are left out), never Zharp or OpenTelemetry.
|
|
40
|
+
def app_frames(locations)
|
|
41
|
+
kept = locations.reject { |l| (l.absolute_path || l.path).to_s.start_with?(Paths::ZHARP_DIR) }
|
|
42
|
+
first_app = kept.index { |l| Paths.app?(l.absolute_path || l.path) } || 0
|
|
43
|
+
kept[first_app..].first(MAX_FRAMES)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The call site as Ruby prints a stack: the first line with the message, then "from" lines.
|
|
47
|
+
def format_stack(frames, error)
|
|
48
|
+
message = Errors.scrub(error.message.to_s).rstrip
|
|
49
|
+
head = frames.first ? "#{frames.first}: #{message} (#{error.class.name})" : "#{message} (#{error.class.name})"
|
|
50
|
+
([head] + frames.drop(1).map { |f| "\tfrom #{f}" }).join("\n") + "\n"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def query_attributes(span_attributes, error)
|
|
54
|
+
out = { "db.system" => first(span_attributes, DB_SYSTEM) || "database" }
|
|
55
|
+
statement = first(span_attributes, STATEMENT) || statement_of(error)
|
|
56
|
+
out["db.query.text"] = statement[0, 4000] if statement
|
|
57
|
+
database = first(span_attributes, DATABASE)
|
|
58
|
+
out["db.namespace"] = database if database
|
|
59
|
+
code, constraint = codes_of(error)
|
|
60
|
+
out["db.error.code"] = code if code
|
|
61
|
+
out["db.error.constraint"] = constraint if constraint
|
|
62
|
+
out
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Active Record's statement, obfuscated as OpenTelemetry's pg instrumentation does.
|
|
66
|
+
def statement_of(error)
|
|
67
|
+
Errors.related(error).each do |e|
|
|
68
|
+
next unless e.respond_to?(:sql) && e.sql
|
|
69
|
+
|
|
70
|
+
return ::OpenTelemetry::Helpers::SqlProcessor.obfuscate_sql(e.sql.to_s) if defined?(::OpenTelemetry::Helpers::SqlProcessor)
|
|
71
|
+
end
|
|
72
|
+
nil
|
|
73
|
+
rescue StandardError
|
|
74
|
+
nil
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Postgres's SQLSTATE and constraint; MySQL's error number.
|
|
78
|
+
def codes_of(error)
|
|
79
|
+
Errors.related(error).each do |e|
|
|
80
|
+
if defined?(::PG::Error) && e.is_a?(::PG::Error) && e.respond_to?(:result) && e.result
|
|
81
|
+
return [e.result.error_field(::PG::PG_DIAG_SQLSTATE), e.result.error_field(::PG::PG_DIAG_CONSTRAINT_NAME)]
|
|
82
|
+
end
|
|
83
|
+
return [e.error_number.to_s, nil] if e.respond_to?(:error_number) && e.error_number
|
|
84
|
+
return [e.error_code.to_s, nil] if e.class.name.to_s.start_with?("Trilogy") && e.respond_to?(:error_code) && e.error_code
|
|
85
|
+
end
|
|
86
|
+
[nil, nil]
|
|
87
|
+
rescue StandardError
|
|
88
|
+
[nil, nil]
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Watches query spans: their failure when they end, and the end of the request or job they ran
|
|
92
|
+
# in. send.call(error, stack, attributes, span_context) reports what nothing else claimed.
|
|
93
|
+
class Processor
|
|
94
|
+
def initialize(send, queries: true)
|
|
95
|
+
@send = send
|
|
96
|
+
@queries = queries
|
|
97
|
+
@lock = Mutex.new
|
|
98
|
+
@held = {}
|
|
99
|
+
@pending = {}
|
|
100
|
+
@local_roots = {}
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Called by the patched Span#record_exception. True when held.
|
|
104
|
+
#
|
|
105
|
+
# A query span's exception, and any exception leaving a span inside a request or job (Active
|
|
106
|
+
# Record's User.create!, an HTTP call, a span of your own): it may still be rescued by the app.
|
|
107
|
+
# It is reported by whatever sees it escape the request or job (then with its query), or, when
|
|
108
|
+
# nothing does, as handled when the request or job ends. Server and job spans, and spans with
|
|
109
|
+
# no local parent, report at once.
|
|
110
|
+
def hold(span, error)
|
|
111
|
+
return false if span.kind == :server || span.kind == :consumer
|
|
112
|
+
|
|
113
|
+
span_id = span.context.span_id
|
|
114
|
+
return false if @lock.synchronize { @local_roots.key?(span_id) }
|
|
115
|
+
|
|
116
|
+
query = @queries && DbErrors.first(span.attributes, DB_SYSTEM)
|
|
117
|
+
site = query ? (caller_locations(2) || []) : nil
|
|
118
|
+
@lock.synchronize { @held[span_id] ||= [error, site] }
|
|
119
|
+
true
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def on_start(span, parent_context)
|
|
123
|
+
parent = ::OpenTelemetry::Trace.current_span(parent_context).context
|
|
124
|
+
return if parent.valid? && !parent.remote?
|
|
125
|
+
|
|
126
|
+
@lock.synchronize do
|
|
127
|
+
@local_roots[span.context.span_id] = true
|
|
128
|
+
@local_roots.shift while @local_roots.size > 10_000
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def on_finishing(span); end
|
|
133
|
+
|
|
134
|
+
def on_finish(span)
|
|
135
|
+
span_id = span.context.span_id
|
|
136
|
+
trace_id = span.context.trace_id
|
|
137
|
+
held = @lock.synchronize { @held.delete(span_id) }
|
|
138
|
+
if held && span.status.code == ::OpenTelemetry::Trace::Status::ERROR
|
|
139
|
+
error, site = held
|
|
140
|
+
if site
|
|
141
|
+
add(trace_id, Failure.new(error, DbErrors.app_frames(site), DbErrors.query_attributes(span.attributes, error), span.context, false))
|
|
142
|
+
else
|
|
143
|
+
add(trace_id, Failure.new(error, nil, {}, span.context, false))
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
root = @lock.synchronize { @local_roots.delete(span_id) }
|
|
147
|
+
return unless root || span.kind == :server || span.kind == :consumer
|
|
148
|
+
|
|
149
|
+
waiting = @lock.synchronize { @pending.key?(trace_id) }
|
|
150
|
+
return unless waiting
|
|
151
|
+
|
|
152
|
+
if held || span.status.code == ::OpenTelemetry::Trace::Status::ERROR
|
|
153
|
+
# The query was the whole trace, or its root failed: the exception may still be on its
|
|
154
|
+
# way to a handler, which reports it itself.
|
|
155
|
+
Thread.new { sleep(LONE_WAIT); flush(trace_id) }
|
|
156
|
+
else
|
|
157
|
+
flush(trace_id)
|
|
158
|
+
end
|
|
159
|
+
rescue StandardError => e
|
|
160
|
+
Zharp.debug("db errors: #{e.message}")
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def add(trace_id, failure)
|
|
164
|
+
@lock.synchronize do
|
|
165
|
+
failures = (@pending[trace_id] ||= [])
|
|
166
|
+
mine = Errors.related(failure.error)
|
|
167
|
+
# The same failure seen by two spans (the query, then Active Record's around it): once,
|
|
168
|
+
# as the outermost exception the app saw, with the query and call site of the first.
|
|
169
|
+
same = failures.find { |f| mine.include?(f.error) || Errors.related(f.error).include?(failure.error) }
|
|
170
|
+
if same
|
|
171
|
+
same.error = failure.error if mine.size > Errors.related(same.error).size
|
|
172
|
+
return
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
failures << failure
|
|
176
|
+
@pending.shift if @pending.size > MAX_PENDING
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# error is being reported: the attributes of the failed query it came from, if any, and that
|
|
181
|
+
# failure is not reported again.
|
|
182
|
+
def claim(error)
|
|
183
|
+
return nil if error.nil?
|
|
184
|
+
|
|
185
|
+
wanted = Errors.related(error)
|
|
186
|
+
@lock.synchronize do
|
|
187
|
+
@pending.each_value do |failures|
|
|
188
|
+
failures.each do |f|
|
|
189
|
+
next if f.done
|
|
190
|
+
next unless wanted.any? { |c| c.equal?(f.error) || Errors.related(f.error).include?(c) }
|
|
191
|
+
|
|
192
|
+
f.done = true
|
|
193
|
+
return f.attributes.dup
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
nil
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def flush(trace_id)
|
|
201
|
+
failures = @lock.synchronize { @pending.delete(trace_id) } || []
|
|
202
|
+
failures.each do |f|
|
|
203
|
+
next if f.done || Errors.reported?(f.error)
|
|
204
|
+
|
|
205
|
+
attributes = f.attributes.dup
|
|
206
|
+
if f.frames
|
|
207
|
+
# A query: where the app ran it (the driver's exception has no app frames of its own).
|
|
208
|
+
code = Source.context_of(f.frames.map { |l| [l.path, l.absolute_path || l.path, l.lineno] })
|
|
209
|
+
attributes[Source::ATTRIBUTE] = code if code
|
|
210
|
+
@send.call(f.error, DbErrors.format_stack(f.frames, f.error), attributes, f.span_context)
|
|
211
|
+
else
|
|
212
|
+
@send.call(f.error, nil, attributes, f.span_context)
|
|
213
|
+
end
|
|
214
|
+
rescue StandardError => e
|
|
215
|
+
Zharp.debug("db errors: #{e.message}")
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def force_flush(timeout: nil)
|
|
220
|
+
@lock.synchronize { @pending.keys }.each { |trace_id| flush(trace_id) }
|
|
221
|
+
::OpenTelemetry::SDK::Trace::Export::SUCCESS
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def shutdown(timeout: nil)
|
|
225
|
+
force_flush
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
end
|
data/lib/zharp/errors.rb
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zharp
|
|
4
|
+
# Errors: one report per exception, marked handled or not.
|
|
5
|
+
#
|
|
6
|
+
# An exception that escapes a request is seen several times: Action Pack records it on the
|
|
7
|
+
# request's span, Rails reports it to Rails.error, the Rack middleware sees it, a job runner's
|
|
8
|
+
# error handler gets it. The first sighting reports it and marks the exception object; later ones
|
|
9
|
+
# keep their span status but don't report it again.
|
|
10
|
+
module Errors
|
|
11
|
+
MARK = :@__zharp_reported
|
|
12
|
+
|
|
13
|
+
# Lines of a database error that carry the row's values: Postgres's "DETAIL: Key (email)=(…)".
|
|
14
|
+
# The statement, the constraint and the code say enough.
|
|
15
|
+
PRIVATE_LINES = /^[ \t]*DETAIL:.*(?:\r?\n|\z)/
|
|
16
|
+
|
|
17
|
+
# Values inside the messages themselves, each replaced by a quoted ?.
|
|
18
|
+
PRIVATE_VALUES = {
|
|
19
|
+
# MySQL 1062 (mysql2, trilogy): Duplicate entry 'a@b.c' for key 'users.index_users_on_email'
|
|
20
|
+
/(Duplicate entry )'.*?'(?= for key)/m => "'?'",
|
|
21
|
+
# MySQL 1366/1292: Incorrect integer value: 'abc' for column 'age' at row 1
|
|
22
|
+
/(Incorrect [\w ]+? value: )'.*?'(?= for |\s*\z)/m => "'?'",
|
|
23
|
+
# MySQL 1292: Truncated incorrect DOUBLE value: 'abc'
|
|
24
|
+
/(Truncated incorrect [\w ]+? value: )'.*?'/m => "'?'",
|
|
25
|
+
# Postgres: invalid input syntax for type integer: "abc"
|
|
26
|
+
/(invalid input syntax for type [\w ]+?: )".*?"/m => '"?"',
|
|
27
|
+
# Postgres: invalid input value for enum mood: "x"
|
|
28
|
+
/(invalid input value for enum \S+: )".*?"/m => '"?"',
|
|
29
|
+
# Postgres: value "123456789012" is out of range for type integer
|
|
30
|
+
/(value )".*?"(?= is out of range)/m => '"?"'
|
|
31
|
+
}.freeze
|
|
32
|
+
# Drivers whose messages are scrubbed. MySQL's 1048/1265/1452 and Postgres's other messages
|
|
33
|
+
# carry no values (a not-null or foreign-key violation's row is in its DETAIL line).
|
|
34
|
+
DB_ERROR_CLASSES = ["PG::", "Mysql2::", "Trilogy", "ActiveRecord::", "SQLite3::"].freeze
|
|
35
|
+
|
|
36
|
+
class << self
|
|
37
|
+
# The failed-query watcher (DbErrors::Processor) when tracing is on.
|
|
38
|
+
attr_accessor :db_errors
|
|
39
|
+
|
|
40
|
+
# Reported already: this exception itself, or one it wraps that escaped (unhandled). A wrapper
|
|
41
|
+
# and what it wraps (ActionView::Template::Error around the template's NoMethodError) are one
|
|
42
|
+
# failure. An exception the app reported as handled and then wrapped in a new one it raised
|
|
43
|
+
# doesn't hide the new one.
|
|
44
|
+
def reported?(error)
|
|
45
|
+
return false if error.nil?
|
|
46
|
+
return true if marked(error)
|
|
47
|
+
|
|
48
|
+
related(error).drop(1).any? { |e| marked(e) == :unhandled }
|
|
49
|
+
rescue StandardError
|
|
50
|
+
false
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def marked(error)
|
|
54
|
+
error.instance_variable_defined?(MARK) ? error.instance_variable_get(MARK) : nil
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Marks the exception; an unhandled one marks its causes too, so whichever of a wrapper and
|
|
58
|
+
# its cause is seen second is not reported again.
|
|
59
|
+
def mark(error, handled: false)
|
|
60
|
+
return if error.nil?
|
|
61
|
+
|
|
62
|
+
(handled ? [error] : related(error)).each do |e|
|
|
63
|
+
e.instance_variable_set(MARK, handled ? :handled : :unhandled) unless marked(e) == :unhandled
|
|
64
|
+
rescue StandardError # a frozen exception
|
|
65
|
+
nil
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# The query attributes of the failed query this error came from, if any ({} when none).
|
|
70
|
+
def query_of(error)
|
|
71
|
+
db_errors ? (db_errors.claim(error) || {}) : {}
|
|
72
|
+
rescue StandardError
|
|
73
|
+
{}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def db_error?(error)
|
|
77
|
+
related(error).any? do |e|
|
|
78
|
+
name = e.class.name.to_s
|
|
79
|
+
name.start_with?(*DB_ERROR_CLASSES)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# An exception and what it wraps (ActiveRecord::StatementInvalid's cause is the driver's error).
|
|
84
|
+
def related(error)
|
|
85
|
+
out = [error]
|
|
86
|
+
inner = error.respond_to?(:cause) ? error.cause : nil
|
|
87
|
+
while inner && out.size < 5 && !out.include?(inner)
|
|
88
|
+
out << inner
|
|
89
|
+
inner = inner.cause
|
|
90
|
+
end
|
|
91
|
+
out
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def scrub(text)
|
|
95
|
+
text = text.to_s.gsub(PRIVATE_LINES, "")
|
|
96
|
+
PRIVATE_VALUES.each { |pattern, value| text = text.gsub(pattern) { "#{Regexp.last_match(1)}#{value}" } }
|
|
97
|
+
text
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def message_of(error)
|
|
101
|
+
message = error.message.to_s
|
|
102
|
+
db_error?(error) ? scrub(message).rstrip : message
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The stack as Ruby prints it, innermost first: what OpenTelemetry's record_exception sends.
|
|
106
|
+
def stack_of(error)
|
|
107
|
+
stack = error.full_message(highlight: false, order: :top)
|
|
108
|
+
stack = scrub(stack) if db_error?(error)
|
|
109
|
+
stack.encode("UTF-8", invalid: :replace, undef: :replace, replace: "�")
|
|
110
|
+
rescue StandardError
|
|
111
|
+
"#{error.class}: #{error.message}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def attributes(error, handled)
|
|
115
|
+
attrs = {
|
|
116
|
+
"exception.type" => error.class.name.to_s,
|
|
117
|
+
"exception.message" => message_of(error),
|
|
118
|
+
"exception.stacktrace" => stack_of(error),
|
|
119
|
+
"zharp.handled" => handled ? "true" : "false"
|
|
120
|
+
}
|
|
121
|
+
code = Source.of_exception(error)
|
|
122
|
+
attrs[Source::ATTRIBUTE] = code if code
|
|
123
|
+
attrs
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Span#record_exception skips an exception already reported, holds a failed query's exception
|
|
128
|
+
# (DbErrors) and says handled=false: instrumentations call it for an exception that escaped
|
|
129
|
+
# their span, which is what an uncaught error is. A caller that knows better passes
|
|
130
|
+
# "zharp.handled" => "true".
|
|
131
|
+
module RecordException
|
|
132
|
+
def record_exception(exception, attributes: nil)
|
|
133
|
+
return if Errors.reported?(exception)
|
|
134
|
+
return if Errors.db_errors&.hold(self, exception)
|
|
135
|
+
|
|
136
|
+
Errors.mark(exception, handled: attributes.is_a?(Hash) && attributes["zharp.handled"] == "true")
|
|
137
|
+
merged = { "zharp.handled" => "false" }
|
|
138
|
+
merged.merge!(Errors.query_of(exception))
|
|
139
|
+
code = Source.of_exception(exception)
|
|
140
|
+
merged[Source::ATTRIBUTE] = code if code
|
|
141
|
+
if Errors.db_error?(exception)
|
|
142
|
+
merged["exception.message"] = Errors.message_of(exception)
|
|
143
|
+
merged["exception.stacktrace"] = Errors.stack_of(exception)
|
|
144
|
+
end
|
|
145
|
+
merged.merge!(attributes) if attributes
|
|
146
|
+
super(exception, attributes: merged)
|
|
147
|
+
rescue StandardError => e
|
|
148
|
+
Zharp.debug("record_exception: #{e.message}")
|
|
149
|
+
super
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def self.patch_record_exception
|
|
154
|
+
span = ::OpenTelemetry::SDK::Trace::Span
|
|
155
|
+
span.prepend(RecordException) unless span.ancestors.include?(RecordException)
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
data/lib/zharp/fork.rb
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
|
|
5
|
+
module Zharp
|
|
6
|
+
# Forking servers (Puma in cluster mode, Unicorn, Resque, Pitchfork). OpenTelemetry's batch
|
|
7
|
+
# processors restart their thread after a fork, but its OTLP exporters keep the parent's
|
|
8
|
+
# Net::HTTP connection: two processes writing to one socket. Zharp hooks Process._fork and gives
|
|
9
|
+
# each child exporter a connection of its own (the parent's is left alone), so preload_app!
|
|
10
|
+
# needs nothing.
|
|
11
|
+
module Fork
|
|
12
|
+
@exporters = []
|
|
13
|
+
@callbacks = []
|
|
14
|
+
|
|
15
|
+
class << self
|
|
16
|
+
attr_reader :exporters
|
|
17
|
+
|
|
18
|
+
def register(exporter)
|
|
19
|
+
@exporters << exporter
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def after_fork(&block)
|
|
23
|
+
@callbacks << block
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def in_child
|
|
27
|
+
@exporters.each { |exporter| reconnect(exporter) }
|
|
28
|
+
@callbacks.each do |callback|
|
|
29
|
+
callback.call
|
|
30
|
+
rescue StandardError => e
|
|
31
|
+
Zharp.debug("after fork: #{e.message}")
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# A fresh, unstarted connection with the same settings; the inherited one is never touched
|
|
36
|
+
# (finishing it would close the parent's TLS session).
|
|
37
|
+
def reconnect(exporter)
|
|
38
|
+
old = exporter.instance_variable_get(:@http)
|
|
39
|
+
return unless old.is_a?(::Net::HTTP)
|
|
40
|
+
|
|
41
|
+
http = ::Net::HTTP.new(old.address, old.port)
|
|
42
|
+
http.use_ssl = old.use_ssl?
|
|
43
|
+
http.verify_mode = old.verify_mode
|
|
44
|
+
http.ca_file = old.ca_file if old.ca_file
|
|
45
|
+
http.cert = old.cert if old.cert
|
|
46
|
+
http.key = old.key if old.key
|
|
47
|
+
http.keep_alive_timeout = old.keep_alive_timeout
|
|
48
|
+
exporter.instance_variable_set(:@http, http)
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
Zharp.debug("reconnect after fork: #{e.message}")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def install
|
|
54
|
+
return if @installed
|
|
55
|
+
|
|
56
|
+
@installed = true
|
|
57
|
+
::Process.singleton_class.prepend(Hook)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Process._fork (Ruby 3.1+) is what fork, Process.fork and Kernel#fork all go through.
|
|
62
|
+
module Hook
|
|
63
|
+
def _fork
|
|
64
|
+
pid = super
|
|
65
|
+
Fork.in_child if pid.zero?
|
|
66
|
+
pid
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zharp
|
|
4
|
+
# Request and response headers on the request's server span, as http.request.header.<name> and
|
|
5
|
+
# http.response.header.<name>. OpenTelemetry's Rack instrumentation only records the headers on
|
|
6
|
+
# its allow-list; Zharp records all of them. Secret headers (authorization, cookies, API keys,
|
|
7
|
+
# tokens, sessions) are recorded as [REDACTED]: you see one was sent, never its value.
|
|
8
|
+
module Headers
|
|
9
|
+
# Header names (searched case-insensitively) whose values are never sent: the Python SDK's list.
|
|
10
|
+
SECRET_HEADERS = %w[
|
|
11
|
+
authorization cookie password passwd secret token api-?key access-?key
|
|
12
|
+
private-?key credential signature session csrf xsrf jwt ^auth$ ^sid$
|
|
13
|
+
].freeze
|
|
14
|
+
REDACTED = "[REDACTED]"
|
|
15
|
+
MAX_VALUE = 1000
|
|
16
|
+
MAX_HEADERS = 100
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
# The secret-header pattern, or nil when header capture is off.
|
|
21
|
+
# capture_headers: false (or ZHARP_CAPTURE_HEADERS=false) turns it off;
|
|
22
|
+
# capture_headers: { redact: ["x-tenant"] } filters more names.
|
|
23
|
+
def settings(option, env_value)
|
|
24
|
+
return nil if option == false || Config.flag(env_value) == false
|
|
25
|
+
|
|
26
|
+
extra = option.is_a?(Hash) ? Array(option[:redact] || option["redact"]).map(&:to_s) : []
|
|
27
|
+
Regexp.new((SECRET_HEADERS + extra).join("|"), Regexp::IGNORECASE)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The request's headers from a Rack env, by lower-case name.
|
|
31
|
+
def from_env(env)
|
|
32
|
+
out = {}
|
|
33
|
+
env.each do |key, value|
|
|
34
|
+
name =
|
|
35
|
+
if key.start_with?("HTTP_") then key[5..]
|
|
36
|
+
elsif key == "CONTENT_TYPE" || key == "CONTENT_LENGTH" then key
|
|
37
|
+
end
|
|
38
|
+
next unless name && value.is_a?(String)
|
|
39
|
+
|
|
40
|
+
out[name.downcase.tr("_", "-")] = value
|
|
41
|
+
break if out.size >= MAX_HEADERS
|
|
42
|
+
end
|
|
43
|
+
out
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Span attributes for one side's headers, secrets redacted.
|
|
47
|
+
def attributes(side, headers, secret)
|
|
48
|
+
out = {}
|
|
49
|
+
return out unless secret && headers
|
|
50
|
+
|
|
51
|
+
headers.each do |name, value|
|
|
52
|
+
name = name.to_s.downcase
|
|
53
|
+
next if name.empty? || name.start_with?("rack.")
|
|
54
|
+
|
|
55
|
+
text = value.is_a?(Array) ? value.join(", ") : value.to_s
|
|
56
|
+
text = REDACTED if secret.match?(name)
|
|
57
|
+
out["http.#{side}.header.#{name}"] = text.length > MAX_VALUE ? text[0, MAX_VALUE] : text
|
|
58
|
+
break if out.size >= MAX_HEADERS
|
|
59
|
+
end
|
|
60
|
+
out
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# The OpenTelemetry instrumentations Zharp ships with (see zharp.gemspec). Requiring one only
|
|
4
|
+
# registers it; Setup installs those whose library the app has loaded.
|
|
5
|
+
module Zharp
|
|
6
|
+
INSTRUMENTATIONS = %w[
|
|
7
|
+
rack rails action_pack action_view active_record active_job active_support action_mailer
|
|
8
|
+
active_storage concurrent_ruby sinatra grape net_http faraday http httpx pg mysql2 trilogy
|
|
9
|
+
redis sidekiq resque delayed_job
|
|
10
|
+
].freeze
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
Zharp::INSTRUMENTATIONS.each do |name|
|
|
14
|
+
require "opentelemetry-instrumentation-#{name}"
|
|
15
|
+
rescue LoadError
|
|
16
|
+
nil
|
|
17
|
+
end
|
data/lib/zharp/logs.rb
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zharp
|
|
4
|
+
# What the app logs, sent from the SDK. Off by default: on a server the Zharp agent reads the
|
|
5
|
+
# app's output and brings those lines in, linked to their request by the stamp. With
|
|
6
|
+
# ZHARP_LOGS=true (where there is no agent), each line written through Ruby's Logger (Rails.logger,
|
|
7
|
+
# lograge) or semantic_logger is sent as an OpenTelemetry log record in its request's context.
|
|
8
|
+
#
|
|
9
|
+
# Either way, an exception passed to a logger (logger.error(e)) is an error: reported once,
|
|
10
|
+
# handled.
|
|
11
|
+
module Logs
|
|
12
|
+
SEVERITY = {
|
|
13
|
+
0 => ["DEBUG", 5], 1 => ["INFO", 9], 2 => ["WARN", 13], 3 => ["ERROR", 17], 4 => ["FATAL", 21], 5 => ["INFO", 9]
|
|
14
|
+
}.freeze
|
|
15
|
+
SEMANTIC = { trace: ["TRACE", 1], debug: ["DEBUG", 5], info: ["INFO", 9], warn: ["WARN", 13], error: ["ERROR", 17], fatal: ["FATAL", 21] }.freeze
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
attr_accessor :sending
|
|
19
|
+
end
|
|
20
|
+
@sending = false
|
|
21
|
+
|
|
22
|
+
module_function
|
|
23
|
+
|
|
24
|
+
def emit(severity_text, severity_number, body, attributes = {}, context = nil)
|
|
25
|
+
logger = Zharp.otel_logger
|
|
26
|
+
return unless logger
|
|
27
|
+
|
|
28
|
+
logger.on_emit(
|
|
29
|
+
timestamp: Time.now,
|
|
30
|
+
severity_text: severity_text,
|
|
31
|
+
severity_number: severity_number,
|
|
32
|
+
body: body.to_s,
|
|
33
|
+
attributes: attributes,
|
|
34
|
+
context: context || ::OpenTelemetry::Context.current
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def own?(logger)
|
|
39
|
+
logger.equal?(::OpenTelemetry.logger) || logger.instance_variable_defined?(:@__zharp_own)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# On Ruby's Logger#add: an exception is reported; with logs on, the line is sent.
|
|
43
|
+
module LoggerPatch
|
|
44
|
+
def add(severity, message = nil, progname = nil, &block)
|
|
45
|
+
return super if Thread.current[:__zharp_in_log] || Logs.own?(self)
|
|
46
|
+
|
|
47
|
+
severity ||= ::Logger::UNKNOWN
|
|
48
|
+
return super if severity < level
|
|
49
|
+
|
|
50
|
+
Thread.current[:__zharp_in_log] = true
|
|
51
|
+
begin
|
|
52
|
+
text = message
|
|
53
|
+
if text.nil?
|
|
54
|
+
if block
|
|
55
|
+
text = yield
|
|
56
|
+
block = nil
|
|
57
|
+
message = text
|
|
58
|
+
else
|
|
59
|
+
text = progname
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
if text.is_a?(Exception)
|
|
63
|
+
Zharp.report(text, handled: true) unless Errors.reported?(text)
|
|
64
|
+
elsif Logs.sending && text && !Logs.duplicate?(severity, text)
|
|
65
|
+
name, number = SEVERITY[severity] || SEVERITY[5]
|
|
66
|
+
Logs.emit(name, number, text.is_a?(String) ? text.strip : text.inspect, { "log.logger" => (progname || "ruby").to_s })
|
|
67
|
+
end
|
|
68
|
+
rescue StandardError => e
|
|
69
|
+
Zharp.debug("logs: #{e.message}")
|
|
70
|
+
ensure
|
|
71
|
+
Thread.current[:__zharp_in_log] = false
|
|
72
|
+
end
|
|
73
|
+
super(severity, message, progname, &block)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Rails 7.1+ broadcasts one line to several loggers (stdout and a file): sent once.
|
|
78
|
+
def duplicate?(severity, text)
|
|
79
|
+
key = [severity, text.object_id]
|
|
80
|
+
last = Thread.current[:__zharp_last_log]
|
|
81
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
82
|
+
Thread.current[:__zharp_last_log] = [key, now]
|
|
83
|
+
last && last[0] == key && now - last[1] < 0.05
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def install_logger
|
|
87
|
+
::Logger.prepend(LoggerPatch) unless ::Logger.ancestors.include?(LoggerPatch)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# semantic_logger: an appender that sends each line, in the context it was logged in.
|
|
91
|
+
# Included in a SemanticLogger::Subscriber subclass made when semantic_logger is present.
|
|
92
|
+
module SemanticAppender
|
|
93
|
+
def log(log)
|
|
94
|
+
# semantic_logger writes on its own thread: the line's context is the one taken when it was logged.
|
|
95
|
+
ids = log.respond_to?(:__zharp_ids) ? log.__zharp_ids : nil
|
|
96
|
+
context = nil
|
|
97
|
+
if ids
|
|
98
|
+
span_context = ::OpenTelemetry::Trace::SpanContext.new(
|
|
99
|
+
trace_id: [ids[0]].pack("H*"), span_id: [ids[1]].pack("H*"), trace_flags: ::OpenTelemetry::Trace::TraceFlags::SAMPLED
|
|
100
|
+
)
|
|
101
|
+
context = ::OpenTelemetry::Trace.context_with_span(::OpenTelemetry::Trace.non_recording_span(span_context))
|
|
102
|
+
end
|
|
103
|
+
if log.exception && !Errors.reported?(log.exception)
|
|
104
|
+
Zharp.report(log.exception, handled: true, context: context)
|
|
105
|
+
end
|
|
106
|
+
return true unless Logs.sending
|
|
107
|
+
|
|
108
|
+
name, number = SEMANTIC[log.level] || SEMANTIC[:info]
|
|
109
|
+
attributes = { "log.logger" => log.name.to_s }
|
|
110
|
+
(log.named_tags || {}).each { |k, v| attributes["log.tag.#{k}"] = v.to_s }
|
|
111
|
+
(log.payload || {}).each { |k, v| attributes[k.to_s] = Context.plain_value(v) } if log.payload.is_a?(Hash)
|
|
112
|
+
Logs.emit(name, number, log.message.to_s, attributes, context)
|
|
113
|
+
true
|
|
114
|
+
rescue StandardError => e
|
|
115
|
+
Zharp.debug("semantic_logger: #{e.message}")
|
|
116
|
+
true
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def install_semantic_logger
|
|
121
|
+
return false unless defined?(::SemanticLogger::Subscriber)
|
|
122
|
+
return true if ::SemanticLogger.appenders.any? { |a| a.is_a?(SemanticAppender) }
|
|
123
|
+
|
|
124
|
+
::SemanticLogger.add_appender(appender: Class.new(::SemanticLogger::Subscriber) { include SemanticAppender }.new)
|
|
125
|
+
true
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|