chronos-ruby 1.0.0 → 1.2.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/README.md +27 -13
- data/contracts/apm-batch-v1.schema.json +51 -1
- data/docs/adr/ADR-010-opentelemetry-interoperability.md +3 -3
- data/docs/adr/ADR-015-bounded-apm-aggregation.md +3 -3
- data/docs/adr/ADR-019-bounded-query-diagnostics.md +44 -0
- data/docs/architecture.md +5 -2
- data/docs/compatibility.md +12 -18
- data/docs/configuration.md +27 -2
- data/docs/data-collected.md +9 -3
- data/docs/examples/plain-ruby.md +6 -0
- data/docs/modules/apm-aggregation.md +18 -5
- data/docs/modules/context.md +1 -1
- data/docs/modules/external-http.md +3 -2
- data/docs/modules/sidekiq-legacy.md +1 -1
- data/docs/modules/sql-monitoring.md +60 -6
- data/docs/modules/telemetry-events.md +1 -1
- data/docs/performance.md +19 -3
- data/docs/privacy-lgpd.md +6 -3
- data/docs/protocol-v1.md +1 -1
- data/docs/release-1.1-readiness.md +51 -0
- data/docs/security-review.md +7 -3
- data/docs/troubleshooting.md +6 -0
- data/lib/chronos/adapters/fiber_local_context_store.rb +57 -0
- data/lib/chronos/agent.rb +19 -3
- data/lib/chronos/application/apm_aggregator.rb +179 -29
- data/lib/chronos/configuration/apm_validation.rb +53 -1
- data/lib/chronos/configuration/validation.rb +2 -2
- data/lib/chronos/configuration.rb +21 -2
- data/lib/chronos/core/metric_aggregate.rb +69 -7
- data/lib/chronos/core/sql_query_analyzer.rb +309 -0
- data/lib/chronos/core/trace_context.rb +38 -0
- data/lib/chronos/integrations/active_job.rb +2 -2
- data/lib/chronos/integrations/faraday.rb +76 -0
- data/lib/chronos/integrations/net_http.rb +5 -1
- data/lib/chronos/integrations/opentelemetry.rb +44 -0
- data/lib/chronos/integrations/rack/middleware.rb +8 -2
- data/lib/chronos/integrations/sidekiq.rb +1 -1
- data/lib/chronos/ports/query_inspector.rb +23 -0
- data/lib/chronos/rails/active_record_query_inspector.rb +235 -0
- data/lib/chronos/rails/error_reporter_subscriber.rb +39 -0
- data/lib/chronos/rails/installer.rb +13 -0
- data/lib/chronos/rails/notifications_subscriber.rb +175 -2
- data/lib/chronos/rails.rb +2 -0
- data/lib/chronos/version.rb +2 -2
- data/lib/chronos.rb +6 -0
- metadata +46 -38
|
@@ -13,28 +13,51 @@ module Chronos
|
|
|
13
13
|
# batches = aggregator.flush
|
|
14
14
|
# @errors Invalid observations are ignored and never escape to the application.
|
|
15
15
|
# @performance Group, transaction, query, bucket, and batch counts are strictly bounded.
|
|
16
|
-
class ApmAggregator
|
|
16
|
+
class ApmAggregator # rubocop:disable Metrics/ClassLength
|
|
17
17
|
include ApmErrorClassifier
|
|
18
18
|
|
|
19
19
|
METRIC_TYPES = %w(request query job external_http).freeze
|
|
20
|
-
|
|
20
|
+
QUERY_ERROR_SIGNALS = {
|
|
21
|
+
"connection_error" => /Connection|NoDatabase|Adapter/i,
|
|
22
|
+
"deadlock" => /Deadlock/i,
|
|
23
|
+
"query_timeout" => /StatementTimeout|QueryCanceled|QueryTimeout|Timeout/i,
|
|
24
|
+
"pool_timeout" => /ConnectionTimeout|PoolTimeout/i,
|
|
25
|
+
"lock_timeout" => /LockWaitTimeout|LockTimeout/i,
|
|
26
|
+
"constraint_violation" => /RecordNotUnique|NotNullViolation|ForeignKeyViolation|InvalidForeignKey|Constraint/i
|
|
27
|
+
}.freeze
|
|
28
|
+
SIGNAL_DIAGNOSTICS = {
|
|
29
|
+
"slow_query" => ["warning", "performance", "Query duration reached the slow threshold"].freeze,
|
|
30
|
+
"long_transaction" => ["warning", "transaction", "Transaction SQL reached the long threshold"].freeze,
|
|
31
|
+
"connection_error" => ["error", "connection", "Database connection failed"].freeze,
|
|
32
|
+
"deadlock" => ["error", "locking", "The database reported a deadlock"].freeze,
|
|
33
|
+
"query_timeout" => ["error", "timeout", "The database query timed out"].freeze,
|
|
34
|
+
"pool_timeout" => ["error", "connection_pool", "Database connection-pool checkout timed out"].freeze,
|
|
35
|
+
"lock_timeout" => ["error", "locking", "The database lock wait timed out"].freeze,
|
|
36
|
+
"constraint_violation" => ["error", "constraint", "The database rejected a constraint"].freeze
|
|
37
|
+
}.freeze
|
|
38
|
+
def initialize(config, options = {})
|
|
21
39
|
@config = config
|
|
40
|
+
@clock = options[:clock] || proc { Time.now.to_f }
|
|
22
41
|
@mutex = Mutex.new
|
|
23
42
|
@groups = {}
|
|
24
43
|
@transactions = {}
|
|
25
44
|
@observations = 0
|
|
26
45
|
@dropped_groups = 0
|
|
46
|
+
@dropped_trace_trackers = 0
|
|
47
|
+
@expired_trace_trackers = 0
|
|
48
|
+
@dropped_query_fingerprints = 0
|
|
27
49
|
end
|
|
28
50
|
|
|
29
51
|
def record(event_type, payload = {}, context = {})
|
|
30
52
|
return [] unless @config.apm_enabled
|
|
31
53
|
|
|
32
54
|
@mutex.synchronize do
|
|
55
|
+
expire_transactions
|
|
33
56
|
type = event_type.to_s
|
|
34
57
|
data = hash(payload)
|
|
35
58
|
execution = hash(context)
|
|
36
|
-
observe_component(type, data, execution)
|
|
37
|
-
add_metric(type, data, execution) if aggregate_metric?(type, data)
|
|
59
|
+
correlation = observe_component(type, data, execution)
|
|
60
|
+
add_metric(type, data, execution, correlation) if aggregate_metric?(type, data)
|
|
38
61
|
@observations += 1 if aggregate_metric?(type, data)
|
|
39
62
|
@observations >= @config.apm_flush_count ? drain_locked : []
|
|
40
63
|
end
|
|
@@ -53,6 +76,9 @@ module Chronos
|
|
|
53
76
|
{
|
|
54
77
|
"groups" => @groups.length, "dropped_groups" => @dropped_groups,
|
|
55
78
|
"transactions" => @transactions.length,
|
|
79
|
+
"dropped_trace_trackers" => @dropped_trace_trackers,
|
|
80
|
+
"expired_trace_trackers" => @expired_trace_trackers,
|
|
81
|
+
"dropped_query_fingerprints" => @dropped_query_fingerprints,
|
|
56
82
|
"tracked_queries" => @transactions.values.inject(0) do |total, transaction|
|
|
57
83
|
total + transaction["queries"].length
|
|
58
84
|
end
|
|
@@ -66,7 +92,7 @@ module Chronos
|
|
|
66
92
|
METRIC_TYPES.include?(type) && !(type == "request" && payload["kind"].to_s == "view")
|
|
67
93
|
end
|
|
68
94
|
|
|
69
|
-
def add_metric(type, payload, context)
|
|
95
|
+
def add_metric(type, payload, context, correlation)
|
|
70
96
|
dimensions = dimensions_for(type, payload)
|
|
71
97
|
key = metric_key(type, dimensions)
|
|
72
98
|
aggregate = @groups[key]
|
|
@@ -80,53 +106,77 @@ module Chronos
|
|
|
80
106
|
end
|
|
81
107
|
duration = non_negative(payload["duration_ms"] || payload[:duration_ms])
|
|
82
108
|
status = ["request", "external_http"].include?(type) ? payload["status"] || payload[:status] : nil
|
|
83
|
-
signals = signals_for(type, payload, context, duration)
|
|
109
|
+
signals = signals_for(type, payload, context, duration, correlation)
|
|
110
|
+
diagnostics = diagnostics_for(type, payload, context, signals, correlation)
|
|
84
111
|
breakdown = breakdown_for(type, payload, context, duration)
|
|
85
112
|
aggregate.observe(
|
|
86
|
-
duration, error?(type, payload), breakdown, signals,
|
|
113
|
+
duration, error?(type, payload), breakdown, signals,
|
|
114
|
+
:status => status, :diagnostics => diagnostics,
|
|
115
|
+
:query_analysis => type == "query" ? hash(payload["analysis"] || payload[:analysis]) : {}
|
|
87
116
|
)
|
|
88
117
|
end
|
|
89
118
|
|
|
90
119
|
def observe_component(type, payload, context)
|
|
91
|
-
return if type == "job"
|
|
120
|
+
return {} if type == "job"
|
|
92
121
|
|
|
93
122
|
trace_id = trace_id(context)
|
|
94
|
-
return if trace_id.empty?
|
|
123
|
+
return {} if trace_id.empty?
|
|
95
124
|
|
|
96
125
|
transaction = transaction_for(trace_id)
|
|
97
|
-
return unless transaction
|
|
126
|
+
return {} unless transaction
|
|
127
|
+
|
|
128
|
+
transaction["last_seen_at"] = now
|
|
98
129
|
|
|
99
130
|
category = component_category(type, payload)
|
|
100
131
|
return observe_query(transaction, payload) if type == "query" && category.nil?
|
|
101
|
-
return unless category
|
|
132
|
+
return {} unless category
|
|
102
133
|
|
|
103
134
|
transaction["breakdown_ms"][category] ||= 0.0
|
|
104
135
|
transaction["breakdown_ms"][category] += non_negative(payload["duration_ms"] || payload[:duration_ms])
|
|
105
|
-
observe_query(transaction, payload) if type == "query"
|
|
136
|
+
return observe_query(transaction, payload) if type == "query"
|
|
137
|
+
|
|
138
|
+
{}
|
|
106
139
|
end
|
|
107
140
|
|
|
108
141
|
def transaction_for(trace_id)
|
|
109
142
|
existing = @transactions[trace_id]
|
|
110
143
|
return existing if existing
|
|
111
|
-
|
|
144
|
+
if @transactions.length >= @config.apm_max_groups
|
|
145
|
+
@dropped_trace_trackers += 1
|
|
146
|
+
return nil
|
|
147
|
+
end
|
|
112
148
|
|
|
113
|
-
@transactions[trace_id] = {
|
|
149
|
+
@transactions[trace_id] = {
|
|
150
|
+
"breakdown_ms" => {}, "signals" => {}, "queries" => {}, "diagnostics" => [],
|
|
151
|
+
"last_seen_at" => now
|
|
152
|
+
}
|
|
114
153
|
end
|
|
115
154
|
|
|
116
155
|
def observe_query(transaction, payload)
|
|
117
156
|
fingerprint = (payload["fingerprint"] || payload[:fingerprint]).to_s
|
|
118
|
-
return if fingerprint.empty?
|
|
157
|
+
return {} if fingerprint.empty?
|
|
119
158
|
|
|
120
159
|
queries = transaction["queries"]
|
|
121
|
-
|
|
160
|
+
unless queries.key?(fingerprint) || queries.length < @config.apm_max_queries_per_request
|
|
161
|
+
@dropped_query_fingerprints += 1
|
|
162
|
+
return {}
|
|
163
|
+
end
|
|
122
164
|
|
|
123
165
|
queries[fingerprint] ||= 0
|
|
124
166
|
queries[fingerprint] += 1
|
|
125
167
|
count = queries[fingerprint]
|
|
126
168
|
slow = non_negative(payload["duration_ms"]) >= @config.apm_slow_query_threshold_ms
|
|
127
169
|
increment_signal(transaction, "slow_query") if slow
|
|
128
|
-
|
|
129
|
-
|
|
170
|
+
current = {}
|
|
171
|
+
if count > 1
|
|
172
|
+
increment_signal(transaction, "repeated_query")
|
|
173
|
+
current["repeated_query"] = 1
|
|
174
|
+
end
|
|
175
|
+
if n_plus_one_candidate?(payload) && count == @config.apm_n_plus_one_threshold
|
|
176
|
+
increment_signal(transaction, "possible_n_plus_one")
|
|
177
|
+
current["possible_n_plus_one"] = 1
|
|
178
|
+
end
|
|
179
|
+
{"signals" => current, "query_count" => count}
|
|
130
180
|
end
|
|
131
181
|
|
|
132
182
|
def breakdown_for(type, payload, context, duration)
|
|
@@ -145,13 +195,75 @@ module Chronos
|
|
|
145
195
|
explicit
|
|
146
196
|
end
|
|
147
197
|
|
|
148
|
-
def signals_for(type, payload, context, duration)
|
|
198
|
+
def signals_for(type, payload, context, duration, correlation)
|
|
149
199
|
return request_signals(context) if type == "request"
|
|
150
|
-
return query_signals(payload, duration) if type == "query"
|
|
200
|
+
return merge_integer_signals(query_signals(payload, duration), hash(correlation["signals"])) if type == "query"
|
|
151
201
|
|
|
152
202
|
{}
|
|
153
203
|
end
|
|
154
204
|
|
|
205
|
+
def diagnostics_for(type, payload, context, signals, correlation)
|
|
206
|
+
return request_diagnostics(context) if type == "request"
|
|
207
|
+
return [] unless type == "query"
|
|
208
|
+
|
|
209
|
+
analysis = hash(payload["analysis"] || payload[:analysis])
|
|
210
|
+
result = Array(analysis["diagnostics"] || analysis[:diagnostics]).first(20)
|
|
211
|
+
result += signal_diagnostics(signals, correlation)
|
|
212
|
+
error_class = (payload["error_class"] || payload[:error_class]).to_s
|
|
213
|
+
unless error_class.empty?
|
|
214
|
+
result << diagnostic("query_execution_error", "error", "execution",
|
|
215
|
+
"The database query raised an exception", "error_class" => error_class[0, 128])
|
|
216
|
+
end
|
|
217
|
+
retain_transaction_diagnostics(context, result)
|
|
218
|
+
result.first(20)
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def signal_diagnostics(signals, correlation)
|
|
222
|
+
result = signal_error_diagnostics(signals)
|
|
223
|
+
if hash(correlation["signals"])["repeated_query"]
|
|
224
|
+
result << diagnostic("repeated_query", "info", "query_pattern",
|
|
225
|
+
"The same query fingerprint repeated in one trace",
|
|
226
|
+
"occurrence" => correlation["query_count"])
|
|
227
|
+
end
|
|
228
|
+
if hash(correlation["signals"])["possible_n_plus_one"]
|
|
229
|
+
result << diagnostic("possible_n_plus_one", "warning", "n_plus_one",
|
|
230
|
+
"Repeated SELECT reached the N+1 threshold",
|
|
231
|
+
"occurrence" => correlation["query_count"])
|
|
232
|
+
item = diagnostic("n_plus_one_eager_loading", "suggestion", "n_plus_one",
|
|
233
|
+
"Review eager loading or batch loading for this query fingerprint")
|
|
234
|
+
item["recommendation"] = "Use includes, preload, eager_load, or an equivalent bounded batch query " \
|
|
235
|
+
"when semantics allow"
|
|
236
|
+
result << item
|
|
237
|
+
end
|
|
238
|
+
result
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def signal_error_diagnostics(signals)
|
|
242
|
+
SIGNAL_DIAGNOSTICS.each_with_object([]) do |(code, values), result|
|
|
243
|
+
result << diagnostic(code, values[0], values[1], values[2]) if signals[code]
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def diagnostic(code, severity, category, message, evidence = {})
|
|
248
|
+
{"code" => code, "severity" => severity, "category" => category,
|
|
249
|
+
"message" => message, "evidence" => evidence}
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def retain_transaction_diagnostics(context, diagnostics)
|
|
253
|
+
transaction = @transactions[trace_id(context)]
|
|
254
|
+
return unless transaction
|
|
255
|
+
|
|
256
|
+
diagnostics.each do |item|
|
|
257
|
+
next if transaction["diagnostics"].any? { |existing| existing["code"] == item["code"] }
|
|
258
|
+
transaction["diagnostics"] << item if transaction["diagnostics"].length < 20
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def request_diagnostics(context)
|
|
263
|
+
transaction = @transactions[trace_id(context)]
|
|
264
|
+
transaction ? transaction["diagnostics"].dup : []
|
|
265
|
+
end
|
|
266
|
+
|
|
155
267
|
def request_signals(context)
|
|
156
268
|
transaction = @transactions[trace_id(context)]
|
|
157
269
|
transaction ? transaction["signals"].dup : {}
|
|
@@ -162,16 +274,30 @@ module Chronos
|
|
|
162
274
|
signals["slow_query"] = 1 if duration >= @config.apm_slow_query_threshold_ms
|
|
163
275
|
signals["long_transaction"] = 1 if long_transaction?(payload, duration)
|
|
164
276
|
error_class = (payload["error_class"] || payload[:error_class]).to_s
|
|
165
|
-
|
|
166
|
-
|
|
277
|
+
QUERY_ERROR_SIGNALS.each do |name, pattern|
|
|
278
|
+
signals[name] = 1 if error_class =~ pattern
|
|
279
|
+
end
|
|
167
280
|
signals
|
|
168
281
|
end
|
|
169
282
|
|
|
170
283
|
def long_transaction?(payload, duration)
|
|
171
284
|
operation = (payload["operation"] || payload[:operation]).to_s
|
|
172
285
|
name = (payload["name"] || payload[:name]).to_s
|
|
173
|
-
transaction = operation == "BEGIN" || operation == "COMMIT" || name =~ /TRANSACTION/i
|
|
174
|
-
|
|
286
|
+
transaction = operation == "BEGIN" || operation == "COMMIT" || operation == "ROLLBACK" || name =~ /TRANSACTION/i
|
|
287
|
+
measured = non_negative(payload["transaction_duration_ms"] || payload[:transaction_duration_ms] || duration)
|
|
288
|
+
transaction && measured >= @config.apm_long_transaction_threshold_ms
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def n_plus_one_candidate?(payload)
|
|
292
|
+
operation = (payload["operation"] || payload[:operation]).to_s
|
|
293
|
+
cached = payload.key?("cached") ? payload["cached"] : payload[:cached]
|
|
294
|
+
operation == "SELECT" && cached != true
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def merge_integer_signals(left, right)
|
|
298
|
+
result = left.dup
|
|
299
|
+
right.each { |key, value| result[key.to_s] = result.fetch(key.to_s, 0) + value.to_i }
|
|
300
|
+
result
|
|
175
301
|
end
|
|
176
302
|
|
|
177
303
|
def dimensions_for(type, payload)
|
|
@@ -238,23 +364,47 @@ module Chronos
|
|
|
238
364
|
|
|
239
365
|
def drain_locked
|
|
240
366
|
metrics = @groups.values.map(&:to_h)
|
|
241
|
-
if metrics.empty?
|
|
242
|
-
@transactions = {}
|
|
243
|
-
return []
|
|
244
|
-
end
|
|
367
|
+
return [] if metrics.empty?
|
|
245
368
|
|
|
246
369
|
dropped = @dropped_groups
|
|
247
370
|
@groups = {}
|
|
248
|
-
@transactions = {}
|
|
249
371
|
@observations = 0
|
|
250
372
|
@dropped_groups = 0
|
|
373
|
+
tracking = {
|
|
374
|
+
"active_traces" => @transactions.length,
|
|
375
|
+
"dropped_trace_trackers" => @dropped_trace_trackers,
|
|
376
|
+
"expired_trace_trackers" => @expired_trace_trackers,
|
|
377
|
+
"dropped_query_fingerprints" => @dropped_query_fingerprints
|
|
378
|
+
}
|
|
379
|
+
@dropped_trace_trackers = 0
|
|
380
|
+
@expired_trace_trackers = 0
|
|
381
|
+
@dropped_query_fingerprints = 0
|
|
251
382
|
batches = []
|
|
252
383
|
metrics.each_slice(@config.apm_batch_size) do |slice|
|
|
384
|
+
slice.first["tracking"] = batches.empty? ? tracking : empty_tracking
|
|
253
385
|
batches << {"metrics" => slice, "dropped_groups" => batches.empty? ? dropped : 0}
|
|
254
386
|
end
|
|
255
387
|
batches
|
|
256
388
|
end
|
|
257
389
|
|
|
390
|
+
def expire_transactions
|
|
391
|
+
threshold = now - @config.apm_trace_ttl_seconds.to_f
|
|
392
|
+
expired = @transactions.keys.select { |key| @transactions[key]["last_seen_at"].to_f < threshold }
|
|
393
|
+
expired.each { |key| @transactions.delete(key) }
|
|
394
|
+
@expired_trace_trackers += expired.length
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
def empty_tracking
|
|
398
|
+
{"active_traces" => @transactions.length, "dropped_trace_trackers" => 0,
|
|
399
|
+
"expired_trace_trackers" => 0, "dropped_query_fingerprints" => 0}
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
def now
|
|
403
|
+
@clock.call.to_f
|
|
404
|
+
rescue StandardError
|
|
405
|
+
Time.now.to_f
|
|
406
|
+
end
|
|
407
|
+
|
|
258
408
|
def non_negative(value)
|
|
259
409
|
number = value.to_f
|
|
260
410
|
number < 0.0 ? 0.0 : number
|
|
@@ -9,7 +9,7 @@ module Chronos
|
|
|
9
9
|
# @thread_safety Reads one mutable configuration instance without shared state.
|
|
10
10
|
# @compatibility Ruby 2.2.10 through Ruby 2.6.
|
|
11
11
|
# @errors Invalid values become messages and never raise from validation.
|
|
12
|
-
module ApmConfigurationValidation
|
|
12
|
+
module ApmConfigurationValidation # rubocop:disable Metrics/ModuleLength
|
|
13
13
|
private
|
|
14
14
|
|
|
15
15
|
def apm_errors
|
|
@@ -32,6 +32,24 @@ module Chronos
|
|
|
32
32
|
unless positive_integer?(apm_max_queries_per_request)
|
|
33
33
|
errors << "apm_max_queries_per_request must be a positive integer"
|
|
34
34
|
end
|
|
35
|
+
errors.concat(query_capacity_errors)
|
|
36
|
+
errors
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def query_capacity_errors
|
|
40
|
+
errors = []
|
|
41
|
+
unless apm_query_analysis_max_queries.is_a?(Integer) &&
|
|
42
|
+
apm_query_analysis_max_queries >= 1 && apm_query_analysis_max_queries <= 500
|
|
43
|
+
errors << "apm_query_analysis_max_queries must be between 1 and 500"
|
|
44
|
+
end
|
|
45
|
+
unless apm_query_inspection_max_queries.is_a?(Integer) &&
|
|
46
|
+
apm_query_inspection_max_queries >= 1 && apm_query_inspection_max_queries <= 100
|
|
47
|
+
errors << "apm_query_inspection_max_queries must be between 1 and 100"
|
|
48
|
+
end
|
|
49
|
+
unless apm_transaction_max_connections.is_a?(Integer) &&
|
|
50
|
+
apm_transaction_max_connections >= 1 && apm_transaction_max_connections <= 500
|
|
51
|
+
errors << "apm_transaction_max_connections must be between 1 and 500"
|
|
52
|
+
end
|
|
35
53
|
errors
|
|
36
54
|
end
|
|
37
55
|
|
|
@@ -46,9 +64,41 @@ module Chronos
|
|
|
46
64
|
unless apm_n_plus_one_threshold.is_a?(Integer) && apm_n_plus_one_threshold >= 2
|
|
47
65
|
errors << "apm_n_plus_one_threshold must be an integer greater than or equal to 2"
|
|
48
66
|
end
|
|
67
|
+
errors.concat(query_threshold_errors)
|
|
49
68
|
errors
|
|
50
69
|
end
|
|
51
70
|
|
|
71
|
+
def query_threshold_errors
|
|
72
|
+
errors = []
|
|
73
|
+
unless positive_number?(apm_trace_ttl_seconds)
|
|
74
|
+
errors << "apm_trace_ttl_seconds must be greater than zero"
|
|
75
|
+
end
|
|
76
|
+
unless non_negative_number?(apm_query_inspection_min_duration_ms)
|
|
77
|
+
errors << "apm_query_inspection_min_duration_ms must be zero or greater"
|
|
78
|
+
end
|
|
79
|
+
query_analysis_boolean_attributes.each do |name, value|
|
|
80
|
+
errors << "#{name} must be true or false" unless boolean?(value)
|
|
81
|
+
end
|
|
82
|
+
if (apm_query_statistics_enabled || apm_query_plan_enabled) && !apm_query_inspection_enabled
|
|
83
|
+
errors << "query statistics and plans require apm_query_inspection_enabled"
|
|
84
|
+
end
|
|
85
|
+
errors
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def query_analysis_boolean_attributes
|
|
89
|
+
{
|
|
90
|
+
:apm_query_analysis_enabled => apm_query_analysis_enabled,
|
|
91
|
+
:apm_query_inspection_enabled => apm_query_inspection_enabled,
|
|
92
|
+
:apm_query_statistics_enabled => apm_query_statistics_enabled,
|
|
93
|
+
:apm_query_plan_enabled => apm_query_plan_enabled,
|
|
94
|
+
:apm_transaction_tracking_enabled => apm_transaction_tracking_enabled
|
|
95
|
+
}
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def non_negative_number?(value)
|
|
99
|
+
value.is_a?(Numeric) && value >= 0
|
|
100
|
+
end
|
|
101
|
+
|
|
52
102
|
def increasing_positive_numbers?(values)
|
|
53
103
|
return false unless values.is_a?(Array) && !values.empty? && values.length <= 19
|
|
54
104
|
return false unless values.all? { |value| positive_number?(value) }
|
|
@@ -60,6 +110,8 @@ module Chronos
|
|
|
60
110
|
errors = []
|
|
61
111
|
errors << "external_http_enabled must be true or false" unless boolean?(external_http_enabled)
|
|
62
112
|
errors << "external_http_trace_headers must be true or false" unless boolean?(external_http_trace_headers)
|
|
113
|
+
errors << "w3c_trace_context must be true or false" unless boolean?(w3c_trace_context)
|
|
114
|
+
errors << "opentelemetry_bridge must be true or false" unless boolean?(opentelemetry_bridge)
|
|
63
115
|
errors << "cache_key_mode must be :none or :sha256" unless [:none, :sha256].include?(cache_key_mode)
|
|
64
116
|
errors << "dependency_reporting must be true or false" unless boolean?(dependency_reporting)
|
|
65
117
|
unless dependency_max_items.is_a?(Integer) && dependency_max_items >= 1 && dependency_max_items <= 200
|
|
@@ -67,8 +67,8 @@ module Chronos
|
|
|
67
67
|
|
|
68
68
|
def context_errors
|
|
69
69
|
errors = []
|
|
70
|
-
unless
|
|
71
|
-
errors << "context_store must be :thread_local or implement get, set, clear, and with_context"
|
|
70
|
+
unless [:thread_local, :fiber_local].include?(context_store) || compatible_context_store?
|
|
71
|
+
errors << "context_store must be :thread_local, :fiber_local, or implement get, set, clear, and with_context"
|
|
72
72
|
end
|
|
73
73
|
errors << "breadcrumb_capacity must be a positive integer" unless positive_integer?(breadcrumb_capacity)
|
|
74
74
|
unless breadcrumb_max_bytes.is_a?(Integer) && breadcrumb_max_bytes >= 128
|
|
@@ -47,7 +47,14 @@ module Chronos
|
|
|
47
47
|
:apm_enabled, :apm_max_groups, :apm_flush_count, :apm_batch_size,
|
|
48
48
|
:apm_max_queries_per_request, :apm_slow_query_threshold_ms,
|
|
49
49
|
:apm_long_transaction_threshold_ms, :apm_n_plus_one_threshold,
|
|
50
|
-
:apm_histogram_buckets, :
|
|
50
|
+
:apm_histogram_buckets, :apm_trace_ttl_seconds,
|
|
51
|
+
:apm_query_analysis_enabled, :apm_query_inspection_enabled,
|
|
52
|
+
:apm_query_analysis_max_queries,
|
|
53
|
+
:apm_query_statistics_enabled, :apm_query_plan_enabled,
|
|
54
|
+
:apm_query_inspection_min_duration_ms, :apm_query_inspection_max_queries,
|
|
55
|
+
:apm_transaction_tracking_enabled, :apm_transaction_max_connections,
|
|
56
|
+
:external_http_enabled, :external_http_trace_headers,
|
|
57
|
+
:w3c_trace_context, :opentelemetry_bridge,
|
|
51
58
|
:cache_key_mode, :dependency_reporting, :dependency_max_items
|
|
52
59
|
].freeze
|
|
53
60
|
|
|
@@ -117,7 +124,7 @@ module Chronos
|
|
|
117
124
|
@user_agent = "chronos-ruby/#{Chronos::VERSION}"
|
|
118
125
|
@max_payload_size = 1_048_576
|
|
119
126
|
@gzip = false
|
|
120
|
-
@context_store = :
|
|
127
|
+
@context_store = :fiber_local
|
|
121
128
|
@breadcrumb_capacity = 20
|
|
122
129
|
@breadcrumb_max_bytes = 2048
|
|
123
130
|
end
|
|
@@ -146,11 +153,23 @@ module Chronos
|
|
|
146
153
|
@apm_long_transaction_threshold_ms = 1000.0
|
|
147
154
|
@apm_n_plus_one_threshold = 5
|
|
148
155
|
@apm_histogram_buckets = [5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0]
|
|
156
|
+
@apm_trace_ttl_seconds = 60.0
|
|
157
|
+
@apm_query_analysis_enabled = true
|
|
158
|
+
@apm_query_analysis_max_queries = 100
|
|
159
|
+
@apm_query_inspection_enabled = false
|
|
160
|
+
@apm_query_statistics_enabled = false
|
|
161
|
+
@apm_query_plan_enabled = false
|
|
162
|
+
@apm_query_inspection_min_duration_ms = 500.0
|
|
163
|
+
@apm_query_inspection_max_queries = 20
|
|
164
|
+
@apm_transaction_tracking_enabled = true
|
|
165
|
+
@apm_transaction_max_connections = 100
|
|
149
166
|
end
|
|
150
167
|
|
|
151
168
|
def initialize_observability_defaults
|
|
152
169
|
@external_http_enabled = false
|
|
153
170
|
@external_http_trace_headers = true
|
|
171
|
+
@w3c_trace_context = false
|
|
172
|
+
@opentelemetry_bridge = true
|
|
154
173
|
@cache_key_mode = :none
|
|
155
174
|
@dependency_reporting = true
|
|
156
175
|
@dependency_max_items = 100
|
|
@@ -2,18 +2,20 @@ module Chronos
|
|
|
2
2
|
module Core
|
|
3
3
|
# Accumulates one bounded APM metric group and serializes aggregate statistics.
|
|
4
4
|
#
|
|
5
|
-
# @responsibility Track counts, errors, durations, histogram, breakdown, and
|
|
5
|
+
# @responsibility Track counts, errors, durations, histogram, breakdown, signals, and diagnostics.
|
|
6
6
|
# @motivation Keep numerical accumulation separate from grouping and request correlation.
|
|
7
|
-
# @limits It does not choose dimensions, retain observations, or calculate percentiles.
|
|
7
|
+
# @limits It does not choose dimensions, retain observations, or calculate exact percentiles.
|
|
8
8
|
# @collaborators ApmAggregator and immutable histogram boundaries.
|
|
9
9
|
# @thread_safety Mutable by design; callers must synchronize access.
|
|
10
10
|
# @compatibility Ruby 2.2.10 through Ruby 2.6.
|
|
11
11
|
# @example
|
|
12
|
-
# metric.observe(12.0, false, {"database" => 3.0}, {})
|
|
12
|
+
# metric.observe(12.0, false, {"database" => 3.0}, {}, :diagnostics => [])
|
|
13
13
|
# @errors Non-numeric durations become zero and never escape.
|
|
14
14
|
# @performance Memory is fixed by the configured histogram boundary count.
|
|
15
15
|
class MetricAggregate
|
|
16
16
|
BREAKDOWN_CATEGORIES = %w(database view external_http cache queue application unknown).freeze
|
|
17
|
+
DIAGNOSTIC_SEVERITIES = %w(error warning info suggestion).freeze
|
|
18
|
+
MAX_DIAGNOSTICS = 20
|
|
17
19
|
|
|
18
20
|
def initialize(metric_type, dimensions, boundaries)
|
|
19
21
|
@metric_type = metric_type
|
|
@@ -28,9 +30,12 @@ module Chronos
|
|
|
28
30
|
@breakdown = {}
|
|
29
31
|
@signals = {}
|
|
30
32
|
@status_codes = {}
|
|
33
|
+
@diagnostics = {}
|
|
34
|
+
@severity_counts = {}
|
|
35
|
+
@query_analysis = nil
|
|
31
36
|
end
|
|
32
37
|
|
|
33
|
-
def observe(duration, error, breakdown, signals,
|
|
38
|
+
def observe(duration, error, breakdown, signals, options = {})
|
|
34
39
|
value = non_negative(duration)
|
|
35
40
|
@count += 1
|
|
36
41
|
@error_count += 1 if error
|
|
@@ -41,7 +46,9 @@ module Chronos
|
|
|
41
46
|
@buckets[bucket || @boundaries.length] += 1
|
|
42
47
|
add_breakdown(breakdown)
|
|
43
48
|
add_signals(signals)
|
|
44
|
-
add_status(status)
|
|
49
|
+
add_status(options[:status])
|
|
50
|
+
add_diagnostics(options[:diagnostics])
|
|
51
|
+
retain_query_analysis(options[:query_analysis])
|
|
45
52
|
self
|
|
46
53
|
end
|
|
47
54
|
|
|
@@ -50,9 +57,10 @@ module Chronos
|
|
|
50
57
|
"metric_type" => @metric_type, "dimensions" => @dimensions,
|
|
51
58
|
"count" => @count, "error_count" => @error_count,
|
|
52
59
|
"error_rate" => (@error_count.to_f / @count).round(6),
|
|
53
|
-
"duration_ms" => duration_summary, "histogram" => histogram,
|
|
60
|
+
"duration_ms" => duration_summary, "percentiles_ms" => percentiles, "histogram" => histogram,
|
|
54
61
|
"breakdown_ms" => rounded_hash(@breakdown), "signals" => @signals,
|
|
55
|
-
"status_codes" => @status_codes
|
|
62
|
+
"status_codes" => @status_codes, "severity_counts" => @severity_counts,
|
|
63
|
+
"diagnostics" => @diagnostics.values, "query_analysis" => @query_analysis || {}
|
|
56
64
|
}
|
|
57
65
|
end
|
|
58
66
|
|
|
@@ -81,6 +89,46 @@ module Chronos
|
|
|
81
89
|
@status_codes[key] += 1
|
|
82
90
|
end
|
|
83
91
|
|
|
92
|
+
def add_diagnostics(values)
|
|
93
|
+
Array(values).first(MAX_DIAGNOSTICS).each do |diagnostic|
|
|
94
|
+
data = hash(diagnostic)
|
|
95
|
+
severity = data["severity"].to_s
|
|
96
|
+
next unless DIAGNOSTIC_SEVERITIES.include?(severity)
|
|
97
|
+
|
|
98
|
+
@severity_counts[severity] ||= 0
|
|
99
|
+
@severity_counts[severity] += 1
|
|
100
|
+
key = diagnostic_key(data)
|
|
101
|
+
existing = @diagnostics[key]
|
|
102
|
+
if existing
|
|
103
|
+
existing["count"] += 1
|
|
104
|
+
elsif @diagnostics.length < MAX_DIAGNOSTICS
|
|
105
|
+
@diagnostics[key] = data.merge("count" => 1)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def diagnostic_key(data)
|
|
111
|
+
evidence = hash(data["evidence"])
|
|
112
|
+
[data["code"], data["severity"], evidence["table"], Array(evidence["columns"]).join(",")].join("|")
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def retain_query_analysis(value)
|
|
116
|
+
return unless value.is_a?(Hash) && !value.empty?
|
|
117
|
+
return if @query_analysis && analysis_score(@query_analysis) >= analysis_score(value)
|
|
118
|
+
|
|
119
|
+
@query_analysis = value.reject { |key, _child| key.to_s == "diagnostics" }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def analysis_score(value)
|
|
123
|
+
data = hash(value)
|
|
124
|
+
inspection = hash(data["inspection"] || data[:inspection])
|
|
125
|
+
score = inspection.empty? ? 0 : 1
|
|
126
|
+
score += 1 unless Array(inspection["indexes"] || inspection[:indexes]).empty?
|
|
127
|
+
score += 1 unless hash(inspection["statistics"] || inspection[:statistics]).empty?
|
|
128
|
+
score += 1 unless hash(inspection["plan"] || inspection[:plan]).empty?
|
|
129
|
+
score
|
|
130
|
+
end
|
|
131
|
+
|
|
84
132
|
def duration_summary
|
|
85
133
|
{
|
|
86
134
|
"total" => @total.round(3), "min" => @min.round(3), "max" => @max.round(3),
|
|
@@ -94,6 +142,20 @@ module Chronos
|
|
|
94
142
|
end
|
|
95
143
|
end
|
|
96
144
|
|
|
145
|
+
def percentiles
|
|
146
|
+
{"p50" => percentile(0.50), "p95" => percentile(0.95), "p99" => percentile(0.99)}
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def percentile(ratio)
|
|
150
|
+
target = (@count * ratio).ceil
|
|
151
|
+
accumulated = 0
|
|
152
|
+
@buckets.each_with_index do |count, index|
|
|
153
|
+
accumulated += count
|
|
154
|
+
return (@boundaries[index] || @max).to_f.round(3) if accumulated >= target
|
|
155
|
+
end
|
|
156
|
+
@max.to_f.round(3)
|
|
157
|
+
end
|
|
158
|
+
|
|
97
159
|
def rounded_hash(values)
|
|
98
160
|
values.each_with_object({}) { |(key, value), result| result[key] = value.round(3) }
|
|
99
161
|
end
|