dead_bro 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dead_bro/client.rb +57 -0
- data/lib/dead_bro/configuration.rb +82 -42
- data/lib/dead_bro/job_subscriber.rb +3 -3
- data/lib/dead_bro/lightweight_memory_tracker.rb +12 -6
- data/lib/dead_bro/memory_tracking_subscriber.rb +27 -15
- data/lib/dead_bro/redis_subscriber.rb +2 -2
- data/lib/dead_bro/sql_subscriber.rb +36 -20
- data/lib/dead_bro/subscriber.rb +13 -11
- data/lib/dead_bro/version.rb +1 -1
- data/lib/dead_bro.rb +11 -17
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7d8e1d14cc8e5511d143de228afefcb1a7557b8ce0308b7302ba40f4bc78bc90
|
|
4
|
+
data.tar.gz: 1d215f5ff69110be5c10e965bcf2ac2f08f1f6cb9a2be72b602f143c3e3ac5e5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9cab5ed48ea05f086512683b1d8d9fdb05030328f4fb4cb908884f074bc23b361ba7914774106bb5f1eba511435cdb506b123b2c4d2ece41634b539e925576a9
|
|
7
|
+
data.tar.gz: '0695e7e7bc6f5835fde69586aa2bc03c134ca9a2515a4bda4e55a0c78242f776315e07018834a2ce66c9a828849c05d807c38e183f99ab77e00ac4d239144943'
|
data/lib/dead_bro/client.rb
CHANGED
|
@@ -31,12 +31,23 @@ module DeadBro
|
|
|
31
31
|
end
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
# Truncate large arrays to avoid 413 Request Entity Too Large
|
|
35
|
+
payload = truncate_payload_for_request(payload)
|
|
36
|
+
|
|
34
37
|
# Make the HTTP request (async)
|
|
35
38
|
make_http_request(event_name, payload, @configuration.api_key)
|
|
36
39
|
|
|
37
40
|
nil
|
|
38
41
|
end
|
|
39
42
|
|
|
43
|
+
def post_heartbeat
|
|
44
|
+
return if @configuration.api_key.nil?
|
|
45
|
+
|
|
46
|
+
@configuration.last_heartbeat_attempt_at = Time.now.utc
|
|
47
|
+
make_http_request("heartbeat", {}, @configuration.api_key)
|
|
48
|
+
nil
|
|
49
|
+
end
|
|
50
|
+
|
|
40
51
|
def post_monitor_stats(payload)
|
|
41
52
|
return if @configuration.api_key.nil?
|
|
42
53
|
return unless @configuration.enabled
|
|
@@ -62,6 +73,42 @@ module DeadBro
|
|
|
62
73
|
|
|
63
74
|
private
|
|
64
75
|
|
|
76
|
+
def apply_settings_from_response(response)
|
|
77
|
+
return unless response.is_a?(Net::HTTPSuccess)
|
|
78
|
+
|
|
79
|
+
body = JSON.parse(response.body)
|
|
80
|
+
return unless body.is_a?(Hash) && body["settings"].is_a?(Hash)
|
|
81
|
+
|
|
82
|
+
@configuration.apply_remote_settings(body["settings"])
|
|
83
|
+
|
|
84
|
+
updated_at_str = body["settings_updated_at"]
|
|
85
|
+
@configuration.settings_received_at = updated_at_str ? Time.iso8601(updated_at_str) : Time.now.utc
|
|
86
|
+
rescue JSON::ParserError, ArgumentError
|
|
87
|
+
# Malformed response — ignore, settings stay as-is
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Limit payload size to avoid 413 from nginx/reverse proxies. Returns a new hash.
|
|
91
|
+
def truncate_payload_for_request(payload)
|
|
92
|
+
return payload unless payload.is_a?(Hash)
|
|
93
|
+
|
|
94
|
+
max_sql = @configuration.max_sql_queries_to_send
|
|
95
|
+
max_logs = @configuration.max_logs_to_send
|
|
96
|
+
|
|
97
|
+
out = payload.dup
|
|
98
|
+
|
|
99
|
+
if out.key?(:sql_queries) && out[:sql_queries].is_a?(Array) && out[:sql_queries].size > max_sql
|
|
100
|
+
out[:sql_queries_total_count] = out[:sql_queries].size
|
|
101
|
+
out[:sql_queries] = out[:sql_queries].first(max_sql)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
if out.key?(:logs) && out[:logs].is_a?(Array) && out[:logs].size > max_logs
|
|
105
|
+
out[:logs_total_count] = out[:logs].size
|
|
106
|
+
out[:logs] = out[:logs].first(max_logs)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
out
|
|
110
|
+
end
|
|
111
|
+
|
|
65
112
|
def create_circuit_breaker
|
|
66
113
|
return nil unless @configuration.circuit_breaker_enabled
|
|
67
114
|
|
|
@@ -85,6 +132,9 @@ module DeadBro
|
|
|
85
132
|
request = Net::HTTP::Post.new(uri.request_uri)
|
|
86
133
|
request["Content-Type"] = "application/json"
|
|
87
134
|
request["Authorization"] = "Bearer #{api_key}"
|
|
135
|
+
if @configuration.settings_received_at
|
|
136
|
+
request["X-Settings-Received-At"] = @configuration.settings_received_at.utc.iso8601
|
|
137
|
+
end
|
|
88
138
|
body = {event: event_name, payload: payload, sent_at: Time.now.utc.iso8601, revision: @configuration.resolve_deploy_id}
|
|
89
139
|
request.body = JSON.dump(body)
|
|
90
140
|
|
|
@@ -101,6 +151,13 @@ module DeadBro
|
|
|
101
151
|
@circuit_breaker.send(:on_failure)
|
|
102
152
|
end
|
|
103
153
|
end
|
|
154
|
+
|
|
155
|
+
# Apply remote settings if the backend included them in the response
|
|
156
|
+
apply_settings_from_response(response)
|
|
157
|
+
|
|
158
|
+
if response.is_a?(Net::HTTPSuccess) && event_name == "heartbeat"
|
|
159
|
+
@configuration.last_heartbeat_at = Time.now.utc
|
|
160
|
+
end
|
|
104
161
|
elsif @circuit_breaker && @configuration.circuit_breaker_enabled
|
|
105
162
|
# Treat nil response as failure for circuit breaker
|
|
106
163
|
@circuit_breaker.send(:on_failure)
|
|
@@ -2,40 +2,102 @@
|
|
|
2
2
|
|
|
3
3
|
module DeadBro
|
|
4
4
|
class Configuration
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
# Local-only settings (not overwritten by API `settings` payloads).
|
|
6
|
+
# Note: `enabled` may still be updated remotely via apply_remote_settings when the backend
|
|
7
|
+
# returns it in a response; local configure() values apply until the next remote update.
|
|
8
|
+
attr_accessor :api_key, :open_timeout, :read_timeout, :enabled, :ruby_dev,
|
|
9
|
+
:circuit_breaker_enabled, :circuit_breaker_failure_threshold, :circuit_breaker_recovery_timeout,
|
|
10
|
+
:circuit_breaker_retry_timeout, :deploy_id, :disk_paths, :interfaces_ignore
|
|
11
|
+
|
|
12
|
+
# Remote-managed settings (overwritten by backend JSON `settings` on successful API responses)
|
|
13
|
+
attr_accessor :memory_tracking_enabled, :allocation_tracking_enabled,
|
|
14
|
+
:sample_rate, :excluded_controllers, :excluded_jobs,
|
|
15
|
+
:exclusive_controllers, :exclusive_jobs, :slow_query_threshold_ms, :explain_analyze_enabled,
|
|
9
16
|
:job_queue_monitoring_enabled, :enable_db_stats, :enable_process_stats, :enable_system_stats,
|
|
10
|
-
:
|
|
17
|
+
:max_sql_queries_to_send, :max_logs_to_send
|
|
18
|
+
|
|
19
|
+
# Tracks when we last received settings from the backend (in-memory only)
|
|
20
|
+
attr_accessor :settings_received_at
|
|
21
|
+
|
|
22
|
+
# Last successful heartbeat HTTP response time while disabled (in-memory only)
|
|
23
|
+
attr_accessor :last_heartbeat_at
|
|
24
|
+
|
|
25
|
+
# Throttles heartbeat attempts to HEARTBEAT_INTERVAL (set when a heartbeat request is started)
|
|
26
|
+
attr_accessor :last_heartbeat_attempt_at
|
|
27
|
+
|
|
28
|
+
HEARTBEAT_INTERVAL = 60 # seconds
|
|
29
|
+
|
|
30
|
+
REMOTE_SETTING_KEYS = %w[
|
|
31
|
+
enabled sample_rate memory_tracking_enabled allocation_tracking_enabled
|
|
32
|
+
explain_analyze_enabled slow_query_threshold_ms max_sql_queries_to_send max_logs_to_send
|
|
33
|
+
excluded_controllers excluded_jobs exclusive_controllers exclusive_jobs
|
|
34
|
+
job_queue_monitoring_enabled enable_db_stats enable_process_stats enable_system_stats
|
|
35
|
+
].freeze
|
|
11
36
|
|
|
12
37
|
def initialize
|
|
13
38
|
@api_key = nil
|
|
14
|
-
@endpoint_url = nil
|
|
15
39
|
@open_timeout = 1.0
|
|
16
40
|
@read_timeout = 1.0
|
|
17
41
|
@enabled = true
|
|
18
42
|
@ruby_dev = false
|
|
19
|
-
@memory_tracking_enabled = true
|
|
20
|
-
@allocation_tracking_enabled = false # Disabled by default for performance
|
|
21
43
|
@circuit_breaker_enabled = true
|
|
22
44
|
@circuit_breaker_failure_threshold = 3
|
|
23
|
-
@circuit_breaker_recovery_timeout = 60
|
|
24
|
-
@circuit_breaker_retry_timeout = 300
|
|
45
|
+
@circuit_breaker_recovery_timeout = 60
|
|
46
|
+
@circuit_breaker_retry_timeout = 300
|
|
47
|
+
@deploy_id = resolve_deploy_id
|
|
48
|
+
@disk_paths = ["/"]
|
|
49
|
+
@interfaces_ignore = %w[lo lo0 docker0]
|
|
50
|
+
|
|
51
|
+
# Remote-managed defaults (used until backend sends real values)
|
|
25
52
|
@sample_rate = 100
|
|
53
|
+
@memory_tracking_enabled = true
|
|
54
|
+
@allocation_tracking_enabled = false
|
|
55
|
+
@explain_analyze_enabled = false
|
|
56
|
+
@slow_query_threshold_ms = 500
|
|
57
|
+
@max_sql_queries_to_send = 500
|
|
58
|
+
@max_logs_to_send = 100
|
|
26
59
|
@excluded_controllers = []
|
|
27
60
|
@excluded_jobs = []
|
|
28
61
|
@exclusive_controllers = []
|
|
29
62
|
@exclusive_jobs = []
|
|
30
|
-
@
|
|
31
|
-
@slow_query_threshold_ms = 500 # Default: 500ms
|
|
32
|
-
@explain_analyze_enabled = false # Enable EXPLAIN ANALYZE for slow queries by default
|
|
33
|
-
@job_queue_monitoring_enabled = false # Disabled by default
|
|
63
|
+
@job_queue_monitoring_enabled = false
|
|
34
64
|
@enable_db_stats = false
|
|
35
65
|
@enable_process_stats = false
|
|
36
66
|
@enable_system_stats = false
|
|
37
|
-
|
|
38
|
-
@
|
|
67
|
+
|
|
68
|
+
@settings_received_at = nil
|
|
69
|
+
@last_heartbeat_at = nil
|
|
70
|
+
@last_heartbeat_attempt_at = nil
|
|
71
|
+
@settings_mutex = Mutex.new
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Apply a settings hash received from the backend response.
|
|
75
|
+
# Only known keys are applied; unknown keys are silently ignored.
|
|
76
|
+
# Serialized so concurrent HTTP threads do not interleave writes with request-thread reads.
|
|
77
|
+
def apply_remote_settings(hash)
|
|
78
|
+
return unless hash.is_a?(Hash)
|
|
79
|
+
|
|
80
|
+
@settings_mutex.synchronize do
|
|
81
|
+
hash.each do |key, value|
|
|
82
|
+
k = key.to_s
|
|
83
|
+
next unless REMOTE_SETTING_KEYS.include?(k)
|
|
84
|
+
|
|
85
|
+
case k
|
|
86
|
+
when "sample_rate", "slow_query_threshold_ms", "max_sql_queries_to_send", "max_logs_to_send"
|
|
87
|
+
send(:"#{k}=", value.to_i)
|
|
88
|
+
when "enabled", "memory_tracking_enabled", "allocation_tracking_enabled", "explain_analyze_enabled",
|
|
89
|
+
"job_queue_monitoring_enabled", "enable_db_stats", "enable_process_stats", "enable_system_stats"
|
|
90
|
+
send(:"#{k}=", !!value)
|
|
91
|
+
when "excluded_controllers", "excluded_jobs", "exclusive_controllers", "exclusive_jobs"
|
|
92
|
+
send(:"#{k}=", Array(value).map(&:to_s))
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def heartbeat_due?
|
|
99
|
+
return false if api_key.nil?
|
|
100
|
+
last_heartbeat_attempt_at.nil? || (Time.now.utc - last_heartbeat_attempt_at) >= HEARTBEAT_INTERVAL
|
|
39
101
|
end
|
|
40
102
|
|
|
41
103
|
def resolve_deploy_id
|
|
@@ -86,6 +148,8 @@ module DeadBro
|
|
|
86
148
|
|
|
87
149
|
def should_sample?
|
|
88
150
|
sample_rate = resolve_sample_rate
|
|
151
|
+
sample_rate = 100 if sample_rate.nil?
|
|
152
|
+
|
|
89
153
|
return true if sample_rate >= 100
|
|
90
154
|
return false if sample_rate <= 0
|
|
91
155
|
|
|
@@ -93,22 +157,9 @@ module DeadBro
|
|
|
93
157
|
rand(1..100) <= sample_rate
|
|
94
158
|
end
|
|
95
159
|
|
|
160
|
+
# Returns the configured sample_rate only (no ENV fallback). Use DeadBro.configure or remote settings.
|
|
96
161
|
def resolve_sample_rate
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
if ENV["dead_bro_SAMPLE_RATE"]
|
|
100
|
-
env_value = ENV["dead_bro_SAMPLE_RATE"].to_s.strip
|
|
101
|
-
# Validate that it's a valid integer string
|
|
102
|
-
if env_value.match?(/^\d+$/)
|
|
103
|
-
parsed = env_value.to_i
|
|
104
|
-
# Ensure it's in valid range (0-100)
|
|
105
|
-
(parsed >= 0 && parsed <= 100) ? parsed : 100
|
|
106
|
-
else
|
|
107
|
-
100 # Invalid format, fall back to default
|
|
108
|
-
end
|
|
109
|
-
else
|
|
110
|
-
100 # default
|
|
111
|
-
end
|
|
162
|
+
@sample_rate
|
|
112
163
|
end
|
|
113
164
|
|
|
114
165
|
def resolve_api_key
|
|
@@ -117,17 +168,6 @@ module DeadBro
|
|
|
117
168
|
ENV["DEAD_BRO_API_KEY"]
|
|
118
169
|
end
|
|
119
170
|
|
|
120
|
-
def sample_rate=(value)
|
|
121
|
-
# Allow nil to use default/resolved value
|
|
122
|
-
return @sample_rate = nil if value.nil?
|
|
123
|
-
|
|
124
|
-
# Allow 0 to disable sampling, or 1-100 for percentage
|
|
125
|
-
unless value.is_a?(Integer) && value >= 0 && value <= 100
|
|
126
|
-
raise ArgumentError, "Sample rate must be an integer between 0 and 100, got: #{value.inspect}"
|
|
127
|
-
end
|
|
128
|
-
@sample_rate = value
|
|
129
|
-
end
|
|
130
|
-
|
|
131
171
|
private
|
|
132
172
|
|
|
133
173
|
def match_name_or_pattern?(name, pattern)
|
|
@@ -30,7 +30,7 @@ module DeadBro
|
|
|
30
30
|
|
|
31
31
|
# Ensure tracking was started (fallback if perform_start.active_job didn't fire)
|
|
32
32
|
# This handles job backends that don't emit perform_start events
|
|
33
|
-
unless
|
|
33
|
+
unless DeadBro::SqlSubscriber.tracking_active?
|
|
34
34
|
DeadBro.logger.clear
|
|
35
35
|
Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
|
|
36
36
|
DeadBro::SqlSubscriber.start_request_tracking
|
|
@@ -103,10 +103,10 @@ module DeadBro
|
|
|
103
103
|
|
|
104
104
|
duration_ms = ((finished - started) * 1000.0).round(2)
|
|
105
105
|
exception = data[:exception_object]
|
|
106
|
+
data[:job].class.name
|
|
106
107
|
|
|
107
108
|
# Ensure tracking was started (fallback if perform_start.active_job didn't fire)
|
|
108
|
-
|
|
109
|
-
unless Thread.current[DeadBro::SqlSubscriber::THREAD_LOCAL_KEY]
|
|
109
|
+
unless DeadBro::SqlSubscriber.tracking_active?
|
|
110
110
|
DeadBro.logger.clear
|
|
111
111
|
Thread.current[DeadBro::TRACKING_START_TIME_KEY] = Time.now
|
|
112
112
|
DeadBro::SqlSubscriber.start_request_tracking
|
|
@@ -8,19 +8,25 @@ module DeadBro
|
|
|
8
8
|
def self.start_request_tracking
|
|
9
9
|
return unless DeadBro.configuration.memory_tracking_enabled
|
|
10
10
|
|
|
11
|
-
#
|
|
12
|
-
|
|
11
|
+
# Stack allows nested job tracking (e.g. one job performing others in the same thread)
|
|
12
|
+
mem_before = lightweight_memory_usage
|
|
13
|
+
frame = {
|
|
13
14
|
gc_before: lightweight_gc_stats,
|
|
14
|
-
memory_before:
|
|
15
|
+
memory_before: mem_before,
|
|
15
16
|
start_time: Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
16
17
|
}
|
|
18
|
+
(Thread.current[THREAD_LOCAL_KEY] ||= []) << frame
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
def self.stop_request_tracking
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
stack = Thread.current[THREAD_LOCAL_KEY]
|
|
23
|
+
unless stack.is_a?(Array) && stack.any?
|
|
24
|
+
Thread.current[THREAD_LOCAL_KEY] = nil
|
|
25
|
+
return {}
|
|
26
|
+
end
|
|
22
27
|
|
|
23
|
-
|
|
28
|
+
events = stack.pop
|
|
29
|
+
Thread.current[THREAD_LOCAL_KEY] = nil if stack.empty?
|
|
24
30
|
|
|
25
31
|
# Calculate only essential metrics
|
|
26
32
|
gc_after = lightweight_gc_stats
|
|
@@ -54,11 +54,18 @@ module DeadBro
|
|
|
54
54
|
# Never raise from instrumentation install
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
+
# Current frame (top of stack) for nested job tracking; nil if none.
|
|
58
|
+
def self.current_events
|
|
59
|
+
stack = Thread.current[THREAD_LOCAL_KEY]
|
|
60
|
+
return nil unless stack.is_a?(Array) && stack.any?
|
|
61
|
+
stack.last
|
|
62
|
+
end
|
|
63
|
+
|
|
57
64
|
def self.start_request_tracking
|
|
58
65
|
# Only track if memory tracking is enabled
|
|
59
66
|
return unless DeadBro.configuration.memory_tracking_enabled
|
|
60
67
|
|
|
61
|
-
|
|
68
|
+
frame = {
|
|
62
69
|
allocations: [],
|
|
63
70
|
memory_snapshots: [],
|
|
64
71
|
large_objects: [],
|
|
@@ -68,11 +75,13 @@ module DeadBro
|
|
|
68
75
|
start_time: Time.now.to_f,
|
|
69
76
|
object_counts_before: count_objects_snapshot
|
|
70
77
|
}
|
|
78
|
+
(Thread.current[THREAD_LOCAL_KEY] ||= []) << frame
|
|
71
79
|
end
|
|
72
80
|
|
|
73
81
|
def self.stop_request_tracking
|
|
74
|
-
|
|
75
|
-
|
|
82
|
+
stack = Thread.current[THREAD_LOCAL_KEY]
|
|
83
|
+
events = (stack.is_a?(Array) && stack.any?) ? stack.pop : nil
|
|
84
|
+
Thread.current[THREAD_LOCAL_KEY] = nil if stack.nil? || stack.empty?
|
|
76
85
|
|
|
77
86
|
if events
|
|
78
87
|
events[:gc_after] = gc_stats
|
|
@@ -92,22 +101,24 @@ module DeadBro
|
|
|
92
101
|
|
|
93
102
|
# Record request-level allocation counters from Rails instrumentation.
|
|
94
103
|
def self.record_request_allocations(allocations:, allocated_bytes:)
|
|
95
|
-
|
|
104
|
+
events = current_events
|
|
105
|
+
return unless events
|
|
96
106
|
|
|
97
|
-
|
|
107
|
+
events[:request_allocations] = {
|
|
98
108
|
allocations: allocations,
|
|
99
109
|
allocated_bytes: allocated_bytes
|
|
100
110
|
}
|
|
101
111
|
end
|
|
102
112
|
|
|
103
113
|
def self.track_allocation(data, started, finished)
|
|
104
|
-
|
|
114
|
+
events = current_events
|
|
115
|
+
return unless events
|
|
105
116
|
|
|
106
117
|
# Only track if we have meaningful allocation data
|
|
107
118
|
return unless data.is_a?(Hash) && data[:count] && data[:size]
|
|
108
119
|
|
|
109
120
|
# Limit allocations per request to prevent memory bloat
|
|
110
|
-
allocations =
|
|
121
|
+
allocations = events[:allocations]
|
|
111
122
|
return if allocations.length >= MAX_ALLOCATIONS_PER_REQUEST
|
|
112
123
|
|
|
113
124
|
# Simplified allocation tracking (avoid expensive operations)
|
|
@@ -124,14 +135,15 @@ module DeadBro
|
|
|
124
135
|
large_object: true,
|
|
125
136
|
size_mb: (data[:size] / 1_000_000.0).round(2)
|
|
126
137
|
)
|
|
127
|
-
|
|
138
|
+
events[:large_objects] << large_object
|
|
128
139
|
end
|
|
129
140
|
|
|
130
|
-
|
|
141
|
+
events[:allocations] << allocation
|
|
131
142
|
end
|
|
132
143
|
|
|
133
144
|
def self.take_memory_snapshot(label = nil)
|
|
134
|
-
|
|
145
|
+
events = current_events
|
|
146
|
+
return unless events
|
|
135
147
|
|
|
136
148
|
snapshot = {
|
|
137
149
|
label: label || "snapshot_#{Time.now.to_i}",
|
|
@@ -142,7 +154,7 @@ module DeadBro
|
|
|
142
154
|
heap_pages: heap_pages
|
|
143
155
|
}
|
|
144
156
|
|
|
145
|
-
|
|
157
|
+
events[:memory_snapshots] << snapshot
|
|
146
158
|
end
|
|
147
159
|
|
|
148
160
|
def self.analyze_memory_performance(memory_events)
|
|
@@ -179,10 +191,10 @@ module DeadBro
|
|
|
179
191
|
# Group allocations by class
|
|
180
192
|
allocations_by_class = allocations.group_by { |a| a[:class_name] }
|
|
181
193
|
.transform_values { |allocs|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
194
|
+
{
|
|
195
|
+
count: allocs.sum { |a| a[:count] },
|
|
196
|
+
size: allocs.sum { |a| a[:size] }
|
|
197
|
+
}
|
|
186
198
|
}
|
|
187
199
|
|
|
188
200
|
# Find top allocating classes
|
|
@@ -25,8 +25,8 @@ module DeadBro
|
|
|
25
25
|
def self.install_redis_client!
|
|
26
26
|
# Only instrument if Redis::Client actually has the call method
|
|
27
27
|
# Check both public and private methods
|
|
28
|
-
has_call = ::Redis::Client.
|
|
29
|
-
::Redis::Client.
|
|
28
|
+
has_call = ::Redis::Client.method_defined?(:call, false) ||
|
|
29
|
+
::Redis::Client.private_method_defined?(:call, false)
|
|
30
30
|
return unless has_call
|
|
31
31
|
|
|
32
32
|
mod = Module.new do
|
|
@@ -16,13 +16,25 @@ module DeadBro
|
|
|
16
16
|
THREAD_LOCAL_EXPLAIN_PENDING_KEY = :dead_bro_explain_pending
|
|
17
17
|
MAX_TRACKED_QUERIES = 1000
|
|
18
18
|
|
|
19
|
+
# True when there is at least one active tracking context (e.g. for nested jobs).
|
|
20
|
+
def self.tracking_active?
|
|
21
|
+
stack = Thread.current[THREAD_LOCAL_KEY]
|
|
22
|
+
stack.is_a?(Array) && stack.any?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Current queries array (top of stack); nil if no active tracking.
|
|
26
|
+
def self.current_queries_array
|
|
27
|
+
stack = Thread.current[THREAD_LOCAL_KEY]
|
|
28
|
+
return nil unless stack.is_a?(Array) && stack.any?
|
|
29
|
+
stack.last
|
|
30
|
+
end
|
|
31
|
+
|
|
19
32
|
# Check if we should continue tracking based on count and time limits
|
|
20
|
-
def self.should_continue_tracking?(
|
|
21
|
-
|
|
22
|
-
return false unless events
|
|
33
|
+
def self.should_continue_tracking?(current_queries_array, max_count)
|
|
34
|
+
return false unless current_queries_array.is_a?(Array)
|
|
23
35
|
|
|
24
36
|
# Check count limit
|
|
25
|
-
return false if
|
|
37
|
+
return false if current_queries_array.length >= max_count
|
|
26
38
|
|
|
27
39
|
# Check time limit
|
|
28
40
|
start_time = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
|
|
@@ -44,9 +56,10 @@ module DeadBro
|
|
|
44
56
|
end
|
|
45
57
|
|
|
46
58
|
ActiveSupport::Notifications.subscribe(SQL_EVENT_NAME) do |name, started, finished, _unique_id, data|
|
|
47
|
-
next if data[:name] == "SCHEMA"
|
|
48
|
-
# Only track queries that are part of the current request
|
|
49
|
-
|
|
59
|
+
next if data[:name] == "SCHEMA" || data[:name] == "CACHE" || data[:name] == "BEGIN" || data[:name] == "COMMIT" || data[:name] == "ROLLBACK" || data[:name] == "SAVEPOINT" || data[:name] == "RELEASE"
|
|
60
|
+
# Only track queries that are part of the current request (top of stack for nested jobs)
|
|
61
|
+
current = current_queries_array
|
|
62
|
+
next unless current
|
|
50
63
|
unique_id = _unique_id
|
|
51
64
|
allocations = nil
|
|
52
65
|
captured_backtrace = nil
|
|
@@ -82,15 +95,16 @@ module DeadBro
|
|
|
82
95
|
start_explain_analyze_background(original_sql, data[:connection_id], query_info, binds)
|
|
83
96
|
end
|
|
84
97
|
|
|
85
|
-
# Add to
|
|
86
|
-
if should_continue_tracking?(
|
|
87
|
-
|
|
98
|
+
# Add to current context (top of stack), but only if we haven't exceeded the limits
|
|
99
|
+
if should_continue_tracking?(current, MAX_TRACKED_QUERIES)
|
|
100
|
+
current << query_info
|
|
88
101
|
end
|
|
89
102
|
end
|
|
90
103
|
end
|
|
91
104
|
|
|
92
105
|
def self.start_request_tracking
|
|
93
|
-
|
|
106
|
+
# Stack allows nested job tracking (e.g. one job performing others in the same thread)
|
|
107
|
+
(Thread.current[THREAD_LOCAL_KEY] ||= []) << []
|
|
94
108
|
Thread.current[THREAD_LOCAL_ALLOC_START_KEY] = {}
|
|
95
109
|
Thread.current[THREAD_LOCAL_ALLOC_RESULTS_KEY] = {}
|
|
96
110
|
Thread.current[THREAD_LOCAL_BACKTRACE_KEY] = {}
|
|
@@ -103,15 +117,17 @@ module DeadBro
|
|
|
103
117
|
# all explain_plan fields are populated
|
|
104
118
|
wait_for_pending_explains(5.0) # 5 second timeout
|
|
105
119
|
|
|
106
|
-
|
|
107
|
-
queries =
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
120
|
+
stack = Thread.current[THREAD_LOCAL_KEY]
|
|
121
|
+
queries = (stack.is_a?(Array) && stack.any?) ? stack.pop : []
|
|
122
|
+
# Clear thread locals when stack is empty so "tracking not started" behaves correctly
|
|
123
|
+
if stack.nil? || stack.empty?
|
|
124
|
+
Thread.current[THREAD_LOCAL_KEY] = nil
|
|
125
|
+
Thread.current[THREAD_LOCAL_ALLOC_START_KEY] = nil
|
|
126
|
+
Thread.current[THREAD_LOCAL_ALLOC_RESULTS_KEY] = nil
|
|
127
|
+
Thread.current[THREAD_LOCAL_BACKTRACE_KEY] = nil
|
|
128
|
+
Thread.current[THREAD_LOCAL_EXPLAIN_PENDING_KEY] = nil
|
|
129
|
+
end
|
|
130
|
+
queries
|
|
115
131
|
end
|
|
116
132
|
|
|
117
133
|
def self.wait_for_pending_explains(timeout_seconds)
|
data/lib/dead_bro/subscriber.rb
CHANGED
|
@@ -8,21 +8,23 @@ module DeadBro
|
|
|
8
8
|
|
|
9
9
|
def self.subscribe!(client: Client.new)
|
|
10
10
|
ActiveSupport::Notifications.subscribe(EVENT_NAME) do |name, started, finished, _unique_id, data|
|
|
11
|
+
# When disabled remotely, fire a heartbeat at most once per minute so the gem
|
|
12
|
+
# can detect when tracking has been re-enabled, then skip all tracking.
|
|
13
|
+
unless DeadBro.configuration.enabled
|
|
14
|
+
client.post_heartbeat if DeadBro.configuration.heartbeat_due?
|
|
15
|
+
next
|
|
16
|
+
end
|
|
17
|
+
|
|
11
18
|
# Skip excluded controllers or controller#action pairs
|
|
12
19
|
# Also check exclusive_controller_actions - if defined, only track those
|
|
20
|
+
notification = data.is_a?(Hash) ? data : {}
|
|
21
|
+
controller_name = notification[:controller].to_s
|
|
22
|
+
action_name = notification[:action].to_s
|
|
13
23
|
begin
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
if DeadBro.configuration.excluded_controller?(controller_name, action_name)
|
|
17
|
-
puts "excluded controller"
|
|
18
|
-
next
|
|
19
|
-
end
|
|
20
|
-
# If exclusive_controller_actions is defined and not empty, only track matching actions
|
|
21
|
-
unless DeadBro.configuration.exclusive_controller?(controller_name, action_name)
|
|
22
|
-
puts "exclusive controller"
|
|
23
|
-
next
|
|
24
|
-
end
|
|
24
|
+
next if DeadBro.configuration.excluded_controller?(controller_name, action_name)
|
|
25
|
+
next unless DeadBro.configuration.exclusive_controller?(controller_name, action_name)
|
|
25
26
|
rescue
|
|
27
|
+
next
|
|
26
28
|
end
|
|
27
29
|
|
|
28
30
|
duration_ms = ((finished - started) * 1000.0).round(2)
|
data/lib/dead_bro/version.rb
CHANGED
data/lib/dead_bro.rb
CHANGED
|
@@ -120,7 +120,7 @@ module DeadBro
|
|
|
120
120
|
|
|
121
121
|
begin
|
|
122
122
|
if defined?(DeadBro::MemoryTrackingSubscriber) &&
|
|
123
|
-
|
|
123
|
+
!Thread.current[DeadBro::MemoryTrackingSubscriber::THREAD_LOCAL_KEY]
|
|
124
124
|
DeadBro::MemoryTrackingSubscriber.start_request_tracking
|
|
125
125
|
memory_tracking_started = true
|
|
126
126
|
end
|
|
@@ -128,10 +128,10 @@ module DeadBro
|
|
|
128
128
|
end
|
|
129
129
|
|
|
130
130
|
begin
|
|
131
|
-
if defined?(DeadBro::MemoryTrackingSubscriber)
|
|
132
|
-
|
|
131
|
+
memory_before_mb = if defined?(DeadBro::MemoryTrackingSubscriber)
|
|
132
|
+
DeadBro::MemoryTrackingSubscriber.memory_usage_mb
|
|
133
133
|
else
|
|
134
|
-
|
|
134
|
+
0.0
|
|
135
135
|
end
|
|
136
136
|
rescue
|
|
137
137
|
memory_before_mb = 0.0
|
|
@@ -148,7 +148,6 @@ module DeadBro
|
|
|
148
148
|
begin
|
|
149
149
|
if defined?(ActiveSupport) && defined?(ActiveSupport::Notifications)
|
|
150
150
|
# Ensure SqlSubscriber is loaded so SQL_EVENT_NAME is defined
|
|
151
|
-
DeadBro::SqlSubscriber
|
|
152
151
|
event_name = DeadBro::SqlSubscriber::SQL_EVENT_NAME
|
|
153
152
|
|
|
154
153
|
sql_notification_subscription =
|
|
@@ -197,11 +196,9 @@ module DeadBro
|
|
|
197
196
|
block_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
198
197
|
|
|
199
198
|
error = nil
|
|
200
|
-
result = nil
|
|
201
|
-
analysis_result = nil
|
|
202
199
|
|
|
203
200
|
begin
|
|
204
|
-
|
|
201
|
+
yield
|
|
205
202
|
rescue => e
|
|
206
203
|
error = e
|
|
207
204
|
ensure
|
|
@@ -225,7 +222,7 @@ module DeadBro
|
|
|
225
222
|
sql_time_ms = local_sql_queries.sum { |q| (q[:duration_ms] || 0.0).to_f }.round(2)
|
|
226
223
|
|
|
227
224
|
# Group SQL queries by normalized pattern to show frequency and cost
|
|
228
|
-
query_signatures = Hash.new { |h, k| h[k] = {
|
|
225
|
+
query_signatures = Hash.new { |h, k| h[k] = {count: 0, total_time_ms: 0.0, type: nil} }
|
|
229
226
|
local_sql_queries.each do |q|
|
|
230
227
|
sig = (q[:sql] || "UNKNOWN").to_s
|
|
231
228
|
entry = query_signatures[sig]
|
|
@@ -237,7 +234,6 @@ module DeadBro
|
|
|
237
234
|
top_query_signatures = query_signatures.sort_by { |_, data| -data[:count] }.first(3)
|
|
238
235
|
|
|
239
236
|
memory_after_mb = memory_before_mb
|
|
240
|
-
memory_delta_mb = 0.0
|
|
241
237
|
detailed_memory_summary = nil
|
|
242
238
|
|
|
243
239
|
raw_events = {}
|
|
@@ -255,14 +251,12 @@ module DeadBro
|
|
|
255
251
|
memory_before_mb = raw_events[:memory_before]
|
|
256
252
|
end
|
|
257
253
|
|
|
258
|
-
if raw_events[:memory_after]
|
|
259
|
-
|
|
254
|
+
memory_after_mb = if raw_events[:memory_after]
|
|
255
|
+
raw_events[:memory_after]
|
|
256
|
+
elsif defined?(DeadBro::MemoryTrackingSubscriber)
|
|
257
|
+
DeadBro::MemoryTrackingSubscriber.memory_usage_mb
|
|
260
258
|
else
|
|
261
|
-
|
|
262
|
-
memory_after_mb = DeadBro::MemoryTrackingSubscriber.memory_usage_mb
|
|
263
|
-
else
|
|
264
|
-
memory_after_mb = memory_before_mb
|
|
265
|
-
end
|
|
259
|
+
memory_before_mb
|
|
266
260
|
end
|
|
267
261
|
rescue
|
|
268
262
|
memory_after_mb = memory_before_mb
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: dead_bro
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Emanuel Comsa
|
|
@@ -69,7 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
69
69
|
- !ruby/object:Gem::Version
|
|
70
70
|
version: '0'
|
|
71
71
|
requirements: []
|
|
72
|
-
rubygems_version: 4.0.
|
|
72
|
+
rubygems_version: 4.0.9
|
|
73
73
|
specification_version: 4
|
|
74
74
|
summary: Minimal APM for Rails apps.
|
|
75
75
|
test_files: []
|