brainzlab 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +52 -0
- data/LICENSE +26 -0
- data/README.md +311 -0
- data/lib/brainzlab/configuration.rb +215 -0
- data/lib/brainzlab/context.rb +91 -0
- data/lib/brainzlab/instrumentation/action_mailer.rb +181 -0
- data/lib/brainzlab/instrumentation/active_record.rb +111 -0
- data/lib/brainzlab/instrumentation/delayed_job.rb +236 -0
- data/lib/brainzlab/instrumentation/elasticsearch.rb +210 -0
- data/lib/brainzlab/instrumentation/faraday.rb +182 -0
- data/lib/brainzlab/instrumentation/grape.rb +293 -0
- data/lib/brainzlab/instrumentation/graphql.rb +251 -0
- data/lib/brainzlab/instrumentation/httparty.rb +194 -0
- data/lib/brainzlab/instrumentation/mongodb.rb +187 -0
- data/lib/brainzlab/instrumentation/net_http.rb +109 -0
- data/lib/brainzlab/instrumentation/redis.rb +331 -0
- data/lib/brainzlab/instrumentation/sidekiq.rb +264 -0
- data/lib/brainzlab/instrumentation.rb +132 -0
- data/lib/brainzlab/pulse/client.rb +132 -0
- data/lib/brainzlab/pulse/instrumentation.rb +364 -0
- data/lib/brainzlab/pulse/propagation.rb +241 -0
- data/lib/brainzlab/pulse/provisioner.rb +114 -0
- data/lib/brainzlab/pulse/tracer.rb +111 -0
- data/lib/brainzlab/pulse.rb +224 -0
- data/lib/brainzlab/rails/log_formatter.rb +801 -0
- data/lib/brainzlab/rails/log_subscriber.rb +341 -0
- data/lib/brainzlab/rails/railtie.rb +590 -0
- data/lib/brainzlab/recall/buffer.rb +64 -0
- data/lib/brainzlab/recall/client.rb +86 -0
- data/lib/brainzlab/recall/logger.rb +118 -0
- data/lib/brainzlab/recall/provisioner.rb +113 -0
- data/lib/brainzlab/recall.rb +155 -0
- data/lib/brainzlab/reflex/breadcrumbs.rb +55 -0
- data/lib/brainzlab/reflex/client.rb +85 -0
- data/lib/brainzlab/reflex/provisioner.rb +116 -0
- data/lib/brainzlab/reflex.rb +374 -0
- data/lib/brainzlab/version.rb +5 -0
- data/lib/brainzlab-sdk.rb +3 -0
- data/lib/brainzlab.rb +140 -0
- data/lib/generators/brainzlab/install/install_generator.rb +61 -0
- data/lib/generators/brainzlab/install/templates/brainzlab.rb.tt +77 -0
- metadata +159 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BrainzLab
|
|
4
|
+
module Instrumentation
|
|
5
|
+
module ActionMailerInstrumentation
|
|
6
|
+
@installed = false
|
|
7
|
+
|
|
8
|
+
class << self
|
|
9
|
+
def install!
|
|
10
|
+
return unless defined?(::ActionMailer::Base)
|
|
11
|
+
return if @installed
|
|
12
|
+
|
|
13
|
+
# Subscribe to deliver notification
|
|
14
|
+
ActiveSupport::Notifications.subscribe("deliver.action_mailer") do |*args|
|
|
15
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
|
16
|
+
record_delivery(event)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Subscribe to process notification (when mail is being prepared)
|
|
20
|
+
ActiveSupport::Notifications.subscribe("process.action_mailer") do |*args|
|
|
21
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
|
22
|
+
record_process(event)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
@installed = true
|
|
26
|
+
BrainzLab.debug_log("ActionMailer instrumentation installed")
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def installed?
|
|
30
|
+
@installed
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def reset!
|
|
34
|
+
@installed = false
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def record_delivery(event)
|
|
40
|
+
payload = event.payload
|
|
41
|
+
mailer = payload[:mailer]
|
|
42
|
+
message_id = payload[:message_id]
|
|
43
|
+
duration_ms = event.duration.round(2)
|
|
44
|
+
|
|
45
|
+
# Get mail details
|
|
46
|
+
mail = payload[:mail]
|
|
47
|
+
to = sanitize_recipients(mail&.to)
|
|
48
|
+
subject = mail&.subject
|
|
49
|
+
delivery_method = payload[:perform_deliveries] ? "delivered" : "skipped"
|
|
50
|
+
|
|
51
|
+
# Add breadcrumb for Reflex
|
|
52
|
+
if BrainzLab.configuration.reflex_enabled
|
|
53
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
54
|
+
"Mail #{delivery_method}: #{mailer}",
|
|
55
|
+
category: "mailer.deliver",
|
|
56
|
+
level: :info,
|
|
57
|
+
data: {
|
|
58
|
+
mailer: mailer,
|
|
59
|
+
to: to,
|
|
60
|
+
subject: truncate_subject(subject),
|
|
61
|
+
message_id: message_id,
|
|
62
|
+
duration_ms: duration_ms
|
|
63
|
+
}.compact
|
|
64
|
+
)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Record span for Pulse
|
|
68
|
+
record_span(
|
|
69
|
+
name: "Mail deliver #{mailer}",
|
|
70
|
+
kind: "mailer",
|
|
71
|
+
started_at: event.time,
|
|
72
|
+
ended_at: event.end,
|
|
73
|
+
duration_ms: duration_ms,
|
|
74
|
+
data: {
|
|
75
|
+
mailer: mailer,
|
|
76
|
+
action: "deliver",
|
|
77
|
+
to: to,
|
|
78
|
+
subject: truncate_subject(subject),
|
|
79
|
+
message_id: message_id,
|
|
80
|
+
delivery_method: delivery_method
|
|
81
|
+
}.compact
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
# Log to Recall
|
|
85
|
+
if BrainzLab.configuration.recall_enabled
|
|
86
|
+
BrainzLab::Recall.info(
|
|
87
|
+
"Mail #{delivery_method}: #{mailer} to #{to} (#{duration_ms}ms)",
|
|
88
|
+
mailer: mailer,
|
|
89
|
+
to: to,
|
|
90
|
+
subject: truncate_subject(subject),
|
|
91
|
+
message_id: message_id,
|
|
92
|
+
duration_ms: duration_ms
|
|
93
|
+
)
|
|
94
|
+
end
|
|
95
|
+
rescue StandardError => e
|
|
96
|
+
BrainzLab.debug_log("ActionMailer delivery recording failed: #{e.message}")
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def record_process(event)
|
|
100
|
+
payload = event.payload
|
|
101
|
+
mailer = payload[:mailer]
|
|
102
|
+
action = payload[:action]
|
|
103
|
+
duration_ms = event.duration.round(2)
|
|
104
|
+
|
|
105
|
+
# Add breadcrumb
|
|
106
|
+
if BrainzLab.configuration.reflex_enabled
|
|
107
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
108
|
+
"Mail process: #{mailer}##{action}",
|
|
109
|
+
category: "mailer.process",
|
|
110
|
+
level: :info,
|
|
111
|
+
data: {
|
|
112
|
+
mailer: mailer,
|
|
113
|
+
action: action,
|
|
114
|
+
duration_ms: duration_ms
|
|
115
|
+
}
|
|
116
|
+
)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Record span for Pulse
|
|
120
|
+
record_span(
|
|
121
|
+
name: "Mail process #{mailer}##{action}",
|
|
122
|
+
kind: "mailer",
|
|
123
|
+
started_at: event.time,
|
|
124
|
+
ended_at: event.end,
|
|
125
|
+
duration_ms: duration_ms,
|
|
126
|
+
data: {
|
|
127
|
+
mailer: mailer,
|
|
128
|
+
action: action
|
|
129
|
+
}
|
|
130
|
+
)
|
|
131
|
+
rescue StandardError => e
|
|
132
|
+
BrainzLab.debug_log("ActionMailer process recording failed: #{e.message}")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def record_span(name:, kind:, started_at:, ended_at:, duration_ms:, data:)
|
|
136
|
+
spans = Thread.current[:brainzlab_pulse_spans]
|
|
137
|
+
return unless spans
|
|
138
|
+
|
|
139
|
+
spans << {
|
|
140
|
+
span_id: SecureRandom.uuid,
|
|
141
|
+
name: name,
|
|
142
|
+
kind: kind,
|
|
143
|
+
started_at: started_at,
|
|
144
|
+
ended_at: ended_at,
|
|
145
|
+
duration_ms: duration_ms,
|
|
146
|
+
data: data
|
|
147
|
+
}
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def sanitize_recipients(recipients)
|
|
151
|
+
return nil unless recipients
|
|
152
|
+
|
|
153
|
+
case recipients
|
|
154
|
+
when Array
|
|
155
|
+
recipients.map { |r| mask_email(r) }.join(", ")
|
|
156
|
+
else
|
|
157
|
+
mask_email(recipients.to_s)
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def mask_email(email)
|
|
162
|
+
return email unless email.include?("@")
|
|
163
|
+
|
|
164
|
+
local, domain = email.split("@", 2)
|
|
165
|
+
if local.length > 2
|
|
166
|
+
"#{local[0..1]}***@#{domain}"
|
|
167
|
+
else
|
|
168
|
+
"***@#{domain}"
|
|
169
|
+
end
|
|
170
|
+
rescue StandardError
|
|
171
|
+
"[email]"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def truncate_subject(subject)
|
|
175
|
+
return nil unless subject
|
|
176
|
+
subject.to_s[0, 100]
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BrainzLab
|
|
4
|
+
module Instrumentation
|
|
5
|
+
class ActiveRecord
|
|
6
|
+
SCHEMA_QUERIES = ["SCHEMA", "EXPLAIN"].freeze
|
|
7
|
+
INTERNAL_TABLES = %w[pg_ information_schema sqlite_ mysql.].freeze
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def install!
|
|
11
|
+
return unless defined?(::ActiveRecord)
|
|
12
|
+
return if @installed
|
|
13
|
+
|
|
14
|
+
ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
|
|
15
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
|
16
|
+
next if skip_query?(event.payload)
|
|
17
|
+
|
|
18
|
+
record_breadcrumb(event)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
@installed = true
|
|
22
|
+
BrainzLab.debug_log("ActiveRecord breadcrumbs installed")
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def installed?
|
|
26
|
+
@installed == true
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def record_breadcrumb(event)
|
|
32
|
+
payload = event.payload
|
|
33
|
+
sql = payload[:sql]
|
|
34
|
+
name = payload[:name] || "SQL"
|
|
35
|
+
duration = event.duration.round(2)
|
|
36
|
+
|
|
37
|
+
# Extract operation type from SQL
|
|
38
|
+
operation = extract_operation(sql)
|
|
39
|
+
|
|
40
|
+
# Build breadcrumb message
|
|
41
|
+
message = if payload[:cached]
|
|
42
|
+
"#{name} (cached)"
|
|
43
|
+
else
|
|
44
|
+
"#{name} (#{duration}ms)"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Determine level based on duration
|
|
48
|
+
level = if duration > 100
|
|
49
|
+
:warning
|
|
50
|
+
elsif duration > 1000
|
|
51
|
+
:error
|
|
52
|
+
else
|
|
53
|
+
:info
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
57
|
+
message,
|
|
58
|
+
category: "db.#{operation}",
|
|
59
|
+
level: level,
|
|
60
|
+
data: {
|
|
61
|
+
sql: truncate_sql(sql),
|
|
62
|
+
duration_ms: duration,
|
|
63
|
+
cached: payload[:cached] || false,
|
|
64
|
+
connection_name: payload[:connection]&.pool&.connection_class&.name
|
|
65
|
+
}.compact
|
|
66
|
+
)
|
|
67
|
+
rescue StandardError => e
|
|
68
|
+
BrainzLab.debug_log("ActiveRecord breadcrumb failed: #{e.message}")
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def extract_operation(sql)
|
|
72
|
+
return "query" unless sql
|
|
73
|
+
|
|
74
|
+
case sql.to_s.strip.upcase
|
|
75
|
+
when /\ASELECT/i then "select"
|
|
76
|
+
when /\AINSERT/i then "insert"
|
|
77
|
+
when /\AUPDATE/i then "update"
|
|
78
|
+
when /\ADELETE/i then "delete"
|
|
79
|
+
when /\ABEGIN/i, /\ACOMMIT/i, /\AROLLBACK/i then "transaction"
|
|
80
|
+
else "query"
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def skip_query?(payload)
|
|
85
|
+
# Skip schema queries
|
|
86
|
+
return true if SCHEMA_QUERIES.include?(payload[:name])
|
|
87
|
+
|
|
88
|
+
# Skip internal/system table queries
|
|
89
|
+
sql = payload[:sql].to_s.downcase
|
|
90
|
+
return true if INTERNAL_TABLES.any? { |t| sql.include?(t) }
|
|
91
|
+
|
|
92
|
+
# Skip if no SQL (shouldn't happen but be safe)
|
|
93
|
+
return true if payload[:sql].nil? || payload[:sql].empty?
|
|
94
|
+
|
|
95
|
+
false
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def truncate_sql(sql)
|
|
99
|
+
return nil unless sql
|
|
100
|
+
|
|
101
|
+
truncated = sql.to_s.gsub(/\s+/, " ").strip
|
|
102
|
+
if truncated.length > 500
|
|
103
|
+
"#{truncated[0, 497]}..."
|
|
104
|
+
else
|
|
105
|
+
truncated
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BrainzLab
|
|
4
|
+
module Instrumentation
|
|
5
|
+
module DelayedJobInstrumentation
|
|
6
|
+
@installed = false
|
|
7
|
+
|
|
8
|
+
class << self
|
|
9
|
+
def install!
|
|
10
|
+
return unless defined?(::Delayed::Job) || defined?(::Delayed::Backend)
|
|
11
|
+
return if @installed
|
|
12
|
+
|
|
13
|
+
# Install lifecycle hooks
|
|
14
|
+
if defined?(::Delayed::Worker)
|
|
15
|
+
install_lifecycle_hooks!
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Install plugin if Delayed::Plugin is available
|
|
19
|
+
if defined?(::Delayed::Plugin)
|
|
20
|
+
::Delayed::Worker.plugins << Plugin
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
@installed = true
|
|
24
|
+
BrainzLab.debug_log("Delayed::Job instrumentation installed")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def installed?
|
|
28
|
+
@installed
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def reset!
|
|
32
|
+
@installed = false
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def install_lifecycle_hooks!
|
|
38
|
+
::Delayed::Worker.lifecycle.around(:invoke_job) do |job, *args, &block|
|
|
39
|
+
around_invoke(job, &block)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
::Delayed::Worker.lifecycle.after(:error) do |worker, job|
|
|
43
|
+
record_error(job)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
::Delayed::Worker.lifecycle.after(:failure) do |worker, job|
|
|
47
|
+
record_failure(job)
|
|
48
|
+
end
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
BrainzLab.debug_log("Delayed::Job lifecycle hooks failed: #{e.message}")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def around_invoke(job, &block)
|
|
54
|
+
started_at = Time.now.utc
|
|
55
|
+
job_name = extract_job_name(job)
|
|
56
|
+
queue = job.queue || "default"
|
|
57
|
+
|
|
58
|
+
# Calculate queue wait time
|
|
59
|
+
queue_wait_ms = job.created_at ? ((started_at - job.created_at) * 1000).round(2) : nil
|
|
60
|
+
|
|
61
|
+
# Set up context
|
|
62
|
+
setup_context(job, queue)
|
|
63
|
+
|
|
64
|
+
# Add breadcrumb
|
|
65
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
66
|
+
"DelayedJob #{job_name}",
|
|
67
|
+
category: "job.delayed_job",
|
|
68
|
+
level: :info,
|
|
69
|
+
data: { job_id: job.id, queue: queue, attempts: job.attempts }
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# Initialize Pulse tracing
|
|
73
|
+
Thread.current[:brainzlab_pulse_spans] = []
|
|
74
|
+
Thread.current[:brainzlab_pulse_breakdown] = nil
|
|
75
|
+
|
|
76
|
+
error_occurred = nil
|
|
77
|
+
begin
|
|
78
|
+
block.call(job)
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
error_occurred = e
|
|
81
|
+
raise
|
|
82
|
+
ensure
|
|
83
|
+
record_trace(
|
|
84
|
+
job: job,
|
|
85
|
+
job_name: job_name,
|
|
86
|
+
queue: queue,
|
|
87
|
+
started_at: started_at,
|
|
88
|
+
queue_wait_ms: queue_wait_ms,
|
|
89
|
+
error: error_occurred
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
cleanup_context
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def setup_context(job, queue)
|
|
97
|
+
BrainzLab::Context.current.set_context(
|
|
98
|
+
job_class: extract_job_name(job),
|
|
99
|
+
job_id: job.id,
|
|
100
|
+
queue_name: queue,
|
|
101
|
+
attempts: job.attempts
|
|
102
|
+
)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def cleanup_context
|
|
106
|
+
Thread.current[:brainzlab_pulse_spans] = nil
|
|
107
|
+
Thread.current[:brainzlab_pulse_breakdown] = nil
|
|
108
|
+
BrainzLab::Context.clear!
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def record_trace(job:, job_name:, queue:, started_at:, queue_wait_ms:, error:)
|
|
112
|
+
return unless BrainzLab.configuration.pulse_enabled
|
|
113
|
+
|
|
114
|
+
ended_at = Time.now.utc
|
|
115
|
+
duration_ms = ((ended_at - started_at) * 1000).round(2)
|
|
116
|
+
|
|
117
|
+
# Collect spans
|
|
118
|
+
spans = Thread.current[:brainzlab_pulse_spans] || []
|
|
119
|
+
breakdown = Thread.current[:brainzlab_pulse_breakdown] || {}
|
|
120
|
+
|
|
121
|
+
formatted_spans = spans.map do |span|
|
|
122
|
+
{
|
|
123
|
+
span_id: span[:span_id],
|
|
124
|
+
name: span[:name],
|
|
125
|
+
kind: span[:kind],
|
|
126
|
+
started_at: format_timestamp(span[:started_at]),
|
|
127
|
+
ended_at: format_timestamp(span[:ended_at]),
|
|
128
|
+
duration_ms: span[:duration_ms],
|
|
129
|
+
data: span[:data]
|
|
130
|
+
}.compact
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
payload = {
|
|
134
|
+
trace_id: SecureRandom.uuid,
|
|
135
|
+
name: job_name,
|
|
136
|
+
kind: "job",
|
|
137
|
+
started_at: started_at.utc.iso8601(3),
|
|
138
|
+
ended_at: ended_at.utc.iso8601(3),
|
|
139
|
+
duration_ms: duration_ms,
|
|
140
|
+
job_class: job_name,
|
|
141
|
+
job_id: job.id.to_s,
|
|
142
|
+
queue: queue,
|
|
143
|
+
queue_wait_ms: queue_wait_ms,
|
|
144
|
+
executions: (job.attempts || 0) + 1,
|
|
145
|
+
db_ms: breakdown[:db_ms],
|
|
146
|
+
error: error.present?,
|
|
147
|
+
error_class: error&.class&.name,
|
|
148
|
+
error_message: error&.message&.slice(0, 1000),
|
|
149
|
+
spans: formatted_spans,
|
|
150
|
+
environment: BrainzLab.configuration.environment,
|
|
151
|
+
commit: BrainzLab.configuration.commit,
|
|
152
|
+
host: BrainzLab.configuration.host
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
BrainzLab::Pulse.client.send_trace(payload.compact)
|
|
156
|
+
rescue StandardError => e
|
|
157
|
+
BrainzLab.debug_log("Delayed::Job trace recording failed: #{e.message}")
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def record_error(job)
|
|
161
|
+
return unless job.last_error
|
|
162
|
+
|
|
163
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
164
|
+
"DelayedJob error: #{extract_job_name(job)}",
|
|
165
|
+
category: "job.delayed_job.error",
|
|
166
|
+
level: :error,
|
|
167
|
+
data: {
|
|
168
|
+
job_id: job.id,
|
|
169
|
+
attempts: job.attempts,
|
|
170
|
+
error: job.last_error&.slice(0, 500)
|
|
171
|
+
}
|
|
172
|
+
)
|
|
173
|
+
rescue StandardError => e
|
|
174
|
+
BrainzLab.debug_log("Delayed::Job error recording failed: #{e.message}")
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def record_failure(job)
|
|
178
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
179
|
+
"DelayedJob failed permanently: #{extract_job_name(job)}",
|
|
180
|
+
category: "job.delayed_job.failure",
|
|
181
|
+
level: :error,
|
|
182
|
+
data: {
|
|
183
|
+
job_id: job.id,
|
|
184
|
+
attempts: job.attempts,
|
|
185
|
+
error: job.last_error&.slice(0, 500)
|
|
186
|
+
}
|
|
187
|
+
)
|
|
188
|
+
rescue StandardError => e
|
|
189
|
+
BrainzLab.debug_log("Delayed::Job failure recording failed: #{e.message}")
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def extract_job_name(job)
|
|
193
|
+
payload = job.payload_object
|
|
194
|
+
case payload
|
|
195
|
+
when ::Delayed::PerformableMethod
|
|
196
|
+
"#{payload.object.class}##{payload.method_name}"
|
|
197
|
+
when ::Delayed::PerformableMailer
|
|
198
|
+
"#{payload.object}##{payload.method_name}"
|
|
199
|
+
else
|
|
200
|
+
payload.class.name
|
|
201
|
+
end
|
|
202
|
+
rescue StandardError
|
|
203
|
+
job.name || "Unknown"
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def format_timestamp(ts)
|
|
207
|
+
return nil unless ts
|
|
208
|
+
|
|
209
|
+
case ts
|
|
210
|
+
when Time, DateTime then ts.utc.iso8601(3)
|
|
211
|
+
when Float, Integer then Time.at(ts).utc.iso8601(3)
|
|
212
|
+
when String then ts
|
|
213
|
+
else ts.to_s
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# Delayed::Job Plugin (alternative installation method)
|
|
219
|
+
class Plugin < ::Delayed::Plugin
|
|
220
|
+
callbacks do |lifecycle|
|
|
221
|
+
lifecycle.around(:invoke_job) do |job, *args, &block|
|
|
222
|
+
DelayedJobInstrumentation.send(:around_invoke, job, &block)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
lifecycle.after(:error) do |worker, job|
|
|
226
|
+
DelayedJobInstrumentation.send(:record_error, job)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
lifecycle.after(:failure) do |worker, job|
|
|
230
|
+
DelayedJobInstrumentation.send(:record_failure, job)
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end if defined?(::Delayed::Plugin)
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
end
|