fluyenta-ruby 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +68 -0
- data/LICENSE +11 -0
- data/README.md +571 -0
- data/lib/brainzlab/beacon/client.rb +227 -0
- data/lib/brainzlab/beacon/provisioner.rb +44 -0
- data/lib/brainzlab/beacon.rb +215 -0
- data/lib/brainzlab/configuration.rb +676 -0
- data/lib/brainzlab/context.rb +90 -0
- data/lib/brainzlab/cortex/cache.rb +59 -0
- data/lib/brainzlab/cortex/client.rb +159 -0
- data/lib/brainzlab/cortex/provisioner.rb +49 -0
- data/lib/brainzlab/cortex.rb +223 -0
- data/lib/brainzlab/debug.rb +305 -0
- data/lib/brainzlab/dendrite/client.rb +250 -0
- data/lib/brainzlab/dendrite/provisioner.rb +44 -0
- data/lib/brainzlab/dendrite.rb +195 -0
- data/lib/brainzlab/development/logger.rb +150 -0
- data/lib/brainzlab/development/store.rb +121 -0
- data/lib/brainzlab/development.rb +72 -0
- data/lib/brainzlab/devtools/assets/devtools.css +1329 -0
- data/lib/brainzlab/devtools/assets/devtools.js +396 -0
- data/lib/brainzlab/devtools/assets/logo.svg +6 -0
- data/lib/brainzlab/devtools/assets/templates/debug_panel.html.erb +511 -0
- data/lib/brainzlab/devtools/assets/templates/error_page.html.erb +1086 -0
- data/lib/brainzlab/devtools/data/collector.rb +248 -0
- data/lib/brainzlab/devtools/middleware/asset_server.rb +63 -0
- data/lib/brainzlab/devtools/middleware/database_handler.rb +177 -0
- data/lib/brainzlab/devtools/middleware/debug_panel.rb +126 -0
- data/lib/brainzlab/devtools/middleware/error_page.rb +377 -0
- data/lib/brainzlab/devtools/renderers/debug_panel_renderer.rb +159 -0
- data/lib/brainzlab/devtools/renderers/error_page_renderer.rb +98 -0
- data/lib/brainzlab/devtools.rb +75 -0
- data/lib/brainzlab/errors.rb +490 -0
- data/lib/brainzlab/flux/buffer.rb +96 -0
- data/lib/brainzlab/flux/client.rb +68 -0
- data/lib/brainzlab/flux/provisioner.rb +124 -0
- data/lib/brainzlab/flux.rb +184 -0
- data/lib/brainzlab/instrumentation/action_cable.rb +351 -0
- data/lib/brainzlab/instrumentation/action_controller.rb +649 -0
- data/lib/brainzlab/instrumentation/action_dispatch.rb +259 -0
- data/lib/brainzlab/instrumentation/action_mailbox.rb +197 -0
- data/lib/brainzlab/instrumentation/action_mailer.rb +182 -0
- data/lib/brainzlab/instrumentation/action_view.rb +380 -0
- data/lib/brainzlab/instrumentation/active_job.rb +569 -0
- data/lib/brainzlab/instrumentation/active_record.rb +559 -0
- data/lib/brainzlab/instrumentation/active_storage.rb +541 -0
- data/lib/brainzlab/instrumentation/active_support_cache.rb +730 -0
- data/lib/brainzlab/instrumentation/aws.rb +183 -0
- data/lib/brainzlab/instrumentation/dalli.rb +108 -0
- data/lib/brainzlab/instrumentation/delayed_job.rb +234 -0
- data/lib/brainzlab/instrumentation/elasticsearch.rb +209 -0
- data/lib/brainzlab/instrumentation/excon.rb +152 -0
- data/lib/brainzlab/instrumentation/faraday.rb +181 -0
- data/lib/brainzlab/instrumentation/good_job.rb +102 -0
- data/lib/brainzlab/instrumentation/grape.rb +293 -0
- data/lib/brainzlab/instrumentation/graphql.rb +252 -0
- data/lib/brainzlab/instrumentation/httparty.rb +193 -0
- data/lib/brainzlab/instrumentation/mongodb.rb +187 -0
- data/lib/brainzlab/instrumentation/net_http.rb +114 -0
- data/lib/brainzlab/instrumentation/rails_deprecation.rb +139 -0
- data/lib/brainzlab/instrumentation/railties.rb +134 -0
- data/lib/brainzlab/instrumentation/redis.rb +324 -0
- data/lib/brainzlab/instrumentation/resque.rb +114 -0
- data/lib/brainzlab/instrumentation/sidekiq.rb +265 -0
- data/lib/brainzlab/instrumentation/solid_queue.rb +194 -0
- data/lib/brainzlab/instrumentation/stripe.rb +163 -0
- data/lib/brainzlab/instrumentation/typhoeus.rb +106 -0
- data/lib/brainzlab/instrumentation.rb +360 -0
- data/lib/brainzlab/nerve/client.rb +235 -0
- data/lib/brainzlab/nerve/provisioner.rb +44 -0
- data/lib/brainzlab/nerve.rb +219 -0
- data/lib/brainzlab/pulse/client.rb +203 -0
- data/lib/brainzlab/pulse/instrumentation.rb +401 -0
- data/lib/brainzlab/pulse/propagation.rb +241 -0
- data/lib/brainzlab/pulse/provisioner.rb +114 -0
- data/lib/brainzlab/pulse/tracer.rb +111 -0
- data/lib/brainzlab/pulse.rb +294 -0
- data/lib/brainzlab/rails/log_formatter.rb +807 -0
- data/lib/brainzlab/rails/log_subscriber.rb +334 -0
- data/lib/brainzlab/rails/railtie.rb +606 -0
- data/lib/brainzlab/recall/buffer.rb +66 -0
- data/lib/brainzlab/recall/client.rb +158 -0
- data/lib/brainzlab/recall/logger.rb +116 -0
- data/lib/brainzlab/recall/provisioner.rb +130 -0
- data/lib/brainzlab/recall.rb +175 -0
- data/lib/brainzlab/reflex/breadcrumbs.rb +55 -0
- data/lib/brainzlab/reflex/client.rb +150 -0
- data/lib/brainzlab/reflex/provisioner.rb +116 -0
- data/lib/brainzlab/reflex.rb +421 -0
- data/lib/brainzlab/sentinel/client.rb +236 -0
- data/lib/brainzlab/sentinel/provisioner.rb +44 -0
- data/lib/brainzlab/sentinel.rb +165 -0
- data/lib/brainzlab/signal/client.rb +60 -0
- data/lib/brainzlab/signal/provisioner.rb +115 -0
- data/lib/brainzlab/signal.rb +136 -0
- data/lib/brainzlab/synapse/client.rb +308 -0
- data/lib/brainzlab/synapse/provisioner.rb +44 -0
- data/lib/brainzlab/synapse.rb +270 -0
- data/lib/brainzlab/testing/event_store.rb +377 -0
- data/lib/brainzlab/testing/helpers.rb +650 -0
- data/lib/brainzlab/testing/matchers.rb +391 -0
- data/lib/brainzlab/testing.rb +327 -0
- data/lib/brainzlab/utilities/circuit_breaker.rb +290 -0
- data/lib/brainzlab/utilities/health_check.rb +294 -0
- data/lib/brainzlab/utilities/log_formatter.rb +254 -0
- data/lib/brainzlab/utilities/rate_limiter.rb +230 -0
- data/lib/brainzlab/utilities.rb +17 -0
- data/lib/brainzlab/vault/cache.rb +80 -0
- data/lib/brainzlab/vault/client.rb +216 -0
- data/lib/brainzlab/vault/provisioner.rb +49 -0
- data/lib/brainzlab/vault.rb +262 -0
- data/lib/brainzlab/version.rb +5 -0
- data/lib/brainzlab/vision/client.rb +175 -0
- data/lib/brainzlab/vision/provisioner.rb +136 -0
- data/lib/brainzlab/vision.rb +155 -0
- data/lib/brainzlab-sdk.rb +3 -0
- data/lib/brainzlab.rb +306 -0
- data/lib/generators/brainzlab/install/install_generator.rb +63 -0
- data/lib/generators/brainzlab/install/templates/brainzlab.rb.tt +77 -0
- metadata +251 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BrainzLab
|
|
4
|
+
module Instrumentation
|
|
5
|
+
module AWSInstrumentation
|
|
6
|
+
class << self
|
|
7
|
+
def install!
|
|
8
|
+
return unless defined?(::Aws)
|
|
9
|
+
|
|
10
|
+
install_plugin!
|
|
11
|
+
|
|
12
|
+
BrainzLab.debug_log('[Instrumentation] AWS SDK instrumentation installed')
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def install_plugin!
|
|
18
|
+
# AWS SDK v3 uses plugins
|
|
19
|
+
if defined?(::Aws::Plugins)
|
|
20
|
+
::Aws.config[:plugins] ||= []
|
|
21
|
+
::Aws.config[:plugins] << BrainzLabPlugin unless ::Aws.config[:plugins].include?(BrainzLabPlugin)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Also hook into Seahorse for lower-level tracking
|
|
25
|
+
return unless defined?(::Seahorse::Client::Base)
|
|
26
|
+
|
|
27
|
+
install_seahorse_handler!
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def install_seahorse_handler!
|
|
31
|
+
handler_class = Class.new(::Seahorse::Client::Handler) do
|
|
32
|
+
def call(context)
|
|
33
|
+
started_at = Time.now
|
|
34
|
+
service = context.client.class.name.split('::')[1] || 'AWS'
|
|
35
|
+
operation = context.operation_name.to_s
|
|
36
|
+
|
|
37
|
+
begin
|
|
38
|
+
response = @handler.call(context)
|
|
39
|
+
track_success(service, operation, started_at, context, response)
|
|
40
|
+
response
|
|
41
|
+
rescue StandardError => e
|
|
42
|
+
track_error(service, operation, started_at, context, e)
|
|
43
|
+
raise
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def track_success(service, operation, started_at, context, _response)
|
|
50
|
+
duration_ms = ((Time.now - started_at) * 1000).round(2)
|
|
51
|
+
|
|
52
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
53
|
+
"AWS #{service}.#{operation}",
|
|
54
|
+
category: 'aws',
|
|
55
|
+
level: :info,
|
|
56
|
+
data: {
|
|
57
|
+
service: service,
|
|
58
|
+
operation: operation,
|
|
59
|
+
region: context.config.region,
|
|
60
|
+
duration_ms: duration_ms
|
|
61
|
+
}
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
return unless BrainzLab.configuration.flux_effectively_enabled?
|
|
65
|
+
|
|
66
|
+
tags = { service: service, operation: operation, region: context.config.region }
|
|
67
|
+
BrainzLab::Flux.distribution('aws.duration_ms', duration_ms, tags: tags)
|
|
68
|
+
BrainzLab::Flux.increment('aws.requests', tags: tags)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def track_error(service, operation, started_at, _context, error)
|
|
72
|
+
((Time.now - started_at) * 1000).round(2)
|
|
73
|
+
|
|
74
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
75
|
+
"AWS #{service}.#{operation} failed: #{error.message}",
|
|
76
|
+
category: 'aws',
|
|
77
|
+
level: :error,
|
|
78
|
+
data: {
|
|
79
|
+
service: service,
|
|
80
|
+
operation: operation,
|
|
81
|
+
error: error.class.name
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
return unless BrainzLab.configuration.flux_effectively_enabled?
|
|
86
|
+
|
|
87
|
+
tags = { service: service, operation: operation, error_class: error.class.name }
|
|
88
|
+
BrainzLab::Flux.increment('aws.errors', tags: tags)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
::Seahorse::Client::Base.add_plugin(
|
|
93
|
+
Class.new(::Seahorse::Client::Plugin) do
|
|
94
|
+
define_method(:add_handlers) do |handlers, _config|
|
|
95
|
+
handlers.add(handler_class, step: :validate, priority: 0)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Aws SDK Plugin
|
|
103
|
+
class BrainzLabPlugin
|
|
104
|
+
def self.add_handlers(handlers, _config)
|
|
105
|
+
handlers.add(Handler, step: :validate, priority: 0)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
class Handler
|
|
109
|
+
def initialize(handler)
|
|
110
|
+
@handler = handler
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def call(context)
|
|
114
|
+
started_at = Time.now
|
|
115
|
+
service = extract_service(context)
|
|
116
|
+
operation = context.operation_name.to_s
|
|
117
|
+
|
|
118
|
+
begin
|
|
119
|
+
response = @handler.call(context)
|
|
120
|
+
track_request(service, operation, started_at, context, response)
|
|
121
|
+
response
|
|
122
|
+
rescue StandardError => e
|
|
123
|
+
track_error(service, operation, started_at, context, e)
|
|
124
|
+
raise
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
private
|
|
129
|
+
|
|
130
|
+
def extract_service(context)
|
|
131
|
+
context.client.class.name.to_s.split('::')[1] || 'AWS'
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def track_request(service, operation, started_at, context, _response)
|
|
135
|
+
duration_ms = ((Time.now - started_at) * 1000).round(2)
|
|
136
|
+
region = begin
|
|
137
|
+
context.config.region
|
|
138
|
+
rescue StandardError
|
|
139
|
+
'unknown'
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
143
|
+
"AWS #{service}.#{operation}",
|
|
144
|
+
category: 'aws',
|
|
145
|
+
level: :info,
|
|
146
|
+
data: {
|
|
147
|
+
service: service,
|
|
148
|
+
operation: operation,
|
|
149
|
+
region: region,
|
|
150
|
+
duration_ms: duration_ms,
|
|
151
|
+
retries: context.retries
|
|
152
|
+
}
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
return unless BrainzLab.configuration.flux_effectively_enabled?
|
|
156
|
+
|
|
157
|
+
tags = { service: service, operation: operation, region: region }
|
|
158
|
+
BrainzLab::Flux.distribution('aws.duration_ms', duration_ms, tags: tags)
|
|
159
|
+
BrainzLab::Flux.increment('aws.requests', tags: tags)
|
|
160
|
+
|
|
161
|
+
return unless context.retries.positive?
|
|
162
|
+
|
|
163
|
+
BrainzLab::Flux.increment('aws.retries', value: context.retries, tags: tags)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def track_error(service, operation, _started_at, _context, error)
|
|
167
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
168
|
+
"AWS #{service}.#{operation} failed",
|
|
169
|
+
category: 'aws',
|
|
170
|
+
level: :error,
|
|
171
|
+
data: { service: service, operation: operation, error: error.class.name }
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
return unless BrainzLab.configuration.flux_effectively_enabled?
|
|
175
|
+
|
|
176
|
+
tags = { service: service, operation: operation, error_class: error.class.name }
|
|
177
|
+
BrainzLab::Flux.increment('aws.errors', tags: tags)
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BrainzLab
|
|
4
|
+
module Instrumentation
|
|
5
|
+
module DalliInstrumentation
|
|
6
|
+
TRACKED_COMMANDS = %w[get set add replace delete incr decr cas get_multi set_multi].freeze
|
|
7
|
+
|
|
8
|
+
class << self
|
|
9
|
+
def install!
|
|
10
|
+
return unless defined?(::Dalli::Client)
|
|
11
|
+
|
|
12
|
+
install_client_instrumentation!
|
|
13
|
+
|
|
14
|
+
BrainzLab.debug_log('[Instrumentation] Dalli/Memcached instrumentation installed')
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def install_client_instrumentation!
|
|
20
|
+
::Dalli::Client.class_eval do
|
|
21
|
+
TRACKED_COMMANDS.each do |cmd|
|
|
22
|
+
original_method = "original_#{cmd}"
|
|
23
|
+
next if method_defined?(original_method)
|
|
24
|
+
|
|
25
|
+
alias_method original_method, cmd
|
|
26
|
+
|
|
27
|
+
define_method(cmd) do |*args, &block|
|
|
28
|
+
BrainzLab::Instrumentation::DalliInstrumentation.track_command(cmd, args) do
|
|
29
|
+
send(original_method, *args, &block)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def self.track_command(command, args)
|
|
38
|
+
started_at = Time.now
|
|
39
|
+
|
|
40
|
+
begin
|
|
41
|
+
result = yield
|
|
42
|
+
track_success(command, args, started_at, result)
|
|
43
|
+
result
|
|
44
|
+
rescue StandardError => e
|
|
45
|
+
track_error(command, args, started_at, e)
|
|
46
|
+
raise
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def self.track_success(command, args, started_at, result)
|
|
51
|
+
duration_ms = ((Time.now - started_at) * 1000).round(2)
|
|
52
|
+
key = extract_key(args)
|
|
53
|
+
|
|
54
|
+
# Add breadcrumb
|
|
55
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
56
|
+
"Memcached #{command.upcase}",
|
|
57
|
+
category: 'cache',
|
|
58
|
+
level: :info,
|
|
59
|
+
data: { command: command, key: key, duration_ms: duration_ms }
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
# Track with Flux
|
|
63
|
+
return unless BrainzLab.configuration.flux_effectively_enabled?
|
|
64
|
+
|
|
65
|
+
tags = { command: command }
|
|
66
|
+
BrainzLab::Flux.distribution('memcached.duration_ms', duration_ms, tags: tags)
|
|
67
|
+
BrainzLab::Flux.increment('memcached.commands', tags: tags)
|
|
68
|
+
|
|
69
|
+
# Track cache hits/misses for get commands
|
|
70
|
+
return unless command == 'get'
|
|
71
|
+
|
|
72
|
+
if result.nil?
|
|
73
|
+
BrainzLab::Flux.increment('memcached.miss', tags: tags)
|
|
74
|
+
else
|
|
75
|
+
BrainzLab::Flux.increment('memcached.hit', tags: tags)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def self.track_error(command, args, started_at, error)
|
|
80
|
+
((Time.now - started_at) * 1000).round(2)
|
|
81
|
+
key = extract_key(args)
|
|
82
|
+
|
|
83
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
84
|
+
"Memcached #{command.upcase} failed: #{error.message}",
|
|
85
|
+
category: 'cache',
|
|
86
|
+
level: :error,
|
|
87
|
+
data: { command: command, key: key, error: error.class.name }
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
return unless BrainzLab.configuration.flux_effectively_enabled?
|
|
91
|
+
|
|
92
|
+
BrainzLab::Flux.increment('memcached.errors', tags: { command: command })
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def self.extract_key(args)
|
|
96
|
+
key = args.first
|
|
97
|
+
case key
|
|
98
|
+
when String
|
|
99
|
+
key.length > 50 ? "#{key[0..47]}..." : key
|
|
100
|
+
when Array
|
|
101
|
+
"[#{key.size} keys]"
|
|
102
|
+
else
|
|
103
|
+
key.to_s[0..50]
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BrainzLab
|
|
4
|
+
module Instrumentation
|
|
5
|
+
module DelayedJobInstrumentation
|
|
6
|
+
@installed = false
|
|
7
|
+
|
|
8
|
+
class << self
|
|
9
|
+
def install!
|
|
10
|
+
return unless defined?(::Delayed::Job) || defined?(::Delayed::Backend)
|
|
11
|
+
return if @installed
|
|
12
|
+
|
|
13
|
+
# Install lifecycle hooks
|
|
14
|
+
install_lifecycle_hooks! if defined?(::Delayed::Worker)
|
|
15
|
+
|
|
16
|
+
# Install plugin if Delayed::Plugin is available
|
|
17
|
+
::Delayed::Worker.plugins << Plugin if defined?(::Delayed::Plugin)
|
|
18
|
+
|
|
19
|
+
@installed = true
|
|
20
|
+
BrainzLab.debug_log('Delayed::Job instrumentation installed')
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def installed?
|
|
24
|
+
@installed
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def reset!
|
|
28
|
+
@installed = false
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def install_lifecycle_hooks!
|
|
34
|
+
::Delayed::Worker.lifecycle.around(:invoke_job) do |job, *_args, &block|
|
|
35
|
+
around_invoke(job, &block)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
::Delayed::Worker.lifecycle.after(:error) do |_worker, job|
|
|
39
|
+
record_error(job)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
::Delayed::Worker.lifecycle.after(:failure) do |_worker, job|
|
|
43
|
+
record_failure(job)
|
|
44
|
+
end
|
|
45
|
+
rescue StandardError => e
|
|
46
|
+
BrainzLab.debug_log("Delayed::Job lifecycle hooks failed: #{e.message}")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def around_invoke(job, &block)
|
|
50
|
+
started_at = Time.now.utc
|
|
51
|
+
job_name = extract_job_name(job)
|
|
52
|
+
queue = job.queue || 'default'
|
|
53
|
+
|
|
54
|
+
# Calculate queue wait time
|
|
55
|
+
queue_wait_ms = job.created_at ? ((started_at - job.created_at) * 1000).round(2) : nil
|
|
56
|
+
|
|
57
|
+
# Set up context
|
|
58
|
+
setup_context(job, queue)
|
|
59
|
+
|
|
60
|
+
# Add breadcrumb
|
|
61
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
62
|
+
"DelayedJob #{job_name}",
|
|
63
|
+
category: 'job.delayed_job',
|
|
64
|
+
level: :info,
|
|
65
|
+
data: { job_id: job.id, queue: queue, attempts: job.attempts }
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
# Initialize Pulse tracing
|
|
69
|
+
Thread.current[:brainzlab_pulse_spans] = []
|
|
70
|
+
Thread.current[:brainzlab_pulse_breakdown] = nil
|
|
71
|
+
|
|
72
|
+
error_occurred = nil
|
|
73
|
+
begin
|
|
74
|
+
block.call(job)
|
|
75
|
+
rescue StandardError => e
|
|
76
|
+
error_occurred = e
|
|
77
|
+
raise
|
|
78
|
+
ensure
|
|
79
|
+
record_trace(
|
|
80
|
+
job: job,
|
|
81
|
+
job_name: job_name,
|
|
82
|
+
queue: queue,
|
|
83
|
+
started_at: started_at,
|
|
84
|
+
queue_wait_ms: queue_wait_ms,
|
|
85
|
+
error: error_occurred
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
cleanup_context
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def setup_context(job, queue)
|
|
93
|
+
BrainzLab::Context.current.set_context(
|
|
94
|
+
job_class: extract_job_name(job),
|
|
95
|
+
job_id: job.id,
|
|
96
|
+
queue_name: queue,
|
|
97
|
+
attempts: job.attempts
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def cleanup_context
|
|
102
|
+
Thread.current[:brainzlab_pulse_spans] = nil
|
|
103
|
+
Thread.current[:brainzlab_pulse_breakdown] = nil
|
|
104
|
+
BrainzLab::Context.clear!
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def record_trace(job:, job_name:, queue:, started_at:, queue_wait_ms:, error:)
|
|
108
|
+
return unless BrainzLab.configuration.pulse_enabled
|
|
109
|
+
|
|
110
|
+
ended_at = Time.now.utc
|
|
111
|
+
duration_ms = ((ended_at - started_at) * 1000).round(2)
|
|
112
|
+
|
|
113
|
+
# Collect spans
|
|
114
|
+
spans = Thread.current[:brainzlab_pulse_spans] || []
|
|
115
|
+
breakdown = Thread.current[:brainzlab_pulse_breakdown] || {}
|
|
116
|
+
|
|
117
|
+
formatted_spans = spans.map do |span|
|
|
118
|
+
{
|
|
119
|
+
span_id: span[:span_id],
|
|
120
|
+
name: span[:name],
|
|
121
|
+
kind: span[:kind],
|
|
122
|
+
started_at: format_timestamp(span[:started_at]),
|
|
123
|
+
ended_at: format_timestamp(span[:ended_at]),
|
|
124
|
+
duration_ms: span[:duration_ms],
|
|
125
|
+
data: span[:data]
|
|
126
|
+
}.compact
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
payload = {
|
|
130
|
+
trace_id: SecureRandom.uuid,
|
|
131
|
+
name: job_name,
|
|
132
|
+
kind: 'job',
|
|
133
|
+
started_at: started_at.utc.iso8601(3),
|
|
134
|
+
ended_at: ended_at.utc.iso8601(3),
|
|
135
|
+
duration_ms: duration_ms,
|
|
136
|
+
job_class: job_name,
|
|
137
|
+
job_id: job.id.to_s,
|
|
138
|
+
queue: queue,
|
|
139
|
+
queue_wait_ms: queue_wait_ms,
|
|
140
|
+
executions: (job.attempts || 0) + 1,
|
|
141
|
+
db_ms: breakdown[:db_ms],
|
|
142
|
+
error: error.present?,
|
|
143
|
+
error_class: error&.class&.name,
|
|
144
|
+
error_message: error&.message&.slice(0, 1000),
|
|
145
|
+
spans: formatted_spans,
|
|
146
|
+
environment: BrainzLab.configuration.environment,
|
|
147
|
+
commit: BrainzLab.configuration.commit,
|
|
148
|
+
host: BrainzLab.configuration.host
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
BrainzLab::Pulse.client.send_trace(payload.compact)
|
|
152
|
+
rescue StandardError => e
|
|
153
|
+
BrainzLab.debug_log("Delayed::Job trace recording failed: #{e.message}")
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def record_error(job)
|
|
157
|
+
return unless job.last_error
|
|
158
|
+
|
|
159
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
160
|
+
"DelayedJob error: #{extract_job_name(job)}",
|
|
161
|
+
category: 'job.delayed_job.error',
|
|
162
|
+
level: :error,
|
|
163
|
+
data: {
|
|
164
|
+
job_id: job.id,
|
|
165
|
+
attempts: job.attempts,
|
|
166
|
+
error: job.last_error&.slice(0, 500)
|
|
167
|
+
}
|
|
168
|
+
)
|
|
169
|
+
rescue StandardError => e
|
|
170
|
+
BrainzLab.debug_log("Delayed::Job error recording failed: #{e.message}")
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def record_failure(job)
|
|
174
|
+
BrainzLab::Reflex.add_breadcrumb(
|
|
175
|
+
"DelayedJob failed permanently: #{extract_job_name(job)}",
|
|
176
|
+
category: 'job.delayed_job.failure',
|
|
177
|
+
level: :error,
|
|
178
|
+
data: {
|
|
179
|
+
job_id: job.id,
|
|
180
|
+
attempts: job.attempts,
|
|
181
|
+
error: job.last_error&.slice(0, 500)
|
|
182
|
+
}
|
|
183
|
+
)
|
|
184
|
+
rescue StandardError => e
|
|
185
|
+
BrainzLab.debug_log("Delayed::Job failure recording failed: #{e.message}")
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def extract_job_name(job)
|
|
189
|
+
payload = job.payload_object
|
|
190
|
+
case payload
|
|
191
|
+
when ::Delayed::PerformableMethod
|
|
192
|
+
"#{payload.object.class}##{payload.method_name}"
|
|
193
|
+
when ::Delayed::PerformableMailer
|
|
194
|
+
"#{payload.object}##{payload.method_name}"
|
|
195
|
+
else
|
|
196
|
+
payload.class.name
|
|
197
|
+
end
|
|
198
|
+
rescue StandardError
|
|
199
|
+
job.name || 'Unknown'
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def format_timestamp(ts)
|
|
203
|
+
return nil unless ts
|
|
204
|
+
|
|
205
|
+
case ts
|
|
206
|
+
when Time, DateTime then ts.utc.iso8601(3)
|
|
207
|
+
when Float, Integer then Time.at(ts).utc.iso8601(3)
|
|
208
|
+
when String then ts
|
|
209
|
+
else ts.to_s
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Delayed::Job Plugin (alternative installation method)
|
|
215
|
+
if defined?(::Delayed::Plugin)
|
|
216
|
+
class Plugin < ::Delayed::Plugin
|
|
217
|
+
callbacks do |lifecycle|
|
|
218
|
+
lifecycle.around(:invoke_job) do |job, *_args, &block|
|
|
219
|
+
DelayedJobInstrumentation.send(:around_invoke, job, &block)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
lifecycle.after(:error) do |_worker, job|
|
|
223
|
+
DelayedJobInstrumentation.send(:record_error, job)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
lifecycle.after(:failure) do |_worker, job|
|
|
227
|
+
DelayedJobInstrumentation.send(:record_failure, job)
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|