forge_ops_tracker 0.7.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +55 -0
- data/README.md +105 -13
- data/lib/forge_ops_tracker/breadcrumb_buffer.rb +45 -0
- data/lib/forge_ops_tracker/client.rb +17 -0
- data/lib/forge_ops_tracker/configuration.rb +61 -0
- data/lib/forge_ops_tracker/error_subscriber.rb +23 -1
- data/lib/forge_ops_tracker/event_builder.rb +10 -3
- data/lib/forge_ops_tracker/failure_event_buffer.rb +73 -0
- data/lib/forge_ops_tracker/histogram_bucketer.rb +32 -0
- data/lib/forge_ops_tracker/infrastructure_metric_buffer.rb +19 -2
- data/lib/forge_ops_tracker/integrations/active_record_pool.rb +41 -0
- data/lib/forge_ops_tracker/integrations/net_http.rb +85 -3
- data/lib/forge_ops_tracker/integrations/redis_client.rb +62 -0
- data/lib/forge_ops_tracker/metric_buffer.rb +20 -3
- data/lib/forge_ops_tracker/middleware/breadcrumb_context.rb +35 -0
- data/lib/forge_ops_tracker/middleware/span_tracing.rb +38 -0
- data/lib/forge_ops_tracker/performance_flusher.rb +33 -5
- data/lib/forge_ops_tracker/performance_instrumentation.rb +32 -0
- data/lib/forge_ops_tracker/railtie.rb +160 -7
- data/lib/forge_ops_tracker/span_buffer.rb +65 -0
- data/lib/forge_ops_tracker/span_queue.rb +61 -0
- data/lib/forge_ops_tracker/version.rb +1 -1
- data/lib/forge_ops_tracker.rb +56 -0
- metadata +27 -4
|
@@ -14,6 +14,20 @@ module ForgeOpsTracker
|
|
|
14
14
|
def performance_flusher
|
|
15
15
|
@performance_flusher ||= ForgeOpsTracker::PerformanceFlusher.new(ForgeOpsTracker.configuration)
|
|
16
16
|
end
|
|
17
|
+
|
|
18
|
+
# Same reasoning as performance_flusher above, exposed the identical way: shared across the
|
|
19
|
+
# two failure-detecting call sites below (the perform.active_job subscription, Net::HTTP's
|
|
20
|
+
# own Timing#request), one buffer/thread rather than each starting a redundant one.
|
|
21
|
+
def failure_event_buffer
|
|
22
|
+
@failure_event_buffer ||= ForgeOpsTracker::FailureEventBuffer.new(ForgeOpsTracker.configuration)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Same reasoning again, exposed the identical way: shared across every call site that might
|
|
26
|
+
# finish a captured trace (Middleware::SpanTracing today; nothing else yet, but the same
|
|
27
|
+
# "one instance, not one per call site" reasoning applies regardless).
|
|
28
|
+
def span_queue
|
|
29
|
+
@span_queue ||= ForgeOpsTracker::SpanQueue.new(ForgeOpsTracker.configuration)
|
|
30
|
+
end
|
|
17
31
|
end
|
|
18
32
|
|
|
19
33
|
initializer "forge_ops_tracker.subscribe_error_reporter" do |app|
|
|
@@ -39,6 +53,26 @@ module ForgeOpsTracker
|
|
|
39
53
|
app.middleware.use ForgeOpsTracker::Middleware::UserContext, configuration: ForgeOpsTracker.configuration
|
|
40
54
|
end
|
|
41
55
|
|
|
56
|
+
# Gives every request its own fresh BreadcrumbBuffer to accumulate into (see
|
|
57
|
+
# ForgeOpsTracker::Middleware::BreadcrumbContext for why this has to be a middleware, the same
|
|
58
|
+
# "Rails.error.subscribe never receives the Rack env" reasoning UserContext's own comment
|
|
59
|
+
# gives): must run before the sql.active_record/process_action.action_controller subscriptions
|
|
60
|
+
# below can have anywhere to record into, but Rack middleware ordering guarantees that already,
|
|
61
|
+
# since this wraps app.call(env), and those notifications only ever fire from *inside* that
|
|
62
|
+
# call.
|
|
63
|
+
initializer "forge_ops_tracker.track_breadcrumbs" do |app|
|
|
64
|
+
app.middleware.use ForgeOpsTracker::Middleware::BreadcrumbContext, configuration: ForgeOpsTracker.configuration
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Same "must run before the notifications below have anywhere to record into" reasoning as
|
|
68
|
+
# track_breadcrumbs's own comment just above, for the exact same structural reason (Rack
|
|
69
|
+
# middleware always wraps the actual controller dispatch those notifications fire from inside
|
|
70
|
+
# of, regardless of this initializer's own position relative to track_breadcrumbs/
|
|
71
|
+
# track_current_user/track_sessions).
|
|
72
|
+
initializer "forge_ops_tracker.track_tracing" do |app|
|
|
73
|
+
app.middleware.use ForgeOpsTracker::Middleware::SpanTracing, configuration: ForgeOpsTracker.configuration
|
|
74
|
+
end
|
|
75
|
+
|
|
42
76
|
# Rails already fires "process_action.action_controller" after every request with the
|
|
43
77
|
# controller/action and how long it took (event.duration, in ms) built in, so unlike session
|
|
44
78
|
# tracking above (which needed a Rack middleware because there's no single existing hook for
|
|
@@ -54,34 +88,123 @@ module ForgeOpsTracker
|
|
|
54
88
|
# kept out of these blocks specifically so that decision logic is unit-testable without a real
|
|
55
89
|
# Rails/ActiveJob boot (this gem's own test suite doesn't require Rails at all; see that
|
|
56
90
|
# file's own comment).
|
|
91
|
+
#
|
|
92
|
+
# The same two subscriptions also record a breadcrumb, gated on track_breadcrumbs
|
|
93
|
+
# independently of track_performance: an app could want the trail without the timing data, or
|
|
94
|
+
# vice versa, and each already has to check its own flag regardless, so there's no real cost
|
|
95
|
+
# to keeping them independent rather than tying breadcrumbs to whether performance monitoring
|
|
96
|
+
# happens to be on.
|
|
97
|
+
#
|
|
98
|
+
# Every event.time below is wrapped in Time.at(...) before being handed to SpanBuffer#record or
|
|
99
|
+
# job_queue_wait_ms, both of which need a real Time. ActiveSupport::Notifications::Event#time
|
|
100
|
+
# returns a plain Float (seconds since the epoch) on the Rails version this gem currently
|
|
101
|
+
# targets, not a Time; Time.at(...) also passes a real Time straight through unchanged, so this
|
|
102
|
+
# stays correct if a future Rails version goes back to returning one.
|
|
57
103
|
initializer "forge_ops_tracker.track_performance" do
|
|
58
104
|
configuration = ForgeOpsTracker.configuration
|
|
59
105
|
flusher = Railtie.performance_flusher
|
|
60
106
|
|
|
61
107
|
ActiveSupport::Notifications.subscribe("process_action.action_controller") do |*args|
|
|
62
|
-
next unless configuration.
|
|
108
|
+
next unless configuration.enabled?
|
|
63
109
|
|
|
64
110
|
event = ActiveSupport::Notifications::Event.new(*args)
|
|
65
111
|
transaction_name = PerformanceInstrumentation.controller_transaction_name(event.payload)
|
|
66
|
-
|
|
112
|
+
|
|
113
|
+
if configuration.track_performance
|
|
114
|
+
flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "controller")
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
if configuration.track_breadcrumbs
|
|
118
|
+
status = event.payload[:status]
|
|
119
|
+
ForgeOpsTracker.add_breadcrumb(
|
|
120
|
+
"#{event.payload[:method]} #{transaction_name}",
|
|
121
|
+
category: "controller",
|
|
122
|
+
level: status && status >= 500 ? "error" : "info",
|
|
123
|
+
data: { status: status, path: event.payload[:path] }.compact
|
|
124
|
+
)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Records this request's own root span, not gated on track_performance/track_breadcrumbs
|
|
128
|
+
# above (its own independent configuration.track_tracing flag instead, the same "several
|
|
129
|
+
# genuinely independent mechanisms recorded from the same instrumentation point" pattern
|
|
130
|
+
# this exact call site already established for breadcrumbs): this is the one call in the
|
|
131
|
+
# whole gem that ever passes root: true, since process_action.action_controller is the only
|
|
132
|
+
# instrumentation point that wraps a genuinely whole request. See SpanBuffer#record's own
|
|
133
|
+
# comment for why root: true has to force parent_span_id nil explicitly here rather than
|
|
134
|
+
# reading it off the (already-popped-by-now) stack.
|
|
135
|
+
if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
|
|
136
|
+
buffer.pop
|
|
137
|
+
buffer.record(
|
|
138
|
+
span_id: buffer.root_span_id, name: transaction_name, kind: "controller",
|
|
139
|
+
started_at: Time.at(event.time), duration_ms: event.duration, root: true
|
|
140
|
+
)
|
|
141
|
+
end
|
|
67
142
|
end
|
|
68
143
|
|
|
69
144
|
ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
|
|
70
|
-
next unless configuration.
|
|
145
|
+
next unless configuration.enabled?
|
|
71
146
|
|
|
72
147
|
event = ActiveSupport::Notifications::Event.new(*args)
|
|
73
148
|
transaction_name = PerformanceInstrumentation.query_transaction_name(event.payload)
|
|
74
149
|
next if transaction_name.nil? # SCHEMA/cached, see PerformanceInstrumentation's own comment
|
|
75
150
|
|
|
76
|
-
|
|
151
|
+
if configuration.track_performance
|
|
152
|
+
flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "query")
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
if configuration.track_breadcrumbs
|
|
156
|
+
ForgeOpsTracker.add_breadcrumb(transaction_name, category: "query", data: { duration_ms: event.duration.round(1) })
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# A leaf span, parented onto whatever's currently open (the controller root, or a
|
|
160
|
+
# manually-wrapped ForgeOpsTracker.span block around it): a query has no children of its
|
|
161
|
+
# own to nest anything under, so unlike the controller root above, this never touches the
|
|
162
|
+
# stack itself, just records against whatever's already on top of it.
|
|
163
|
+
if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
|
|
164
|
+
buffer.record(
|
|
165
|
+
span_id: SecureRandom.hex(8), name: transaction_name, kind: "database",
|
|
166
|
+
started_at: Time.at(event.time), duration_ms: event.duration
|
|
167
|
+
)
|
|
168
|
+
end
|
|
77
169
|
end
|
|
78
170
|
|
|
79
171
|
ActiveSupport::Notifications.subscribe("perform.active_job") do |*args|
|
|
80
|
-
|
|
172
|
+
# Permissive enough for either concern below to run on its own: track_performance off but
|
|
173
|
+
# track_failures on (or vice versa) still needs this subscription to fire at all.
|
|
174
|
+
next unless (configuration.track_performance || configuration.track_failures) && configuration.enabled?
|
|
81
175
|
|
|
82
176
|
event = ActiveSupport::Notifications::Event.new(*args)
|
|
83
|
-
|
|
84
|
-
|
|
177
|
+
|
|
178
|
+
if configuration.track_performance
|
|
179
|
+
transaction_name = PerformanceInstrumentation.job_transaction_name(event.payload)
|
|
180
|
+
flusher.record(transaction_name: transaction_name, duration_ms: event.duration, kind: "job")
|
|
181
|
+
|
|
182
|
+
# A second, independent sample from the same notification: how long this job actually
|
|
183
|
+
# waited in its queue before this perform even started, not how long perform itself took
|
|
184
|
+
# (the "job" kind above). Grouped by queue name, not job class, since queue-level backlog
|
|
185
|
+
# is the actionable unit here (a slow queue affects every job class routed through it),
|
|
186
|
+
# the same reason Solid Queue's own StatsReporter already reports queue_depth per queue
|
|
187
|
+
# name rather than per job class. See PerformanceInstrumentation.job_queue_wait_ms's own
|
|
188
|
+
# comment for why this is nil, and skipped, for a job that was never actually enqueued.
|
|
189
|
+
wait_ms = PerformanceInstrumentation.job_queue_wait_ms(event.payload, at: Time.at(event.time))
|
|
190
|
+
if wait_ms
|
|
191
|
+
flusher.record(transaction_name: event.payload[:job].queue_name, duration_ms: wait_ms, kind: "job_latency")
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Independent of track_performance above, same reasoning breadcrumbs already established
|
|
196
|
+
# at this exact call site: a job whose perform raised, via
|
|
197
|
+
# PerformanceInstrumentation.job_failure_details' own use of ActiveSupport::Notifications'
|
|
198
|
+
# automatic payload[:exception_object] (see that method's own comment).
|
|
199
|
+
if configuration.track_failures
|
|
200
|
+
failure = PerformanceInstrumentation.job_failure_details(event.payload)
|
|
201
|
+
if failure
|
|
202
|
+
Railtie.failure_event_buffer.record(
|
|
203
|
+
kind: "job", transaction_name: PerformanceInstrumentation.job_transaction_name(event.payload),
|
|
204
|
+
error_class: failure[:error_class], error_message: failure[:error_message]
|
|
205
|
+
)
|
|
206
|
+
end
|
|
207
|
+
end
|
|
85
208
|
end
|
|
86
209
|
end
|
|
87
210
|
|
|
@@ -124,6 +247,19 @@ module ForgeOpsTracker
|
|
|
124
247
|
ForgeOpsTracker::Integrations::Puma::StatsReporter.start(configuration: ForgeOpsTracker.configuration)
|
|
125
248
|
end
|
|
126
249
|
|
|
250
|
+
# Same defined? guard as track_sidekiq/track_solid_queue/track_puma above, not
|
|
251
|
+
# track_net_http's unconditional one: ActiveRecord is neither Ruby stdlib nor a declared
|
|
252
|
+
# dependency of this gem itself (this gem has none at all beyond Rails::Railtie being
|
|
253
|
+
# defined; see the gemspec's own comment on why even Sidekiq/Puma are dev-only dependencies),
|
|
254
|
+
# so an API-only Rails app with ActiveRecord genuinely removed is a real, if uncommon, case to
|
|
255
|
+
# guard against rather than assume away.
|
|
256
|
+
initializer "forge_ops_tracker.track_active_record_pool" do
|
|
257
|
+
next unless defined?(::ActiveRecord::Base)
|
|
258
|
+
|
|
259
|
+
require "forge_ops_tracker/integrations/active_record_pool"
|
|
260
|
+
ForgeOpsTracker::Integrations::ActiveRecordPool::StatsReporter.start(configuration: ForgeOpsTracker.configuration)
|
|
261
|
+
end
|
|
262
|
+
|
|
127
263
|
# Net::HTTP is always present (Ruby's own standard library), so this always prepends; the
|
|
128
264
|
# actual on/off decision happens at call time inside Timing#request itself (checking
|
|
129
265
|
# configuration.track_performance/enabled? the same way the subscriptions above do), so
|
|
@@ -133,5 +269,22 @@ module ForgeOpsTracker
|
|
|
133
269
|
require "forge_ops_tracker/integrations/net_http"
|
|
134
270
|
Net::HTTP.prepend(ForgeOpsTracker::Integrations::NetHTTP::Timing)
|
|
135
271
|
end
|
|
272
|
+
|
|
273
|
+
# defined?(::RedisClient) guarded, the same way track_sidekiq/track_solid_queue/track_puma
|
|
274
|
+
# above are: RedisClient is neither Ruby stdlib nor a declared dependency of this gem (see the
|
|
275
|
+
# gemspec's own comment on why even Sidekiq/Puma are dev-only dependencies), so a host app with
|
|
276
|
+
# no Redis at all is a real case to guard against, not assume away. Runs after Rails itself has
|
|
277
|
+
# finished booting (config.after_initialize, not a plain initializer like track_net_http
|
|
278
|
+
# above): unlike Net::HTTP (Ruby stdlib, always loaded), whether RedisClient is defined yet at
|
|
279
|
+
# plain initializer time depends on load order relative to whatever gem requires it (redis,
|
|
280
|
+
# sidekiq, ActiveJob's redis queue adapter, ...), which this gem has no control over; deferring
|
|
281
|
+
# the defined? check to after every railtie (including this app's own dependencies) has
|
|
282
|
+
# finished initializing avoids a load-order-dependent false negative.
|
|
283
|
+
config.after_initialize do
|
|
284
|
+
next unless defined?(::RedisClient)
|
|
285
|
+
|
|
286
|
+
require "forge_ops_tracker/integrations/redis_client"
|
|
287
|
+
::RedisClient.prepend(ForgeOpsTracker::Integrations::RedisClientTiming)
|
|
288
|
+
end
|
|
136
289
|
end
|
|
137
290
|
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
require "securerandom"
|
|
2
|
+
|
|
3
|
+
module ForgeOpsTracker
|
|
4
|
+
# Accumulates one request's own nested call tree in-process; the tracing analog to
|
|
5
|
+
# BreadcrumbBuffer's per-thread trail. ForgeOpsTracker::Middleware::SpanTracing resets this to a
|
|
6
|
+
# fresh buffer at the start of every request, with a freshly generated trace_id and root
|
|
7
|
+
# span_id, and reads it back once the request is over to decide whether this trace is worth
|
|
8
|
+
# sending at all (see that middleware's own comment for exactly where and why).
|
|
9
|
+
#
|
|
10
|
+
# The stack exists so nesting can be reconstructed correctly even though most of what's recorded
|
|
11
|
+
# here (a query, an outbound HTTP call, a Redis call) only has a start+duration known *after* the
|
|
12
|
+
# fact, not a real "this span is now open" moment the way a manually-wrapped ForgeOpsTracker.span
|
|
13
|
+
# block has. Whatever's on top of the stack at the moment a leaf span is recorded is that leaf's
|
|
14
|
+
# real parent: the nearest currently-open span, whether that's the request's own root, or a
|
|
15
|
+
# service-level span a customer wrapped their own code in. The root span_id is pushed onto the
|
|
16
|
+
# stack immediately, at construction, specifically so every span recorded before the controller
|
|
17
|
+
# action itself finishes (which is every one of them, since process_action.action_controller's
|
|
18
|
+
# own subscription only fires once the whole action, including everything nested inside it, has
|
|
19
|
+
# already run) naturally nests under it without the root having to exist as a real recorded span
|
|
20
|
+
# first.
|
|
21
|
+
class SpanBuffer
|
|
22
|
+
attr_reader :trace_id, :root_span_id, :spans
|
|
23
|
+
|
|
24
|
+
def initialize
|
|
25
|
+
@trace_id = SecureRandom.hex(16)
|
|
26
|
+
@root_span_id = SecureRandom.hex(8)
|
|
27
|
+
@spans = []
|
|
28
|
+
@stack = [ @root_span_id ]
|
|
29
|
+
@root_duration_ms = nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def current_parent_id
|
|
33
|
+
@stack.last
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def push(span_id)
|
|
37
|
+
@stack.push(span_id)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def pop
|
|
41
|
+
@stack.pop
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# root: true only for the one call that records the request's own root span (see Railtie's
|
|
45
|
+
# process_action.action_controller subscription): forces parent_span_id nil explicitly, since
|
|
46
|
+
# by the time that fires, the root's own span_id has already been popped off the stack (there's
|
|
47
|
+
# nothing left on it to read back as "the parent" at that point - correctly, since the root has
|
|
48
|
+
# no parent of its own).
|
|
49
|
+
def record(span_id:, name:, kind:, started_at:, duration_ms:, data: {}, root: false)
|
|
50
|
+
parent_span_id = root ? nil : current_parent_id
|
|
51
|
+
spans << {
|
|
52
|
+
span_id: span_id, parent_span_id: parent_span_id, name: name, kind: kind,
|
|
53
|
+
started_at: started_at.utc.iso8601(3), duration_ms: duration_ms, data: data
|
|
54
|
+
}
|
|
55
|
+
@root_duration_ms = duration_ms if root
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# nil (never sent) until the root span has actually been recorded; a request that somehow never
|
|
59
|
+
# fires process_action.action_controller at all (a request Rails itself never routes) has no
|
|
60
|
+
# duration to compare against a threshold, so it's never mistakenly treated as "slow."
|
|
61
|
+
def slow?(threshold_ms)
|
|
62
|
+
@root_duration_ms && @root_duration_ms >= threshold_ms
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
require "thread"
|
|
2
|
+
|
|
3
|
+
module ForgeOpsTracker
|
|
4
|
+
# Same shape and reasoning as DeliveryQueue: a small in-process background thread + bounded
|
|
5
|
+
# queue, so delivering a captured trace never adds latency to the very request it was just
|
|
6
|
+
# measuring - the one thing that would be actively counterproductive here, blocking an already-
|
|
7
|
+
# slow request even longer just to report that it was slow. One whole trace (a trace_id plus
|
|
8
|
+
# every span belonging to it) is one queue entry, delivered as one client call, not batched
|
|
9
|
+
# together with other traces the way PerformanceFlusher/FailureEventBuffer batch over a time
|
|
10
|
+
# window: a trace is already a complete, immediately-relevant unit the moment a request
|
|
11
|
+
# finishes, so it's delivered promptly rather than held for up to a full flush interval.
|
|
12
|
+
class SpanQueue
|
|
13
|
+
def initialize(configuration, client: Client.new(configuration))
|
|
14
|
+
@configuration = configuration
|
|
15
|
+
@client = client
|
|
16
|
+
@queue = SizedQueue.new(configuration.queue_size)
|
|
17
|
+
@mutex = Mutex.new
|
|
18
|
+
@thread = nil
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Dropping a whole trace silently (never blocking the caller) if the queue is already full,
|
|
22
|
+
# the same backpressure-must-never-reach-the-host-app posture DeliveryQueue#push already
|
|
23
|
+
# documents; a burst of several slow requests at once must never stall a request that's
|
|
24
|
+
# already been identified as slow even further.
|
|
25
|
+
def push(trace_id:, spans:)
|
|
26
|
+
ensure_worker_started
|
|
27
|
+
@queue.push({ trace_id: trace_id, spans: spans }, true)
|
|
28
|
+
true
|
|
29
|
+
rescue ThreadError
|
|
30
|
+
log { "span delivery queue full, dropping trace" }
|
|
31
|
+
false
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
attr_reader :configuration, :client
|
|
36
|
+
|
|
37
|
+
def ensure_worker_started
|
|
38
|
+
return if @thread&.alive?
|
|
39
|
+
|
|
40
|
+
@mutex.synchronize do
|
|
41
|
+
return if @thread&.alive?
|
|
42
|
+
|
|
43
|
+
@thread = Thread.new { run }
|
|
44
|
+
@thread.abort_on_exception = false
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def run
|
|
49
|
+
loop do
|
|
50
|
+
entry = @queue.pop
|
|
51
|
+
client.deliver_spans(trace_id: entry[:trace_id], spans: entry[:spans])
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
log { "span delivery thread error: #{e.class}: #{e.message}" }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def log
|
|
58
|
+
configuration.logger&.debug { "[ForgeOpsTracker] #{yield}" }
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
data/lib/forge_ops_tracker.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
require "securerandom"
|
|
1
2
|
require "forge_ops_tracker/version"
|
|
2
3
|
require "forge_ops_tracker/configuration"
|
|
3
4
|
require "forge_ops_tracker/pii_scrubber"
|
|
@@ -6,13 +7,20 @@ require "forge_ops_tracker/client"
|
|
|
6
7
|
require "forge_ops_tracker/delivery_queue"
|
|
7
8
|
require "forge_ops_tracker/error_subscriber"
|
|
8
9
|
require "forge_ops_tracker/session_flusher"
|
|
10
|
+
require "forge_ops_tracker/histogram_bucketer"
|
|
9
11
|
require "forge_ops_tracker/performance_flusher"
|
|
10
12
|
require "forge_ops_tracker/performance_instrumentation"
|
|
11
13
|
require "forge_ops_tracker/metric_buffer"
|
|
12
14
|
require "forge_ops_tracker/infrastructure_metric_buffer"
|
|
15
|
+
require "forge_ops_tracker/failure_event_buffer"
|
|
13
16
|
require "forge_ops_tracker/periodic_poller"
|
|
17
|
+
require "forge_ops_tracker/breadcrumb_buffer"
|
|
18
|
+
require "forge_ops_tracker/span_buffer"
|
|
19
|
+
require "forge_ops_tracker/span_queue"
|
|
14
20
|
require "forge_ops_tracker/middleware/session_tracking"
|
|
15
21
|
require "forge_ops_tracker/middleware/user_context"
|
|
22
|
+
require "forge_ops_tracker/middleware/breadcrumb_context"
|
|
23
|
+
require "forge_ops_tracker/middleware/span_tracing"
|
|
16
24
|
|
|
17
25
|
module ForgeOpsTracker
|
|
18
26
|
class << self
|
|
@@ -59,6 +67,54 @@ module ForgeOpsTracker
|
|
|
59
67
|
Thread.current[:forge_ops_tracker_current_user] = user.empty? ? nil : user
|
|
60
68
|
end
|
|
61
69
|
|
|
70
|
+
# Adds one breadcrumb to the current request's trail (or, outside a request entirely, a
|
|
71
|
+
# background job or console session's own trail for the rest of this thread: see
|
|
72
|
+
# ForgeOpsTracker::Middleware::BreadcrumbContext for how a request gets its own buffer to start
|
|
73
|
+
# with). Lazily creates a buffer on this thread if BreadcrumbContext never ran (Sidekiq/Solid
|
|
74
|
+
# Queue jobs don't go through Rack middleware at all, an IRB/rails console session even less
|
|
75
|
+
# so), the same "works standalone, no specific setup required" shape ForgeOpsTracker.set_user
|
|
76
|
+
# already has. Works whether or not track_breadcrumbs is on: that flag only gates the automatic
|
|
77
|
+
# sql.active_record/process_action.action_controller/Net::HTTP sources Railtie installs, never
|
|
78
|
+
# this manual call.
|
|
79
|
+
def add_breadcrumb(message, category: "custom", level: "info", data: {})
|
|
80
|
+
buffer = (Thread.current[:forge_ops_tracker_breadcrumbs] ||= BreadcrumbBuffer.new(configuration))
|
|
81
|
+
buffer.add(category: category, message: message, level: level, data: data)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Wraps a customer's own service-level code as one named, nested span in the current request's
|
|
85
|
+
# trace (e.g. `ForgeOpsTracker.span("PaymentService#charge") { ... }`), so its database/Redis/
|
|
86
|
+
# outbound-HTTP calls show up nested underneath it in the waterfall rather than appearing to
|
|
87
|
+
# hang directly off the controller root - there's no automatic way to detect "this is a
|
|
88
|
+
# logically distinct service layer" the way a SQL query or an HTTP request already has a real
|
|
89
|
+
# instrumentation hook to detect, so this is deliberately explicit, opt-in, by hand, unlike
|
|
90
|
+
# every other span kind this gem records automatically.
|
|
91
|
+
#
|
|
92
|
+
# A plain pass-through (still runs the block, records nothing) with no error and no special
|
|
93
|
+
# handling whenever there's no current request trace to nest under at all: outside a request
|
|
94
|
+
# entirely (a background job, a console session), when track_tracing is off, or when the gem
|
|
95
|
+
# isn't enabled - the same "never break the host app's own code path" posture every other
|
|
96
|
+
# capture method in this gem already takes. Unlike add_breadcrumb, this does NOT lazily create
|
|
97
|
+
# a standalone buffer for itself: a lone span with no request-level trace around it, and no
|
|
98
|
+
# middleware left running to ever flush it, would just accumulate in this thread's memory
|
|
99
|
+
# forever with nothing to ever send it, which is worse than never recording it at all.
|
|
100
|
+
def span(name, kind: "service", data: {})
|
|
101
|
+
return yield unless configuration.enabled? && configuration.track_tracing
|
|
102
|
+
|
|
103
|
+
buffer = Thread.current[:forge_ops_tracker_spans]
|
|
104
|
+
return yield unless buffer
|
|
105
|
+
|
|
106
|
+
span_id = SecureRandom.hex(8)
|
|
107
|
+
started_at = Time.now.utc
|
|
108
|
+
buffer.push(span_id)
|
|
109
|
+
begin
|
|
110
|
+
yield
|
|
111
|
+
ensure
|
|
112
|
+
buffer.pop
|
|
113
|
+
duration_ms = (Time.now.utc - started_at) * 1000.0
|
|
114
|
+
buffer.record(span_id: span_id, name: name, kind: kind, started_at: started_at, duration_ms: duration_ms, data: data)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
62
118
|
private
|
|
63
119
|
def metric_buffer
|
|
64
120
|
@metric_buffer ||= MetricBuffer.new(configuration)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: forge_ops_tracker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.10.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- ForgeOps
|
|
@@ -65,8 +65,22 @@ dependencies:
|
|
|
65
65
|
- - "~>"
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: '1.8'
|
|
68
|
-
|
|
69
|
-
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: redis-client
|
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - "~>"
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '0.30'
|
|
75
|
+
type: :development
|
|
76
|
+
prerelease: false
|
|
77
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '0.30'
|
|
82
|
+
description: Hooks Rails' error reporter and reports exceptions to ForgeOps over HTTP,
|
|
83
|
+
without ever raising back into the host application.
|
|
70
84
|
executables: []
|
|
71
85
|
extensions: []
|
|
72
86
|
extra_rdoc_files: []
|
|
@@ -75,18 +89,25 @@ files:
|
|
|
75
89
|
- LICENSE.txt
|
|
76
90
|
- README.md
|
|
77
91
|
- lib/forge_ops_tracker.rb
|
|
92
|
+
- lib/forge_ops_tracker/breadcrumb_buffer.rb
|
|
78
93
|
- lib/forge_ops_tracker/client.rb
|
|
79
94
|
- lib/forge_ops_tracker/configuration.rb
|
|
80
95
|
- lib/forge_ops_tracker/delivery_queue.rb
|
|
81
96
|
- lib/forge_ops_tracker/error_subscriber.rb
|
|
82
97
|
- lib/forge_ops_tracker/event_builder.rb
|
|
98
|
+
- lib/forge_ops_tracker/failure_event_buffer.rb
|
|
99
|
+
- lib/forge_ops_tracker/histogram_bucketer.rb
|
|
83
100
|
- lib/forge_ops_tracker/infrastructure_metric_buffer.rb
|
|
101
|
+
- lib/forge_ops_tracker/integrations/active_record_pool.rb
|
|
84
102
|
- lib/forge_ops_tracker/integrations/net_http.rb
|
|
85
103
|
- lib/forge_ops_tracker/integrations/puma.rb
|
|
104
|
+
- lib/forge_ops_tracker/integrations/redis_client.rb
|
|
86
105
|
- lib/forge_ops_tracker/integrations/sidekiq.rb
|
|
87
106
|
- lib/forge_ops_tracker/integrations/solid_queue.rb
|
|
88
107
|
- lib/forge_ops_tracker/metric_buffer.rb
|
|
108
|
+
- lib/forge_ops_tracker/middleware/breadcrumb_context.rb
|
|
89
109
|
- lib/forge_ops_tracker/middleware/session_tracking.rb
|
|
110
|
+
- lib/forge_ops_tracker/middleware/span_tracing.rb
|
|
90
111
|
- lib/forge_ops_tracker/middleware/user_context.rb
|
|
91
112
|
- lib/forge_ops_tracker/performance_flusher.rb
|
|
92
113
|
- lib/forge_ops_tracker/performance_instrumentation.rb
|
|
@@ -94,6 +115,8 @@ files:
|
|
|
94
115
|
- lib/forge_ops_tracker/pii_scrubber.rb
|
|
95
116
|
- lib/forge_ops_tracker/railtie.rb
|
|
96
117
|
- lib/forge_ops_tracker/session_flusher.rb
|
|
118
|
+
- lib/forge_ops_tracker/span_buffer.rb
|
|
119
|
+
- lib/forge_ops_tracker/span_queue.rb
|
|
97
120
|
- lib/forge_ops_tracker/version.rb
|
|
98
121
|
homepage: https://getforgeops.net
|
|
99
122
|
licenses:
|
|
@@ -117,5 +140,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
117
140
|
requirements: []
|
|
118
141
|
rubygems_version: 4.0.11
|
|
119
142
|
specification_version: 4
|
|
120
|
-
summary: Rails exception reporting client for
|
|
143
|
+
summary: Rails exception reporting client for ForgeOps
|
|
121
144
|
test_files: []
|