forge_ops_tracker 0.5.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +72 -0
- data/README.md +214 -1
- data/lib/forge_ops_tracker/breadcrumb_buffer.rb +45 -0
- data/lib/forge_ops_tracker/client.rb +37 -0
- data/lib/forge_ops_tracker/configuration.rb +117 -0
- data/lib/forge_ops_tracker/error_subscriber.rb +28 -1
- data/lib/forge_ops_tracker/event_builder.rb +15 -6
- data/lib/forge_ops_tracker/failure_event_buffer.rb +73 -0
- data/lib/forge_ops_tracker/histogram_bucketer.rb +32 -0
- data/lib/forge_ops_tracker/infrastructure_metric_buffer.rb +86 -0
- data/lib/forge_ops_tracker/integrations/active_record_pool.rb +41 -0
- data/lib/forge_ops_tracker/integrations/net_http.rb +114 -0
- data/lib/forge_ops_tracker/integrations/puma.rb +55 -0
- data/lib/forge_ops_tracker/integrations/redis_client.rb +62 -0
- data/lib/forge_ops_tracker/integrations/sidekiq.rb +69 -0
- data/lib/forge_ops_tracker/integrations/solid_queue.rb +37 -0
- data/lib/forge_ops_tracker/metric_buffer.rb +97 -0
- data/lib/forge_ops_tracker/middleware/breadcrumb_context.rb +35 -0
- data/lib/forge_ops_tracker/middleware/span_tracing.rb +38 -0
- data/lib/forge_ops_tracker/middleware/user_context.rb +52 -0
- data/lib/forge_ops_tracker/performance_flusher.rb +129 -0
- data/lib/forge_ops_tracker/performance_instrumentation.rb +92 -0
- data/lib/forge_ops_tracker/periodic_poller.rb +50 -0
- data/lib/forge_ops_tracker/railtie.rb +272 -0
- data/lib/forge_ops_tracker/span_buffer.rb +65 -0
- data/lib/forge_ops_tracker/span_queue.rb +61 -0
- data/lib/forge_ops_tracker/version.rb +1 -1
- data/lib/forge_ops_tracker.rb +106 -0
- metadata +76 -1
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
require "thread"
|
|
2
|
+
|
|
3
|
+
module ForgeOpsTracker
|
|
4
|
+
# Collects individual job/dependency failures in-process (see Railtie's own perform.active_job
|
|
5
|
+
# subscription and Integrations::NetHTTP::Timing#request, the two places that actually call
|
|
6
|
+
# #record) and periodically flushes them as one batch, the same shape MetricBuffer already uses
|
|
7
|
+
# for the identical reason: a real failure's own detail (which job, what error) is exactly the
|
|
8
|
+
# kind of thing worth an accurate individual record of, not something to pre-aggregate away the
|
|
9
|
+
# way PerformanceFlusher's own count/sum/max buckets duration.
|
|
10
|
+
class FailureEventBuffer
|
|
11
|
+
def initialize(configuration, client: Client.new(configuration))
|
|
12
|
+
@configuration = configuration
|
|
13
|
+
@client = client
|
|
14
|
+
@mutex = Mutex.new
|
|
15
|
+
@entries = []
|
|
16
|
+
@thread = nil
|
|
17
|
+
|
|
18
|
+
# Flushes whatever's already buffered on a normal process exit, so the last partial window
|
|
19
|
+
# isn't silently dropped; same reasoning every other buffer/flusher in this gem documents.
|
|
20
|
+
at_exit { flush }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def record(kind:, transaction_name:, error_class: nil, error_message: nil)
|
|
24
|
+
ensure_worker_started
|
|
25
|
+
|
|
26
|
+
@mutex.synchronize do
|
|
27
|
+
@entries << {
|
|
28
|
+
kind: kind, transaction_name: transaction_name, error_class: error_class, error_message: error_message,
|
|
29
|
+
environment: configuration.environment.to_s, release: configuration.release,
|
|
30
|
+
occurred_at: Time.now.utc.iso8601
|
|
31
|
+
}
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Snapshots and resets the buffered entries, then delivers them as one batch. Same "a failed
|
|
36
|
+
# delivery keeps every entry for the next attempt" contract as MetricBuffer#flush.
|
|
37
|
+
def flush
|
|
38
|
+
snapshot = nil
|
|
39
|
+
|
|
40
|
+
@mutex.synchronize do
|
|
41
|
+
return if @entries.empty?
|
|
42
|
+
snapshot = @entries
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
return unless client.deliver_failure_events(snapshot)
|
|
46
|
+
|
|
47
|
+
@mutex.synchronize { @entries = [] }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
attr_reader :configuration, :client
|
|
52
|
+
|
|
53
|
+
def ensure_worker_started
|
|
54
|
+
return if @thread&.alive?
|
|
55
|
+
|
|
56
|
+
@mutex.synchronize do
|
|
57
|
+
return if @thread&.alive?
|
|
58
|
+
|
|
59
|
+
@thread = Thread.new { run }
|
|
60
|
+
@thread.abort_on_exception = false
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def run
|
|
65
|
+
loop do
|
|
66
|
+
sleep configuration.failure_event_flush_interval
|
|
67
|
+
flush
|
|
68
|
+
rescue StandardError => e
|
|
69
|
+
configuration.logger&.debug { "[ForgeOpsTracker] failure event flush thread error: #{e.class}: #{e.message}" }
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
# Buckets a single duration into one of a fixed set of latency-range labels, the building block
|
|
3
|
+
# PerformanceFlusher uses to accumulate an approximate distribution (not just count/sum/max)
|
|
4
|
+
# alongside every [transaction_name, kind] bucket it already tallies. The server merges these
|
|
5
|
+
# counts across matching samples at read time and walks cumulative counts to approximate a
|
|
6
|
+
# percentile, the same "accurate to the bucket width, not exact" trade-off Prometheus's own
|
|
7
|
+
# histogram_quantile makes: this SDK does not store, or need, the raw duration list a true
|
|
8
|
+
# percentile would require.
|
|
9
|
+
#
|
|
10
|
+
# BOUNDARIES_MS is duplicated (not shared via a gem dependency) on the server side, in
|
|
11
|
+
# app/services/histogram_percentile.rb: the gem and the Rails app it reports to are separate
|
|
12
|
+
# deployables, the same "duplicated, not imported, so this package's own dependency graph stays
|
|
13
|
+
# independent" reasoning already applied elsewhere in this repo (e.g. the React Native SDK's own
|
|
14
|
+
# copy of the TypeScript SDK's PII scrubber). Change one, change the other, or a released gem
|
|
15
|
+
# version and the server version it talks to would silently disagree about what each bucket label
|
|
16
|
+
# means.
|
|
17
|
+
module HistogramBucketer
|
|
18
|
+
BOUNDARIES_MS = [ 50, 100, 250, 500, 1000, 2500, 5000, 10_000 ].freeze
|
|
19
|
+
|
|
20
|
+
# Returns the label (a String) of the smallest boundary duration_ms fits under, or "inf" for
|
|
21
|
+
# anything larger than the largest boundary. String, not Integer/Float: this travels as a JSON
|
|
22
|
+
# object key once flushed (see PerformanceFlusher#flush), and JSON object keys are always
|
|
23
|
+
# strings regardless of what Ruby type builds the Hash locally, so returning a String here
|
|
24
|
+
# keeps the in-process Hash's own keys identical to what actually goes over the wire, rather
|
|
25
|
+
# than looking different locally and then silently getting stringified only at serialization
|
|
26
|
+
# time.
|
|
27
|
+
def self.bucket_for(duration_ms)
|
|
28
|
+
boundary = BOUNDARIES_MS.find { |b| duration_ms <= b }
|
|
29
|
+
boundary ? boundary.to_s : "inf"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
require "thread"
|
|
2
|
+
|
|
3
|
+
module ForgeOpsTracker
|
|
4
|
+
# Collects individual ForgeOpsTracker.capture_infrastructure_metric readings in-process and
|
|
5
|
+
# periodically flushes them as one batch, same shape as MetricBuffer (a sibling class, not a
|
|
6
|
+
# shared base -- this gem keeps SessionFlusher/PerformanceFlusher as separate, independently-
|
|
7
|
+
# readable classes despite their own similar shape too, and these two follow that precedent).
|
|
8
|
+
# Its main real caller is a short-lived cron/script process that exits right after a handful of
|
|
9
|
+
# capture calls, so the at_exit flush below is what actually matters in practice; the periodic
|
|
10
|
+
# background-thread timer still runs for a long-lived process that chooses to call this
|
|
11
|
+
# continuously instead, but rarely gets the chance to fire in the cron case.
|
|
12
|
+
class InfrastructureMetricBuffer
|
|
13
|
+
# Once this many entries are buffered, further ones are dropped until a flush succeeds: a plan
|
|
14
|
+
# without the feature answers 403 on every flush, and an uncapped buffer would then grow for as
|
|
15
|
+
# long as the process lives. Dropping the newest rather than the oldest keeps the entries a flush
|
|
16
|
+
# is delivering at the front of the array, which is what makes removing exactly them exact.
|
|
17
|
+
MAX_ENTRIES = 1000
|
|
18
|
+
|
|
19
|
+
def initialize(configuration, client: Client.new(configuration))
|
|
20
|
+
@configuration = configuration
|
|
21
|
+
@client = client
|
|
22
|
+
@mutex = Mutex.new
|
|
23
|
+
@entries = []
|
|
24
|
+
@thread = nil
|
|
25
|
+
|
|
26
|
+
at_exit { flush }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def record(metric_name:, value:, hostname:)
|
|
30
|
+
return false unless keepable?(value)
|
|
31
|
+
|
|
32
|
+
ensure_worker_started
|
|
33
|
+
|
|
34
|
+
@mutex.synchronize do
|
|
35
|
+
return false if @entries.size >= MAX_ENTRIES
|
|
36
|
+
|
|
37
|
+
@entries << { metric_name: metric_name, value: value, hostname: hostname, recorded_at: Time.now.utc.iso8601 }
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Same reset-on-success/keep-on-failure contract as MetricBuffer#flush; see that class's own
|
|
42
|
+
# comment for the accepted race this shares with it.
|
|
43
|
+
def flush
|
|
44
|
+
snapshot = nil
|
|
45
|
+
|
|
46
|
+
@mutex.synchronize do
|
|
47
|
+
return if @entries.empty?
|
|
48
|
+
snapshot = @entries.dup
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
return unless client.deliver_infrastructure_metrics(snapshot)
|
|
52
|
+
|
|
53
|
+
# Exactly the entries just delivered: anything recorded while the request was in flight sits
|
|
54
|
+
# after them and stays for the next flush (resetting the whole array here lost it).
|
|
55
|
+
@mutex.synchronize { @entries.shift(snapshot.size) }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
attr_reader :configuration, :client
|
|
60
|
+
|
|
61
|
+
# A NaN or infinite value is not valid JSON and would make the server reject the whole batch.
|
|
62
|
+
def keepable?(value)
|
|
63
|
+
value.is_a?(Numeric) && value.to_f.finite?
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def ensure_worker_started
|
|
67
|
+
return if @thread&.alive?
|
|
68
|
+
|
|
69
|
+
@mutex.synchronize do
|
|
70
|
+
return if @thread&.alive?
|
|
71
|
+
|
|
72
|
+
@thread = Thread.new { run }
|
|
73
|
+
@thread.abort_on_exception = false
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def run
|
|
78
|
+
loop do
|
|
79
|
+
sleep configuration.infrastructure_metric_flush_interval
|
|
80
|
+
flush
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
configuration.logger&.debug { "[ForgeOpsTracker] infrastructure metric flush thread error: #{e.class}: #{e.message}" }
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module ActiveRecordPool
|
|
4
|
+
# Reports this process's own ActiveRecord connection pool utilization as infrastructure
|
|
5
|
+
# metrics on a periodic timer, the same shape as Sidekiq::StatsReporter/SolidQueue::StatsReporter
|
|
6
|
+
# (see either's own comment), and, like both of those, only ever loaded when ActiveRecord is
|
|
7
|
+
# actually present (checked with defined? in Railtie, not assumed).
|
|
8
|
+
#
|
|
9
|
+
# Scoped to the current process's own primary connection pool only, via the bare
|
|
10
|
+
# ActiveRecord::Base.connection_pool this process is actually using: a known v1 limitation,
|
|
11
|
+
# not an oversight, for an app with multiple databases (ActiveRecord::Base.connection_pool is
|
|
12
|
+
# only ever one of potentially several pools such an app has configured). Reporting every
|
|
13
|
+
# configured pool would need ActiveRecord::Base.connection_handler's own registry, a genuinely
|
|
14
|
+
# separate piece of work; this integration reports what a single-database app (the common
|
|
15
|
+
# case, and the only case this gem's own test suite/README examples ever set up) actually has.
|
|
16
|
+
class StatsReporter
|
|
17
|
+
def self.start(configuration: ForgeOpsTracker.configuration)
|
|
18
|
+
return unless configuration.track_performance && configuration.enabled?
|
|
19
|
+
|
|
20
|
+
PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def report
|
|
24
|
+
stat = ::ActiveRecord::Base.connection_pool.stat
|
|
25
|
+
size = stat[:size].to_i
|
|
26
|
+
busy = stat[:busy].to_i
|
|
27
|
+
|
|
28
|
+
# 0/0 (a pool nobody's ever checked a connection out of yet, or, in practice, never) reads
|
|
29
|
+
# as 0% utilized, not NaN/an error: a pool that's never been used is exactly as "not under
|
|
30
|
+
# pressure" as a pool with plenty of headroom, not a value worth surfacing as broken.
|
|
31
|
+
utilization_pct = size.zero? ? 0.0 : (100.0 * busy / size).round(1)
|
|
32
|
+
|
|
33
|
+
ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.utilization_pct", value: utilization_pct)
|
|
34
|
+
ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.size", value: size)
|
|
35
|
+
ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.busy", value: busy)
|
|
36
|
+
ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.waiting", value: stat[:waiting].to_i)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module NetHTTP
|
|
4
|
+
# Times every outbound Net::HTTP call, applied once via Net::HTTP.prepend(Timing) from
|
|
5
|
+
# Railtie. Ruby's standard library has no ActiveSupport::Notifications instrumentation of
|
|
6
|
+
# its own for outbound HTTP the way it does for ActiveRecord/ActionController, so wrapping
|
|
7
|
+
# #request directly is the only hook available; most other HTTP client libraries in the Ruby
|
|
8
|
+
# ecosystem (Faraday's own net_http adapter, HTTParty, RestClient) ultimately call through
|
|
9
|
+
# Net::HTTP too, so this covers those as a side effect without needing a wrapper per library.
|
|
10
|
+
#
|
|
11
|
+
# transaction_name is "<HTTP method> <host>", not the full URL: a request's own path or
|
|
12
|
+
# query string could carry an id or a token, the same low-cardinality/no-secrets-in-a-label
|
|
13
|
+
# reasoning every other transaction_name in this system already follows.
|
|
14
|
+
#
|
|
15
|
+
# Known, honest gap, confirmed directly against Net::HTTP's own source and real behavior, not
|
|
16
|
+
# assumed: Net::HTTP.get_response and the Net::HTTP.start { |http| ... } block form both open
|
|
17
|
+
# the TCP connection inside #start, *before* the block (and so before #request, the only
|
|
18
|
+
# method this wraps) ever runs, so a connection-refused/DNS-failure error raised during that
|
|
19
|
+
# setup is structurally invisible here, for timing, breadcrumbs, and the failure detection
|
|
20
|
+
# below alike (a pre-existing limitation this round's own test suite is what actually
|
|
21
|
+
# surfaced, not something newly introduced). Net::HTTP.new(...).request(...), the other
|
|
22
|
+
# common calling convention (no explicit #start at all), is different and unaffected:
|
|
23
|
+
# confirmed directly that Net::HTTP connects lazily, inside #request itself, the first time
|
|
24
|
+
# it's called on a not-yet-started instance, so that path's own connection failures are
|
|
25
|
+
# caught exactly like any other #request failure.
|
|
26
|
+
module Timing
|
|
27
|
+
# The guard (return super unless ...) skips timing, but the ensure block below still runs
|
|
28
|
+
# on every exit path regardless, same as any ensure; start staying nil is what actually
|
|
29
|
+
# skips recording there, not a second guard duplicating this one.
|
|
30
|
+
def request(req, body = nil, &block)
|
|
31
|
+
configuration = ForgeOpsTracker.configuration
|
|
32
|
+
return super unless (configuration.track_performance || configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
|
|
33
|
+
|
|
34
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
35
|
+
response = super
|
|
36
|
+
rescue StandardError => e
|
|
37
|
+
# Only track_failures below needs the actual exception object (for error_class/
|
|
38
|
+
# error_message); everything else keys off response staying nil, already true on this
|
|
39
|
+
# path without capturing anything. re-raised unchanged: this gem must never swallow or
|
|
40
|
+
# alter what the host app's own call to Net::HTTP#request would have raised.
|
|
41
|
+
raised_error = e
|
|
42
|
+
raise
|
|
43
|
+
ensure
|
|
44
|
+
if start
|
|
45
|
+
duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
|
|
46
|
+
# Computed once, unconditionally, not separately inside each flag-gated block below:
|
|
47
|
+
# both the breadcrumb and track_failures sections need it, and a value only ever
|
|
48
|
+
# assigned inside one conditional branch but read from another relies on Ruby's own
|
|
49
|
+
# "a local is declared the moment its assignment is lexically parsed, regardless of
|
|
50
|
+
# whether that branch actually ran" scoping, a real but easy-to-break-on-reorder
|
|
51
|
+
# subtlety not worth the risk here.
|
|
52
|
+
status = response&.code&.to_i
|
|
53
|
+
if configuration.track_performance
|
|
54
|
+
Railtie.performance_flusher.record(transaction_name: "#{req.method} #{address}", duration_ms: duration_ms, kind: "http")
|
|
55
|
+
end
|
|
56
|
+
# The same "method + host, never the full path/query string" reasoning
|
|
57
|
+
# transaction_name above already follows: a URL's own path or query could carry an id
|
|
58
|
+
# or a token, and this is a breadcrumb message, not a value this gem controls the
|
|
59
|
+
# cardinality of. Mirrors sdks/typescript's own fetch/xhr breadcrumbs: category
|
|
60
|
+
# "http", status + duration_ms in data, level "warning" for a non-2xx/3xx response.
|
|
61
|
+
if configuration.track_breadcrumbs
|
|
62
|
+
# No status at all means the request itself raised (a timeout, a connection
|
|
63
|
+
# refused, ...) before a response ever came back: worth "error", not the misleading
|
|
64
|
+
# "info" a bare status-based check would default a nil status to, the same "the
|
|
65
|
+
# request truly failed" case sdks/typescript's own fetch breadcrumb catch block
|
|
66
|
+
# marks "error" for.
|
|
67
|
+
level = if status.nil?
|
|
68
|
+
"error"
|
|
69
|
+
elsif status >= 400
|
|
70
|
+
"warning"
|
|
71
|
+
else
|
|
72
|
+
"info"
|
|
73
|
+
end
|
|
74
|
+
ForgeOpsTracker.add_breadcrumb(
|
|
75
|
+
"#{req.method} #{address}",
|
|
76
|
+
category: "http",
|
|
77
|
+
level: level,
|
|
78
|
+
data: { status: status, duration_ms: duration_ms.round(1) }.compact
|
|
79
|
+
)
|
|
80
|
+
end
|
|
81
|
+
# Independent of track_performance/track_breadcrumbs above, same "several genuinely
|
|
82
|
+
# independent mechanisms recorded from the same instrumentation point" pattern this
|
|
83
|
+
# exact call site already established for breadcrumbs. A failure is the request
|
|
84
|
+
# raising, or a 5xx: not a 4xx, the identical distinction the breadcrumb level just
|
|
85
|
+
# above already draws between "the dependency is unhealthy" (worth "error"/failure)
|
|
86
|
+
# and "the dependency responded, just not with success" (only "warning").
|
|
87
|
+
if configuration.track_failures && (status.nil? || status >= 500)
|
|
88
|
+
Railtie.failure_event_buffer.record(
|
|
89
|
+
kind: "dependency", transaction_name: "#{req.method} #{address}",
|
|
90
|
+
error_class: raised_error&.class&.name, error_message: raised_error&.message
|
|
91
|
+
)
|
|
92
|
+
end
|
|
93
|
+
# A leaf span, the same "parented onto whatever's currently open, never touches the
|
|
94
|
+
# stack itself" shape the sql.active_record subscription's own span recording already
|
|
95
|
+
# takes: an outbound HTTP call has no children of its own to nest anything under
|
|
96
|
+
# either.
|
|
97
|
+
if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
|
|
98
|
+
# duration_ms itself is exact (computed from the monotonic clock above); started_at
|
|
99
|
+
# here is only derived from it (now minus duration), an approximation off the wall
|
|
100
|
+
# clock rather than a second real timestamp captured at the top of this method - a
|
|
101
|
+
# few milliseconds of drift on where the bar starts in a waterfall, never on how long
|
|
102
|
+
# it actually took.
|
|
103
|
+
start_time = ::Time.now.utc - (duration_ms / 1000.0)
|
|
104
|
+
buffer.record(
|
|
105
|
+
span_id: SecureRandom.hex(8), name: "#{req.method} #{address}", kind: "http",
|
|
106
|
+
started_at: start_time, duration_ms: duration_ms, data: { status: status }.compact
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module Puma
|
|
4
|
+
# Reports Puma's own backlog/running-threads/pool-capacity as gauge-style infrastructure
|
|
5
|
+
# metrics on a periodic timer (see PeriodicPoller). Only ever loaded when Puma is already
|
|
6
|
+
# present (checked with defined? in Railtie).
|
|
7
|
+
#
|
|
8
|
+
# Puma.stats_hash's own shape genuinely differs by deployment mode, confirmed directly
|
|
9
|
+
# against Puma 6.6.1's real source rather than assumed: single-mode (Puma::Single, no
|
|
10
|
+
# `workers` config) has backlog/running/pool_capacity at the top level (Server#stats merged
|
|
11
|
+
# with ThreadPool#stats); clustered mode (Puma::Cluster) has none of those at the top level
|
|
12
|
+
# at all, only a worker_status array, one entry per forked worker, each carrying its own
|
|
13
|
+
# last_status hash with the identical three keys (parsed from that worker's own periodic
|
|
14
|
+
# IPC ping, see Puma::Cluster::WorkerHandle's own STATUS_PATTERN), summed across workers
|
|
15
|
+
# here for one whole-process reading, rather than reported per worker, since ForgeOps'
|
|
16
|
+
# infrastructure dataset already aggregates by hostname, not by worker index. Puma.stats_object
|
|
17
|
+
# is set unconditionally by Puma::Launcher on every normal boot (not just when a control/
|
|
18
|
+
# stats app is configured), so Puma.stats_hash is populated by the time this ever runs; still
|
|
19
|
+
# guarded against nil/an unrecognized shape, both because that wiring could change in a
|
|
20
|
+
# future Puma version and because this must never be the thing that crashes a poll tick.
|
|
21
|
+
class StatsReporter
|
|
22
|
+
def self.start(configuration: ForgeOpsTracker.configuration)
|
|
23
|
+
return unless configuration.track_performance && configuration.enabled?
|
|
24
|
+
|
|
25
|
+
PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def report
|
|
29
|
+
stats = ::Puma.stats_hash
|
|
30
|
+
return unless stats
|
|
31
|
+
|
|
32
|
+
if stats[:backlog] && stats[:running]
|
|
33
|
+
report_metrics(stats[:backlog], stats[:running], stats[:pool_capacity])
|
|
34
|
+
elsif stats[:worker_status]
|
|
35
|
+
statuses = stats[:worker_status].filter_map { |worker| worker[:last_status] }.reject(&:empty?)
|
|
36
|
+
return if statuses.empty?
|
|
37
|
+
|
|
38
|
+
report_metrics(
|
|
39
|
+
statuses.sum { |s| s[:backlog] || 0 },
|
|
40
|
+
statuses.sum { |s| s[:running] || 0 },
|
|
41
|
+
statuses.sum { |s| s[:pool_capacity] || 0 }
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
def report_metrics(backlog, running, pool_capacity)
|
|
48
|
+
ForgeOpsTracker.capture_infrastructure_metric("puma.backlog", value: backlog)
|
|
49
|
+
ForgeOpsTracker.capture_infrastructure_metric("puma.running_threads", value: running)
|
|
50
|
+
ForgeOpsTracker.capture_infrastructure_metric("puma.pool_capacity", value: pool_capacity)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module RedisClientTiming
|
|
4
|
+
# Times every Redis command, applied once via RedisClient.prepend(Timing) from Railtie, the
|
|
5
|
+
# same Module#prepend + super shape Integrations::NetHTTP::Timing already uses for the
|
|
6
|
+
# identical reason: neither Ruby's Redis ecosystem nor RedisClient itself fires an
|
|
7
|
+
# ActiveSupport::Notifications event of its own, so wrapping #call directly is the only hook
|
|
8
|
+
# available. Targets RedisClient specifically (the modern, actively-maintained low-level
|
|
9
|
+
# client both the current redis gem (5.x+) and standalone RedisClient users go through), not
|
|
10
|
+
# the older redis-rb 4.x Redis::Client - confirmed directly (not assumed) that redis 5.x no
|
|
11
|
+
# longer defines Redis::Client at all, so a hook there would silently do nothing on any
|
|
12
|
+
# currently-installed version of the gem.
|
|
13
|
+
#
|
|
14
|
+
# Unlike controller/query timing, this is span-only: there's no "redis" PerformanceSample
|
|
15
|
+
# kind for this to feed into the aggregate Performance page (that page has no Redis panel at
|
|
16
|
+
# all yet), so this only ever contributes to a captured trace's own waterfall, plus a
|
|
17
|
+
# breadcrumb and a dependency failure record, the same two other independent mechanisms
|
|
18
|
+
# NetHTTP::Timing's own call site already establishes.
|
|
19
|
+
#
|
|
20
|
+
# name is "Redis <COMMAND>" (the command verb alone, upcased: GET, SET, LPUSH, ...), never
|
|
21
|
+
# the key or any argument - the same low-cardinality, no-secrets-in-a-label reasoning every
|
|
22
|
+
# other transaction_name/span name in this gem already follows; a Redis key can easily carry
|
|
23
|
+
# a customer's own id or other sensitive value.
|
|
24
|
+
def call(*command, **kwargs)
|
|
25
|
+
configuration = ForgeOpsTracker.configuration
|
|
26
|
+
return super unless (configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
|
|
27
|
+
|
|
28
|
+
name = "Redis #{command.first.to_s.upcase}"
|
|
29
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
30
|
+
result = super
|
|
31
|
+
rescue StandardError => e
|
|
32
|
+
raised_error = e
|
|
33
|
+
raise
|
|
34
|
+
ensure
|
|
35
|
+
if start
|
|
36
|
+
duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
|
|
37
|
+
|
|
38
|
+
if configuration.track_breadcrumbs
|
|
39
|
+
ForgeOpsTracker.add_breadcrumb(
|
|
40
|
+
name, category: "redis", level: raised_error ? "error" : "info",
|
|
41
|
+
data: { duration_ms: duration_ms.round(1) }
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
if configuration.track_failures && raised_error
|
|
46
|
+
Railtie.failure_event_buffer.record(
|
|
47
|
+
kind: "dependency", transaction_name: name,
|
|
48
|
+
error_class: raised_error.class.name, error_message: raised_error.message
|
|
49
|
+
)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
|
|
53
|
+
buffer.record(
|
|
54
|
+
span_id: SecureRandom.hex(8), name: name, kind: "redis",
|
|
55
|
+
started_at: ::Time.now.utc - (duration_ms / 1000.0), duration_ms: duration_ms
|
|
56
|
+
)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Sidekiq::Stats/Sidekiq::Queue (used by StatsReporter below) live in sidekiq/api, not
|
|
2
|
+
# autoloaded by a bare "sidekiq" require; a real Sidekiq server process almost always already has
|
|
3
|
+
# this loaded some other way, but requiring it explicitly here doesn't depend on that happening to
|
|
4
|
+
# be true.
|
|
5
|
+
require "sidekiq/api"
|
|
6
|
+
|
|
7
|
+
module ForgeOpsTracker
|
|
8
|
+
module Integrations
|
|
9
|
+
module Sidekiq
|
|
10
|
+
# Times a raw (non-ActiveJob) Sidekiq::Worker's own #perform. Registered via
|
|
11
|
+
# Sidekiq.configure_server { |c| c.server_middleware { |chain| chain.add ... } } (see
|
|
12
|
+
# Railtie); only ever loaded when Sidekiq is already present (checked with defined? there),
|
|
13
|
+
# matching every other optional integration in this gem.
|
|
14
|
+
#
|
|
15
|
+
# Skips (yields, but doesn't record) any job whose worker is ActiveJob's own Sidekiq wrapper:
|
|
16
|
+
# see PerformanceInstrumentation.sidekiq_worker_is_active_job_wrapper?'s own comment for why
|
|
17
|
+
# that job is already counted by the perform.active_job subscription instead, and would be
|
|
18
|
+
# double-counted here otherwise.
|
|
19
|
+
class Middleware
|
|
20
|
+
include ::Sidekiq::ServerMiddleware
|
|
21
|
+
|
|
22
|
+
def initialize(configuration: ForgeOpsTracker.configuration, flusher: Railtie.performance_flusher)
|
|
23
|
+
@configuration = configuration
|
|
24
|
+
@flusher = flusher
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def call(job_instance, _msg, _queue)
|
|
28
|
+
skip = !@configuration.track_performance || !@configuration.enabled? ||
|
|
29
|
+
PerformanceInstrumentation.sidekiq_worker_is_active_job_wrapper?(job_instance)
|
|
30
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) unless skip
|
|
31
|
+
|
|
32
|
+
yield
|
|
33
|
+
ensure
|
|
34
|
+
unless skip
|
|
35
|
+
duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
|
|
36
|
+
@flusher.record(transaction_name: job_instance.class.name, duration_ms: duration_ms, kind: "job")
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Reports Sidekiq's own aggregate stats (Sidekiq::Stats, Sidekiq::Queue) as gauge-style
|
|
42
|
+
# infrastructure metrics on a periodic timer (see PeriodicPoller), via the same public
|
|
43
|
+
# ForgeOpsTracker.capture_infrastructure_metric every customer's own manual infrastructure-
|
|
44
|
+
# monitoring script already uses; no new ingestion endpoint or dashboard dataset needed.
|
|
45
|
+
class StatsReporter
|
|
46
|
+
def self.start(configuration: ForgeOpsTracker.configuration)
|
|
47
|
+
return unless configuration.track_performance && configuration.enabled?
|
|
48
|
+
|
|
49
|
+
PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def report
|
|
53
|
+
stats = ::Sidekiq::Stats.new
|
|
54
|
+
|
|
55
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.processed", value: stats.processed)
|
|
56
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.failed", value: stats.failed)
|
|
57
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.scheduled_size", value: stats.scheduled_size)
|
|
58
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.retry_size", value: stats.retry_size)
|
|
59
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.dead_size", value: stats.dead_size)
|
|
60
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.workers_size", value: stats.workers_size)
|
|
61
|
+
|
|
62
|
+
::Sidekiq::Queue.all.each do |queue|
|
|
63
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.queue_depth.#{queue.name}", value: queue.size)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module SolidQueue
|
|
4
|
+
# Reports Solid Queue's own gauge-style state as infrastructure metrics on a periodic timer
|
|
5
|
+
# (see PeriodicPoller and Sidekiq::StatsReporter's own comment, the same shape). Only ever
|
|
6
|
+
# loaded when Solid Queue is already present (checked with defined? in Railtie).
|
|
7
|
+
#
|
|
8
|
+
# No job *duration*/*count* reporting here: Solid Queue jobs are ActiveJob jobs, already
|
|
9
|
+
# covered end to end by the perform.active_job subscription in Railtie regardless of which
|
|
10
|
+
# backend actually runs them, so there's nothing left for this integration to time.
|
|
11
|
+
#
|
|
12
|
+
# kind is Solid Queue's own attribute on a registered process (a plain String, "Worker" or
|
|
13
|
+
# "Dispatcher": confirmed directly against solid_queue's own Processes::Base#kind, which is
|
|
14
|
+
# literally self.class.name.demodulize, not assumed from its docs), so this reads it exactly
|
|
15
|
+
# as Solid Queue itself defines it rather than re-deriving it a different way.
|
|
16
|
+
class StatsReporter
|
|
17
|
+
def self.start(configuration: ForgeOpsTracker.configuration)
|
|
18
|
+
return unless configuration.track_performance && configuration.enabled?
|
|
19
|
+
|
|
20
|
+
PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def report
|
|
24
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.active_workers", value: ::SolidQueue::Process.where(kind: "Worker").count)
|
|
25
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.active_dispatchers", value: ::SolidQueue::Process.where(kind: "Dispatcher").count)
|
|
26
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.failed", value: ::SolidQueue::FailedExecution.count)
|
|
27
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.scheduled", value: ::SolidQueue::ScheduledExecution.count)
|
|
28
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.blocked", value: ::SolidQueue::BlockedExecution.count)
|
|
29
|
+
|
|
30
|
+
::SolidQueue::Queue.all.each do |queue|
|
|
31
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.queue_depth.#{queue.name}", value: queue.size)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|