forge_ops_tracker 0.5.0 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +72 -0
  3. data/README.md +214 -1
  4. data/lib/forge_ops_tracker/breadcrumb_buffer.rb +45 -0
  5. data/lib/forge_ops_tracker/client.rb +37 -0
  6. data/lib/forge_ops_tracker/configuration.rb +117 -0
  7. data/lib/forge_ops_tracker/error_subscriber.rb +28 -1
  8. data/lib/forge_ops_tracker/event_builder.rb +15 -6
  9. data/lib/forge_ops_tracker/failure_event_buffer.rb +73 -0
  10. data/lib/forge_ops_tracker/histogram_bucketer.rb +32 -0
  11. data/lib/forge_ops_tracker/infrastructure_metric_buffer.rb +86 -0
  12. data/lib/forge_ops_tracker/integrations/active_record_pool.rb +41 -0
  13. data/lib/forge_ops_tracker/integrations/net_http.rb +114 -0
  14. data/lib/forge_ops_tracker/integrations/puma.rb +55 -0
  15. data/lib/forge_ops_tracker/integrations/redis_client.rb +62 -0
  16. data/lib/forge_ops_tracker/integrations/sidekiq.rb +69 -0
  17. data/lib/forge_ops_tracker/integrations/solid_queue.rb +37 -0
  18. data/lib/forge_ops_tracker/metric_buffer.rb +97 -0
  19. data/lib/forge_ops_tracker/middleware/breadcrumb_context.rb +35 -0
  20. data/lib/forge_ops_tracker/middleware/span_tracing.rb +38 -0
  21. data/lib/forge_ops_tracker/middleware/user_context.rb +52 -0
  22. data/lib/forge_ops_tracker/performance_flusher.rb +129 -0
  23. data/lib/forge_ops_tracker/performance_instrumentation.rb +92 -0
  24. data/lib/forge_ops_tracker/periodic_poller.rb +50 -0
  25. data/lib/forge_ops_tracker/railtie.rb +272 -0
  26. data/lib/forge_ops_tracker/span_buffer.rb +65 -0
  27. data/lib/forge_ops_tracker/span_queue.rb +61 -0
  28. data/lib/forge_ops_tracker/version.rb +1 -1
  29. data/lib/forge_ops_tracker.rb +106 -0
  30. metadata +76 -1
@@ -0,0 +1,73 @@
1
+ require "thread"
2
+
3
+ module ForgeOpsTracker
4
+ # Collects individual job/dependency failures in-process (see Railtie's own perform.active_job
5
+ # subscription and Integrations::NetHTTP::Timing#request, the two places that actually call
6
+ # #record) and periodically flushes them as one batch, the same shape MetricBuffer already uses
7
+ # for the identical reason: a real failure's own detail (which job, what error) is exactly the
8
+ # kind of thing worth an accurate individual record of, not something to pre-aggregate away the
9
+ # way PerformanceFlusher's own count/sum/max buckets duration.
10
+ class FailureEventBuffer
11
+ def initialize(configuration, client: Client.new(configuration))
12
+ @configuration = configuration
13
+ @client = client
14
+ @mutex = Mutex.new
15
+ @entries = []
16
+ @thread = nil
17
+
18
+ # Flushes whatever's already buffered on a normal process exit, so the last partial window
19
+ # isn't silently dropped; same reasoning every other buffer/flusher in this gem documents.
20
+ at_exit { flush }
21
+ end
22
+
23
+ def record(kind:, transaction_name:, error_class: nil, error_message: nil)
24
+ ensure_worker_started
25
+
26
+ @mutex.synchronize do
27
+ @entries << {
28
+ kind: kind, transaction_name: transaction_name, error_class: error_class, error_message: error_message,
29
+ environment: configuration.environment.to_s, release: configuration.release,
30
+ occurred_at: Time.now.utc.iso8601
31
+ }
32
+ end
33
+ end
34
+
35
+ # Snapshots and resets the buffered entries, then delivers them as one batch. Same "a failed
36
+ # delivery keeps every entry for the next attempt" contract as MetricBuffer#flush.
37
+ def flush
38
+ snapshot = nil
39
+
40
+ @mutex.synchronize do
41
+ return if @entries.empty?
42
+ snapshot = @entries
43
+ end
44
+
45
+ return unless client.deliver_failure_events(snapshot)
46
+
47
+ @mutex.synchronize { @entries = [] }
48
+ end
49
+
50
+ private
51
+ attr_reader :configuration, :client
52
+
53
+ def ensure_worker_started
54
+ return if @thread&.alive?
55
+
56
+ @mutex.synchronize do
57
+ return if @thread&.alive?
58
+
59
+ @thread = Thread.new { run }
60
+ @thread.abort_on_exception = false
61
+ end
62
+ end
63
+
64
+ def run
65
+ loop do
66
+ sleep configuration.failure_event_flush_interval
67
+ flush
68
+ rescue StandardError => e
69
+ configuration.logger&.debug { "[ForgeOpsTracker] failure event flush thread error: #{e.class}: #{e.message}" }
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,32 @@
1
+ module ForgeOpsTracker
2
+ # Buckets a single duration into one of a fixed set of latency-range labels, the building block
3
+ # PerformanceFlusher uses to accumulate an approximate distribution (not just count/sum/max)
4
+ # alongside every [transaction_name, kind] bucket it already tallies. The server merges these
5
+ # counts across matching samples at read time and walks cumulative counts to approximate a
6
+ # percentile, the same "accurate to the bucket width, not exact" trade-off Prometheus's own
7
+ # histogram_quantile makes: this SDK does not store, or need, the raw duration list a true
8
+ # percentile would require.
9
+ #
10
+ # BOUNDARIES_MS is duplicated (not shared via a gem dependency) on the server side, in
11
+ # app/services/histogram_percentile.rb: the gem and the Rails app it reports to are separate
12
+ # deployables, the same "duplicated, not imported, so this package's own dependency graph stays
13
+ # independent" reasoning already applied elsewhere in this repo (e.g. the React Native SDK's own
14
+ # copy of the TypeScript SDK's PII scrubber). Change one, change the other, or a released gem
15
+ # version and the server version it talks to would silently disagree about what each bucket label
16
+ # means.
17
+ module HistogramBucketer
18
+ BOUNDARIES_MS = [ 50, 100, 250, 500, 1000, 2500, 5000, 10_000 ].freeze
19
+
20
+ # Returns the label (a String) of the smallest boundary duration_ms fits under, or "inf" for
21
+ # anything larger than the largest boundary. String, not Integer/Float: this travels as a JSON
22
+ # object key once flushed (see PerformanceFlusher#flush), and JSON object keys are always
23
+ # strings regardless of what Ruby type builds the Hash locally, so returning a String here
24
+ # keeps the in-process Hash's own keys identical to what actually goes over the wire, rather
25
+ # than looking different locally and then silently getting stringified only at serialization
26
+ # time.
27
+ def self.bucket_for(duration_ms)
28
+ boundary = BOUNDARIES_MS.find { |b| duration_ms <= b }
29
+ boundary ? boundary.to_s : "inf"
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,86 @@
1
+ require "thread"
2
+
3
+ module ForgeOpsTracker
4
+ # Collects individual ForgeOpsTracker.capture_infrastructure_metric readings in-process and
5
+ # periodically flushes them as one batch, same shape as MetricBuffer (a sibling class, not a
6
+ # shared base -- this gem keeps SessionFlusher/PerformanceFlusher as separate, independently-
7
+ # readable classes despite their own similar shape too, and these two follow that precedent).
8
+ # Its main real caller is a short-lived cron/script process that exits right after a handful of
9
+ # capture calls, so the at_exit flush below is what actually matters in practice; the periodic
10
+ # background-thread timer still runs for a long-lived process that chooses to call this
11
+ # continuously instead, but rarely gets the chance to fire in the cron case.
12
+ class InfrastructureMetricBuffer
13
+ # Once this many entries are buffered, further ones are dropped until a flush succeeds: a plan
14
+ # without the feature answers 403 on every flush, and an uncapped buffer would then grow for as
15
+ # long as the process lives. Dropping the newest rather than the oldest keeps the entries a flush
16
+ # is delivering at the front of the array, which is what makes removing exactly them exact.
17
+ MAX_ENTRIES = 1000
18
+
19
+ def initialize(configuration, client: Client.new(configuration))
20
+ @configuration = configuration
21
+ @client = client
22
+ @mutex = Mutex.new
23
+ @entries = []
24
+ @thread = nil
25
+
26
+ at_exit { flush }
27
+ end
28
+
29
+ def record(metric_name:, value:, hostname:)
30
+ return false unless keepable?(value)
31
+
32
+ ensure_worker_started
33
+
34
+ @mutex.synchronize do
35
+ return false if @entries.size >= MAX_ENTRIES
36
+
37
+ @entries << { metric_name: metric_name, value: value, hostname: hostname, recorded_at: Time.now.utc.iso8601 }
38
+ end
39
+ end
40
+
41
+ # Same reset-on-success/keep-on-failure contract as MetricBuffer#flush; see that class's own
42
+ # comment for the accepted race this shares with it.
43
+ def flush
44
+ snapshot = nil
45
+
46
+ @mutex.synchronize do
47
+ return if @entries.empty?
48
+ snapshot = @entries.dup
49
+ end
50
+
51
+ return unless client.deliver_infrastructure_metrics(snapshot)
52
+
53
+ # Exactly the entries just delivered: anything recorded while the request was in flight sits
54
+ # after them and stays for the next flush (resetting the whole array here lost it).
55
+ @mutex.synchronize { @entries.shift(snapshot.size) }
56
+ end
57
+
58
+ private
59
+ attr_reader :configuration, :client
60
+
61
+ # A NaN or infinite value is not valid JSON and would make the server reject the whole batch.
62
+ def keepable?(value)
63
+ value.is_a?(Numeric) && value.to_f.finite?
64
+ end
65
+
66
+ def ensure_worker_started
67
+ return if @thread&.alive?
68
+
69
+ @mutex.synchronize do
70
+ return if @thread&.alive?
71
+
72
+ @thread = Thread.new { run }
73
+ @thread.abort_on_exception = false
74
+ end
75
+ end
76
+
77
+ def run
78
+ loop do
79
+ sleep configuration.infrastructure_metric_flush_interval
80
+ flush
81
+ rescue StandardError => e
82
+ configuration.logger&.debug { "[ForgeOpsTracker] infrastructure metric flush thread error: #{e.class}: #{e.message}" }
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,41 @@
1
+ module ForgeOpsTracker
2
+ module Integrations
3
+ module ActiveRecordPool
4
+ # Reports this process's own ActiveRecord connection pool utilization as infrastructure
5
+ # metrics on a periodic timer, the same shape as Sidekiq::StatsReporter/SolidQueue::StatsReporter
6
+ # (see either's own comment), and, like both of those, only ever loaded when ActiveRecord is
7
+ # actually present (checked with defined? in Railtie, not assumed).
8
+ #
9
+ # Scoped to the current process's own primary connection pool only, via the bare
10
+ # ActiveRecord::Base.connection_pool this process is actually using: a known v1 limitation,
11
+ # not an oversight, for an app with multiple databases (ActiveRecord::Base.connection_pool is
12
+ # only ever one of potentially several pools such an app has configured). Reporting every
13
+ # configured pool would need ActiveRecord::Base.connection_handler's own registry, a genuinely
14
+ # separate piece of work; this integration reports what a single-database app (the common
15
+ # case, and the only case this gem's own test suite/README examples ever set up) actually has.
16
+ class StatsReporter
17
+ def self.start(configuration: ForgeOpsTracker.configuration)
18
+ return unless configuration.track_performance && configuration.enabled?
19
+
20
+ PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
21
+ end
22
+
23
+ def report
24
+ stat = ::ActiveRecord::Base.connection_pool.stat
25
+ size = stat[:size].to_i
26
+ busy = stat[:busy].to_i
27
+
28
+ # 0/0 (a pool nobody's ever checked a connection out of yet, or, in practice, never) reads
29
+ # as 0% utilized, not NaN/an error: a pool that's never been used is exactly as "not under
30
+ # pressure" as a pool with plenty of headroom, not a value worth surfacing as broken.
31
+ utilization_pct = size.zero? ? 0.0 : (100.0 * busy / size).round(1)
32
+
33
+ ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.utilization_pct", value: utilization_pct)
34
+ ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.size", value: size)
35
+ ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.busy", value: busy)
36
+ ForgeOpsTracker.capture_infrastructure_metric("active_record_pool.waiting", value: stat[:waiting].to_i)
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,114 @@
1
+ module ForgeOpsTracker
2
+ module Integrations
3
+ module NetHTTP
4
+ # Times every outbound Net::HTTP call, applied once via Net::HTTP.prepend(Timing) from
5
+ # Railtie. Ruby's standard library has no ActiveSupport::Notifications instrumentation of
6
+ # its own for outbound HTTP the way it does for ActiveRecord/ActionController, so wrapping
7
+ # #request directly is the only hook available; most other HTTP client libraries in the Ruby
8
+ # ecosystem (Faraday's own net_http adapter, HTTParty, RestClient) ultimately call through
9
+ # Net::HTTP too, so this covers those as a side effect without needing a wrapper per library.
10
+ #
11
+ # transaction_name is "<HTTP method> <host>", not the full URL: a request's own path or
12
+ # query string could carry an id or a token, the same low-cardinality/no-secrets-in-a-label
13
+ # reasoning every other transaction_name in this system already follows.
14
+ #
15
+ # Known, honest gap, confirmed directly against Net::HTTP's own source and real behavior, not
16
+ # assumed: Net::HTTP.get_response and the Net::HTTP.start { |http| ... } block form both open
17
+ # the TCP connection inside #start, *before* the block (and so before #request, the only
18
+ # method this wraps) ever runs, so a connection-refused/DNS-failure error raised during that
19
+ # setup is structurally invisible here, for timing, breadcrumbs, and the failure detection
20
+ # below alike (a pre-existing limitation this round's own test suite is what actually
21
+ # surfaced, not something newly introduced). Net::HTTP.new(...).request(...), the other
22
+ # common calling convention (no explicit #start at all), is different and unaffected:
23
+ # confirmed directly that Net::HTTP connects lazily, inside #request itself, the first time
24
+ # it's called on a not-yet-started instance, so that path's own connection failures are
25
+ # caught exactly like any other #request failure.
26
+ module Timing
27
+ # The guard (return super unless ...) skips timing, but the ensure block below still runs
28
+ # on every exit path regardless, same as any ensure; start staying nil is what actually
29
+ # skips recording there, not a second guard duplicating this one.
30
+ def request(req, body = nil, &block)
31
+ configuration = ForgeOpsTracker.configuration
32
+ return super unless (configuration.track_performance || configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
33
+
34
+ start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
35
+ response = super
36
+ rescue StandardError => e
37
+ # Only track_failures below needs the actual exception object (for error_class/
38
+ # error_message); everything else keys off response staying nil, already true on this
39
+ # path without capturing anything. re-raised unchanged: this gem must never swallow or
40
+ # alter what the host app's own call to Net::HTTP#request would have raised.
41
+ raised_error = e
42
+ raise
43
+ ensure
44
+ if start
45
+ duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
46
+ # Computed once, unconditionally, not separately inside each flag-gated block below:
47
+ # both the breadcrumb and track_failures sections need it, and a value only ever
48
+ # assigned inside one conditional branch but read from another relies on Ruby's own
49
+ # "a local is declared the moment its assignment is lexically parsed, regardless of
50
+ # whether that branch actually ran" scoping, a real but easy-to-break-on-reorder
51
+ # subtlety not worth the risk here.
52
+ status = response&.code&.to_i
53
+ if configuration.track_performance
54
+ Railtie.performance_flusher.record(transaction_name: "#{req.method} #{address}", duration_ms: duration_ms, kind: "http")
55
+ end
56
+ # The same "method + host, never the full path/query string" reasoning
57
+ # transaction_name above already follows: a URL's own path or query could carry an id
58
+ # or a token, and this is a breadcrumb message, not a value this gem controls the
59
+ # cardinality of. Mirrors sdks/typescript's own fetch/xhr breadcrumbs: category
60
+ # "http", status + duration_ms in data, level "warning" for a non-2xx/3xx response.
61
+ if configuration.track_breadcrumbs
62
+ # No status at all means the request itself raised (a timeout, a connection
63
+ # refused, ...) before a response ever came back: worth "error", not the misleading
64
+ # "info" a bare status-based check would default a nil status to, the same "the
65
+ # request truly failed" case sdks/typescript's own fetch breadcrumb catch block
66
+ # marks "error" for.
67
+ level = if status.nil?
68
+ "error"
69
+ elsif status >= 400
70
+ "warning"
71
+ else
72
+ "info"
73
+ end
74
+ ForgeOpsTracker.add_breadcrumb(
75
+ "#{req.method} #{address}",
76
+ category: "http",
77
+ level: level,
78
+ data: { status: status, duration_ms: duration_ms.round(1) }.compact
79
+ )
80
+ end
81
+ # Independent of track_performance/track_breadcrumbs above, same "several genuinely
82
+ # independent mechanisms recorded from the same instrumentation point" pattern this
83
+ # exact call site already established for breadcrumbs. A failure is the request
84
+ # raising, or a 5xx: not a 4xx, the identical distinction the breadcrumb level just
85
+ # above already draws between "the dependency is unhealthy" (worth "error"/failure)
86
+ # and "the dependency responded, just not with success" (only "warning").
87
+ if configuration.track_failures && (status.nil? || status >= 500)
88
+ Railtie.failure_event_buffer.record(
89
+ kind: "dependency", transaction_name: "#{req.method} #{address}",
90
+ error_class: raised_error&.class&.name, error_message: raised_error&.message
91
+ )
92
+ end
93
+ # A leaf span, the same "parented onto whatever's currently open, never touches the
94
+ # stack itself" shape the sql.active_record subscription's own span recording already
95
+ # takes: an outbound HTTP call has no children of its own to nest anything under
96
+ # either.
97
+ if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
98
+ # duration_ms itself is exact (computed from the monotonic clock above); started_at
99
+ # here is only derived from it (now minus duration), an approximation off the wall
100
+ # clock rather than a second real timestamp captured at the top of this method - a
101
+ # few milliseconds of drift on where the bar starts in a waterfall, never on how long
102
+ # it actually took.
103
+ start_time = ::Time.now.utc - (duration_ms / 1000.0)
104
+ buffer.record(
105
+ span_id: SecureRandom.hex(8), name: "#{req.method} #{address}", kind: "http",
106
+ started_at: start_time, duration_ms: duration_ms, data: { status: status }.compact
107
+ )
108
+ end
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,55 @@
1
+ module ForgeOpsTracker
2
+ module Integrations
3
+ module Puma
4
+ # Reports Puma's own backlog/running-threads/pool-capacity as gauge-style infrastructure
5
+ # metrics on a periodic timer (see PeriodicPoller). Only ever loaded when Puma is already
6
+ # present (checked with defined? in Railtie).
7
+ #
8
+ # Puma.stats_hash's own shape genuinely differs by deployment mode, confirmed directly
9
+ # against Puma 6.6.1's real source rather than assumed: single-mode (Puma::Single, no
10
+ # `workers` config) has backlog/running/pool_capacity at the top level (Server#stats merged
11
+ # with ThreadPool#stats); clustered mode (Puma::Cluster) has none of those at the top level
12
+ # at all, only a worker_status array, one entry per forked worker, each carrying its own
13
+ # last_status hash with the identical three keys (parsed from that worker's own periodic
14
+ # IPC ping, see Puma::Cluster::WorkerHandle's own STATUS_PATTERN), summed across workers
15
+ # here for one whole-process reading, rather than reported per worker, since ForgeOps'
16
+ # infrastructure dataset already aggregates by hostname, not by worker index. Puma.stats_object
17
+ # is set unconditionally by Puma::Launcher on every normal boot (not just when a control/
18
+ # stats app is configured), so Puma.stats_hash is populated by the time this ever runs; still
19
+ # guarded against nil/an unrecognized shape, both because that wiring could change in a
20
+ # future Puma version and because this must never be the thing that crashes a poll tick.
21
+ class StatsReporter
22
+ def self.start(configuration: ForgeOpsTracker.configuration)
23
+ return unless configuration.track_performance && configuration.enabled?
24
+
25
+ PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
26
+ end
27
+
28
+ def report
29
+ stats = ::Puma.stats_hash
30
+ return unless stats
31
+
32
+ if stats[:backlog] && stats[:running]
33
+ report_metrics(stats[:backlog], stats[:running], stats[:pool_capacity])
34
+ elsif stats[:worker_status]
35
+ statuses = stats[:worker_status].filter_map { |worker| worker[:last_status] }.reject(&:empty?)
36
+ return if statuses.empty?
37
+
38
+ report_metrics(
39
+ statuses.sum { |s| s[:backlog] || 0 },
40
+ statuses.sum { |s| s[:running] || 0 },
41
+ statuses.sum { |s| s[:pool_capacity] || 0 }
42
+ )
43
+ end
44
+ end
45
+
46
+ private
47
+ def report_metrics(backlog, running, pool_capacity)
48
+ ForgeOpsTracker.capture_infrastructure_metric("puma.backlog", value: backlog)
49
+ ForgeOpsTracker.capture_infrastructure_metric("puma.running_threads", value: running)
50
+ ForgeOpsTracker.capture_infrastructure_metric("puma.pool_capacity", value: pool_capacity)
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,62 @@
1
+ module ForgeOpsTracker
2
+ module Integrations
3
+ module RedisClientTiming
4
+ # Times every Redis command, applied once via RedisClient.prepend(Timing) from Railtie, the
5
+ # same Module#prepend + super shape Integrations::NetHTTP::Timing already uses for the
6
+ # identical reason: neither Ruby's Redis ecosystem nor RedisClient itself fires an
7
+ # ActiveSupport::Notifications event of its own, so wrapping #call directly is the only hook
8
+ # available. Targets RedisClient specifically (the modern, actively-maintained low-level
9
+ # client both the current redis gem (5.x+) and standalone RedisClient users go through), not
10
+ # the older redis-rb 4.x Redis::Client - confirmed directly (not assumed) that redis 5.x no
11
+ # longer defines Redis::Client at all, so a hook there would silently do nothing on any
12
+ # currently-installed version of the gem.
13
+ #
14
+ # Unlike controller/query timing, this is span-only: there's no "redis" PerformanceSample
15
+ # kind for this to feed into the aggregate Performance page (that page has no Redis panel at
16
+ # all yet), so this only ever contributes to a captured trace's own waterfall, plus a
17
+ # breadcrumb and a dependency failure record, the same two other independent mechanisms
18
+ # NetHTTP::Timing's own call site already establishes.
19
+ #
20
+ # name is "Redis <COMMAND>" (the command verb alone, upcased: GET, SET, LPUSH, ...), never
21
+ # the key or any argument - the same low-cardinality, no-secrets-in-a-label reasoning every
22
+ # other transaction_name/span name in this gem already follows; a Redis key can easily carry
23
+ # a customer's own id or other sensitive value.
24
+ def call(*command, **kwargs)
25
+ configuration = ForgeOpsTracker.configuration
26
+ return super unless (configuration.track_breadcrumbs || configuration.track_failures || configuration.track_tracing) && configuration.enabled?
27
+
28
+ name = "Redis #{command.first.to_s.upcase}"
29
+ start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
30
+ result = super
31
+ rescue StandardError => e
32
+ raised_error = e
33
+ raise
34
+ ensure
35
+ if start
36
+ duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
37
+
38
+ if configuration.track_breadcrumbs
39
+ ForgeOpsTracker.add_breadcrumb(
40
+ name, category: "redis", level: raised_error ? "error" : "info",
41
+ data: { duration_ms: duration_ms.round(1) }
42
+ )
43
+ end
44
+
45
+ if configuration.track_failures && raised_error
46
+ Railtie.failure_event_buffer.record(
47
+ kind: "dependency", transaction_name: name,
48
+ error_class: raised_error.class.name, error_message: raised_error.message
49
+ )
50
+ end
51
+
52
+ if configuration.track_tracing && (buffer = Thread.current[:forge_ops_tracker_spans])
53
+ buffer.record(
54
+ span_id: SecureRandom.hex(8), name: name, kind: "redis",
55
+ started_at: ::Time.now.utc - (duration_ms / 1000.0), duration_ms: duration_ms
56
+ )
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,69 @@
1
+ # Sidekiq::Stats/Sidekiq::Queue (used by StatsReporter below) live in sidekiq/api, not
2
+ # autoloaded by a bare "sidekiq" require; a real Sidekiq server process almost always already has
3
+ # this loaded some other way, but requiring it explicitly here doesn't depend on that happening to
4
+ # be true.
5
+ require "sidekiq/api"
6
+
7
+ module ForgeOpsTracker
8
+ module Integrations
9
+ module Sidekiq
10
+ # Times a raw (non-ActiveJob) Sidekiq::Worker's own #perform. Registered via
11
+ # Sidekiq.configure_server { |c| c.server_middleware { |chain| chain.add ... } } (see
12
+ # Railtie); only ever loaded when Sidekiq is already present (checked with defined? there),
13
+ # matching every other optional integration in this gem.
14
+ #
15
+ # Skips (yields, but doesn't record) any job whose worker is ActiveJob's own Sidekiq wrapper:
16
+ # see PerformanceInstrumentation.sidekiq_worker_is_active_job_wrapper?'s own comment for why
17
+ # that job is already counted by the perform.active_job subscription instead, and would be
18
+ # double-counted here otherwise.
19
+ class Middleware
20
+ include ::Sidekiq::ServerMiddleware
21
+
22
+ def initialize(configuration: ForgeOpsTracker.configuration, flusher: Railtie.performance_flusher)
23
+ @configuration = configuration
24
+ @flusher = flusher
25
+ end
26
+
27
+ def call(job_instance, _msg, _queue)
28
+ skip = !@configuration.track_performance || !@configuration.enabled? ||
29
+ PerformanceInstrumentation.sidekiq_worker_is_active_job_wrapper?(job_instance)
30
+ start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) unless skip
31
+
32
+ yield
33
+ ensure
34
+ unless skip
35
+ duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
36
+ @flusher.record(transaction_name: job_instance.class.name, duration_ms: duration_ms, kind: "job")
37
+ end
38
+ end
39
+ end
40
+
41
+ # Reports Sidekiq's own aggregate stats (Sidekiq::Stats, Sidekiq::Queue) as gauge-style
42
+ # infrastructure metrics on a periodic timer (see PeriodicPoller), via the same public
43
+ # ForgeOpsTracker.capture_infrastructure_metric every customer's own manual infrastructure-
44
+ # monitoring script already uses; no new ingestion endpoint or dashboard dataset needed.
45
+ class StatsReporter
46
+ def self.start(configuration: ForgeOpsTracker.configuration)
47
+ return unless configuration.track_performance && configuration.enabled?
48
+
49
+ PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
50
+ end
51
+
52
+ def report
53
+ stats = ::Sidekiq::Stats.new
54
+
55
+ ForgeOpsTracker.capture_infrastructure_metric("sidekiq.processed", value: stats.processed)
56
+ ForgeOpsTracker.capture_infrastructure_metric("sidekiq.failed", value: stats.failed)
57
+ ForgeOpsTracker.capture_infrastructure_metric("sidekiq.scheduled_size", value: stats.scheduled_size)
58
+ ForgeOpsTracker.capture_infrastructure_metric("sidekiq.retry_size", value: stats.retry_size)
59
+ ForgeOpsTracker.capture_infrastructure_metric("sidekiq.dead_size", value: stats.dead_size)
60
+ ForgeOpsTracker.capture_infrastructure_metric("sidekiq.workers_size", value: stats.workers_size)
61
+
62
+ ::Sidekiq::Queue.all.each do |queue|
63
+ ForgeOpsTracker.capture_infrastructure_metric("sidekiq.queue_depth.#{queue.name}", value: queue.size)
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,37 @@
1
+ module ForgeOpsTracker
2
+ module Integrations
3
+ module SolidQueue
4
+ # Reports Solid Queue's own gauge-style state as infrastructure metrics on a periodic timer
5
+ # (see PeriodicPoller and Sidekiq::StatsReporter's own comment, the same shape). Only ever
6
+ # loaded when Solid Queue is already present (checked with defined? in Railtie).
7
+ #
8
+ # No job *duration*/*count* reporting here: Solid Queue jobs are ActiveJob jobs, already
9
+ # covered end to end by the perform.active_job subscription in Railtie regardless of which
10
+ # backend actually runs them, so there's nothing left for this integration to time.
11
+ #
12
+ # kind is Solid Queue's own attribute on a registered process (a plain String, "Worker" or
13
+ # "Dispatcher": confirmed directly against solid_queue's own Processes::Base#kind, which is
14
+ # literally self.class.name.demodulize, not assumed from its docs), so this reads it exactly
15
+ # as Solid Queue itself defines it rather than re-deriving it a different way.
16
+ class StatsReporter
17
+ def self.start(configuration: ForgeOpsTracker.configuration)
18
+ return unless configuration.track_performance && configuration.enabled?
19
+
20
+ PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
21
+ end
22
+
23
+ def report
24
+ ForgeOpsTracker.capture_infrastructure_metric("solid_queue.active_workers", value: ::SolidQueue::Process.where(kind: "Worker").count)
25
+ ForgeOpsTracker.capture_infrastructure_metric("solid_queue.active_dispatchers", value: ::SolidQueue::Process.where(kind: "Dispatcher").count)
26
+ ForgeOpsTracker.capture_infrastructure_metric("solid_queue.failed", value: ::SolidQueue::FailedExecution.count)
27
+ ForgeOpsTracker.capture_infrastructure_metric("solid_queue.scheduled", value: ::SolidQueue::ScheduledExecution.count)
28
+ ForgeOpsTracker.capture_infrastructure_metric("solid_queue.blocked", value: ::SolidQueue::BlockedExecution.count)
29
+
30
+ ::SolidQueue::Queue.all.each do |queue|
31
+ ForgeOpsTracker.capture_infrastructure_metric("solid_queue.queue_depth.#{queue.name}", value: queue.size)
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end