forge_ops_tracker 0.2.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +49 -0
- data/README.md +191 -10
- data/lib/forge_ops_tracker/client.rb +40 -10
- data/lib/forge_ops_tracker/configuration.rb +89 -3
- data/lib/forge_ops_tracker/delivery_queue.rb +1 -1
- data/lib/forge_ops_tracker/error_subscriber.rb +6 -1
- data/lib/forge_ops_tracker/event_builder.rb +64 -8
- data/lib/forge_ops_tracker/infrastructure_metric_buffer.rb +69 -0
- data/lib/forge_ops_tracker/integrations/net_http.rb +32 -0
- data/lib/forge_ops_tracker/integrations/puma.rb +55 -0
- data/lib/forge_ops_tracker/integrations/sidekiq.rb +69 -0
- data/lib/forge_ops_tracker/integrations/solid_queue.rb +37 -0
- data/lib/forge_ops_tracker/metric_buffer.rb +80 -0
- data/lib/forge_ops_tracker/middleware/session_tracking.rb +38 -0
- data/lib/forge_ops_tracker/middleware/user_context.rb +52 -0
- data/lib/forge_ops_tracker/performance_flusher.rb +101 -0
- data/lib/forge_ops_tracker/performance_instrumentation.rb +60 -0
- data/lib/forge_ops_tracker/periodic_poller.rb +50 -0
- data/lib/forge_ops_tracker/pii_scrubber.rb +2 -2
- data/lib/forge_ops_tracker/railtie.rb +123 -0
- data/lib/forge_ops_tracker/session_flusher.rb +85 -0
- data/lib/forge_ops_tracker/version.rb +1 -1
- data/lib/forge_ops_tracker.rb +52 -0
- metadata +58 -4
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
require "thread"
|
|
2
|
+
|
|
3
|
+
module ForgeOpsTracker
|
|
4
|
+
# Collects individual ForgeOpsTracker.capture_infrastructure_metric readings in-process and
|
|
5
|
+
# periodically flushes them as one batch, same shape as MetricBuffer (a sibling class, not a
|
|
6
|
+
# shared base -- this gem keeps SessionFlusher/PerformanceFlusher as separate, independently-
|
|
7
|
+
# readable classes despite their own similar shape too, and these two follow that precedent).
|
|
8
|
+
# Its main real caller is a short-lived cron/script process that exits right after a handful of
|
|
9
|
+
# capture calls, so the at_exit flush below is what actually matters in practice; the periodic
|
|
10
|
+
# background-thread timer still runs for a long-lived process that chooses to call this
|
|
11
|
+
# continuously instead, but rarely gets the chance to fire in the cron case.
|
|
12
|
+
class InfrastructureMetricBuffer
|
|
13
|
+
def initialize(configuration, client: Client.new(configuration))
|
|
14
|
+
@configuration = configuration
|
|
15
|
+
@client = client
|
|
16
|
+
@mutex = Mutex.new
|
|
17
|
+
@entries = []
|
|
18
|
+
@thread = nil
|
|
19
|
+
|
|
20
|
+
at_exit { flush }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def record(metric_name:, value:, hostname:)
|
|
24
|
+
ensure_worker_started
|
|
25
|
+
|
|
26
|
+
@mutex.synchronize do
|
|
27
|
+
@entries << { metric_name: metric_name, value: value, hostname: hostname, recorded_at: Time.now.utc.iso8601 }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Same reset-on-success/keep-on-failure contract as MetricBuffer#flush; see that class's own
|
|
32
|
+
# comment for the accepted race this shares with it.
|
|
33
|
+
def flush
|
|
34
|
+
snapshot = nil
|
|
35
|
+
|
|
36
|
+
@mutex.synchronize do
|
|
37
|
+
return if @entries.empty?
|
|
38
|
+
snapshot = @entries
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
return unless client.deliver_infrastructure_metrics(snapshot)
|
|
42
|
+
|
|
43
|
+
@mutex.synchronize { @entries = [] }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
attr_reader :configuration, :client
|
|
48
|
+
|
|
49
|
+
def ensure_worker_started
|
|
50
|
+
return if @thread&.alive?
|
|
51
|
+
|
|
52
|
+
@mutex.synchronize do
|
|
53
|
+
return if @thread&.alive?
|
|
54
|
+
|
|
55
|
+
@thread = Thread.new { run }
|
|
56
|
+
@thread.abort_on_exception = false
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def run
|
|
61
|
+
loop do
|
|
62
|
+
sleep configuration.infrastructure_metric_flush_interval
|
|
63
|
+
flush
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
configuration.logger&.debug { "[ForgeOpsTracker] infrastructure metric flush thread error: #{e.class}: #{e.message}" }
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module NetHTTP
|
|
4
|
+
# Times every outbound Net::HTTP call, applied once via Net::HTTP.prepend(Timing) from
|
|
5
|
+
# Railtie. Ruby's standard library has no ActiveSupport::Notifications instrumentation of
|
|
6
|
+
# its own for outbound HTTP the way it does for ActiveRecord/ActionController, so wrapping
|
|
7
|
+
# #request directly is the only hook available; most other HTTP client libraries in the Ruby
|
|
8
|
+
# ecosystem (Faraday's own net_http adapter, HTTParty, RestClient) ultimately call through
|
|
9
|
+
# Net::HTTP too, so this covers those as a side effect without needing a wrapper per library.
|
|
10
|
+
#
|
|
11
|
+
# transaction_name is "<HTTP method> <host>", not the full URL: a request's own path or
|
|
12
|
+
# query string could carry an id or a token, the same low-cardinality/no-secrets-in-a-label
|
|
13
|
+
# reasoning every other transaction_name in this system already follows.
|
|
14
|
+
module Timing
|
|
15
|
+
# The guard (return super unless ...) skips timing, but the ensure block below still runs
|
|
16
|
+
# on every exit path regardless, same as any ensure; start staying nil is what actually
|
|
17
|
+
# skips recording there, not a second guard duplicating this one.
|
|
18
|
+
def request(req, body = nil, &block)
|
|
19
|
+
return super unless ForgeOpsTracker.configuration.track_performance && ForgeOpsTracker.configuration.enabled?
|
|
20
|
+
|
|
21
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
22
|
+
super
|
|
23
|
+
ensure
|
|
24
|
+
if start
|
|
25
|
+
duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
|
|
26
|
+
Railtie.performance_flusher.record(transaction_name: "#{req.method} #{address}", duration_ms: duration_ms, kind: "http")
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module Puma
|
|
4
|
+
# Reports Puma's own backlog/running-threads/pool-capacity as gauge-style infrastructure
|
|
5
|
+
# metrics on a periodic timer (see PeriodicPoller). Only ever loaded when Puma is already
|
|
6
|
+
# present (checked with defined? in Railtie).
|
|
7
|
+
#
|
|
8
|
+
# Puma.stats_hash's own shape genuinely differs by deployment mode, confirmed directly
|
|
9
|
+
# against Puma 6.6.1's real source rather than assumed: single-mode (Puma::Single, no
|
|
10
|
+
# `workers` config) has backlog/running/pool_capacity at the top level (Server#stats merged
|
|
11
|
+
# with ThreadPool#stats); clustered mode (Puma::Cluster) has none of those at the top level
|
|
12
|
+
# at all, only a worker_status array, one entry per forked worker, each carrying its own
|
|
13
|
+
# last_status hash with the identical three keys (parsed from that worker's own periodic
|
|
14
|
+
# IPC ping, see Puma::Cluster::WorkerHandle's own STATUS_PATTERN), summed across workers
|
|
15
|
+
# here for one whole-process reading, rather than reported per worker, since ForgeOps'
|
|
16
|
+
# infrastructure dataset already aggregates by hostname, not by worker index. Puma.stats_object
|
|
17
|
+
# is set unconditionally by Puma::Launcher on every normal boot (not just when a control/
|
|
18
|
+
# stats app is configured), so Puma.stats_hash is populated by the time this ever runs; still
|
|
19
|
+
# guarded against nil/an unrecognized shape, both because that wiring could change in a
|
|
20
|
+
# future Puma version and because this must never be the thing that crashes a poll tick.
|
|
21
|
+
class StatsReporter
|
|
22
|
+
def self.start(configuration: ForgeOpsTracker.configuration)
|
|
23
|
+
return unless configuration.track_performance && configuration.enabled?
|
|
24
|
+
|
|
25
|
+
PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def report
|
|
29
|
+
stats = ::Puma.stats_hash
|
|
30
|
+
return unless stats
|
|
31
|
+
|
|
32
|
+
if stats[:backlog] && stats[:running]
|
|
33
|
+
report_metrics(stats[:backlog], stats[:running], stats[:pool_capacity])
|
|
34
|
+
elsif stats[:worker_status]
|
|
35
|
+
statuses = stats[:worker_status].filter_map { |worker| worker[:last_status] }.reject(&:empty?)
|
|
36
|
+
return if statuses.empty?
|
|
37
|
+
|
|
38
|
+
report_metrics(
|
|
39
|
+
statuses.sum { |s| s[:backlog] || 0 },
|
|
40
|
+
statuses.sum { |s| s[:running] || 0 },
|
|
41
|
+
statuses.sum { |s| s[:pool_capacity] || 0 }
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
def report_metrics(backlog, running, pool_capacity)
|
|
48
|
+
ForgeOpsTracker.capture_infrastructure_metric("puma.backlog", value: backlog)
|
|
49
|
+
ForgeOpsTracker.capture_infrastructure_metric("puma.running_threads", value: running)
|
|
50
|
+
ForgeOpsTracker.capture_infrastructure_metric("puma.pool_capacity", value: pool_capacity)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Sidekiq::Stats/Sidekiq::Queue (used by StatsReporter below) live in sidekiq/api, not
|
|
2
|
+
# autoloaded by a bare "sidekiq" require; a real Sidekiq server process almost always already has
|
|
3
|
+
# this loaded some other way, but requiring it explicitly here doesn't depend on that happening to
|
|
4
|
+
# be true.
|
|
5
|
+
require "sidekiq/api"
|
|
6
|
+
|
|
7
|
+
module ForgeOpsTracker
|
|
8
|
+
module Integrations
|
|
9
|
+
module Sidekiq
|
|
10
|
+
# Times a raw (non-ActiveJob) Sidekiq::Worker's own #perform. Registered via
|
|
11
|
+
# Sidekiq.configure_server { |c| c.server_middleware { |chain| chain.add ... } } (see
|
|
12
|
+
# Railtie); only ever loaded when Sidekiq is already present (checked with defined? there),
|
|
13
|
+
# matching every other optional integration in this gem.
|
|
14
|
+
#
|
|
15
|
+
# Skips (yields, but doesn't record) any job whose worker is ActiveJob's own Sidekiq wrapper:
|
|
16
|
+
# see PerformanceInstrumentation.sidekiq_worker_is_active_job_wrapper?'s own comment for why
|
|
17
|
+
# that job is already counted by the perform.active_job subscription instead, and would be
|
|
18
|
+
# double-counted here otherwise.
|
|
19
|
+
class Middleware
|
|
20
|
+
include ::Sidekiq::ServerMiddleware
|
|
21
|
+
|
|
22
|
+
def initialize(configuration: ForgeOpsTracker.configuration, flusher: Railtie.performance_flusher)
|
|
23
|
+
@configuration = configuration
|
|
24
|
+
@flusher = flusher
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def call(job_instance, _msg, _queue)
|
|
28
|
+
skip = !@configuration.track_performance || !@configuration.enabled? ||
|
|
29
|
+
PerformanceInstrumentation.sidekiq_worker_is_active_job_wrapper?(job_instance)
|
|
30
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) unless skip
|
|
31
|
+
|
|
32
|
+
yield
|
|
33
|
+
ensure
|
|
34
|
+
unless skip
|
|
35
|
+
duration_ms = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start) * 1000.0
|
|
36
|
+
@flusher.record(transaction_name: job_instance.class.name, duration_ms: duration_ms, kind: "job")
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Reports Sidekiq's own aggregate stats (Sidekiq::Stats, Sidekiq::Queue) as gauge-style
|
|
42
|
+
# infrastructure metrics on a periodic timer (see PeriodicPoller), via the same public
|
|
43
|
+
# ForgeOpsTracker.capture_infrastructure_metric every customer's own manual infrastructure-
|
|
44
|
+
# monitoring script already uses; no new ingestion endpoint or dashboard dataset needed.
|
|
45
|
+
class StatsReporter
|
|
46
|
+
def self.start(configuration: ForgeOpsTracker.configuration)
|
|
47
|
+
return unless configuration.track_performance && configuration.enabled?
|
|
48
|
+
|
|
49
|
+
PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def report
|
|
53
|
+
stats = ::Sidekiq::Stats.new
|
|
54
|
+
|
|
55
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.processed", value: stats.processed)
|
|
56
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.failed", value: stats.failed)
|
|
57
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.scheduled_size", value: stats.scheduled_size)
|
|
58
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.retry_size", value: stats.retry_size)
|
|
59
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.dead_size", value: stats.dead_size)
|
|
60
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.workers_size", value: stats.workers_size)
|
|
61
|
+
|
|
62
|
+
::Sidekiq::Queue.all.each do |queue|
|
|
63
|
+
ForgeOpsTracker.capture_infrastructure_metric("sidekiq.queue_depth.#{queue.name}", value: queue.size)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Integrations
|
|
3
|
+
module SolidQueue
|
|
4
|
+
# Reports Solid Queue's own gauge-style state as infrastructure metrics on a periodic timer
|
|
5
|
+
# (see PeriodicPoller and Sidekiq::StatsReporter's own comment, the same shape). Only ever
|
|
6
|
+
# loaded when Solid Queue is already present (checked with defined? in Railtie).
|
|
7
|
+
#
|
|
8
|
+
# No job *duration*/*count* reporting here: Solid Queue jobs are ActiveJob jobs, already
|
|
9
|
+
# covered end to end by the perform.active_job subscription in Railtie regardless of which
|
|
10
|
+
# backend actually runs them, so there's nothing left for this integration to time.
|
|
11
|
+
#
|
|
12
|
+
# kind is Solid Queue's own attribute on a registered process (a plain String, "Worker" or
|
|
13
|
+
# "Dispatcher": confirmed directly against solid_queue's own Processes::Base#kind, which is
|
|
14
|
+
# literally self.class.name.demodulize, not assumed from its docs), so this reads it exactly
|
|
15
|
+
# as Solid Queue itself defines it rather than re-deriving it a different way.
|
|
16
|
+
class StatsReporter
|
|
17
|
+
def self.start(configuration: ForgeOpsTracker.configuration)
|
|
18
|
+
return unless configuration.track_performance && configuration.enabled?
|
|
19
|
+
|
|
20
|
+
PeriodicPoller.start(configuration.gauge_poll_interval, logger: configuration.logger) { new.report }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def report
|
|
24
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.active_workers", value: ::SolidQueue::Process.where(kind: "Worker").count)
|
|
25
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.active_dispatchers", value: ::SolidQueue::Process.where(kind: "Dispatcher").count)
|
|
26
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.failed", value: ::SolidQueue::FailedExecution.count)
|
|
27
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.scheduled", value: ::SolidQueue::ScheduledExecution.count)
|
|
28
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.blocked", value: ::SolidQueue::BlockedExecution.count)
|
|
29
|
+
|
|
30
|
+
::SolidQueue::Queue.all.each do |queue|
|
|
31
|
+
ForgeOpsTracker.capture_infrastructure_metric("solid_queue.queue_depth.#{queue.name}", value: queue.size)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
require "thread"
|
|
2
|
+
|
|
3
|
+
module ForgeOpsTracker
|
|
4
|
+
# Collects individual ForgeOpsTracker.capture_metric calls in-process and periodically flushes
|
|
5
|
+
# them as one batch, rather than one network call per capture -- the same "lazily start a
|
|
6
|
+
# background thread on first use" pattern SessionFlusher/PerformanceFlusher already use, so each
|
|
7
|
+
# forked Puma/Passenger worker gets its own fresh thread instead of inheriting a dead one across
|
|
8
|
+
# fork. Unlike those two, this collects a *list* of individually-meaningful entries rather than
|
|
9
|
+
# summing them into buckets: a customer's own signup or payment is exactly the kind of thing
|
|
10
|
+
# they'll want a genuinely accurate count/sum of later, not something to pre-aggregate away
|
|
11
|
+
# client-side, so CustomMetricsController stores one row per entry as-is.
|
|
12
|
+
class MetricBuffer
|
|
13
|
+
def initialize(configuration, client: Client.new(configuration))
|
|
14
|
+
@configuration = configuration
|
|
15
|
+
@client = client
|
|
16
|
+
@mutex = Mutex.new
|
|
17
|
+
@entries = []
|
|
18
|
+
@thread = nil
|
|
19
|
+
|
|
20
|
+
# Flushes whatever's already buffered on a normal process exit, so the last partial window
|
|
21
|
+
# isn't silently dropped; same reasoning every other buffer/flusher in this gem documents.
|
|
22
|
+
at_exit { flush }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def record(metric_name:, value:)
|
|
26
|
+
ensure_worker_started
|
|
27
|
+
|
|
28
|
+
@mutex.synchronize do
|
|
29
|
+
@entries << {
|
|
30
|
+
metric_name: metric_name, value: value,
|
|
31
|
+
environment: configuration.environment.to_s, release: configuration.release,
|
|
32
|
+
recorded_at: Time.now.utc.iso8601
|
|
33
|
+
}
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Snapshots and resets the buffered entries, then delivers them as one batch. A failed
|
|
38
|
+
# delivery keeps every entry where it is rather than resetting, so the next flush's batch
|
|
39
|
+
# just grows instead of losing what was already buffered; same reasoning SessionFlusher#flush
|
|
40
|
+
# already documents, including the same small, accepted race (an entry recorded during the
|
|
41
|
+
# in-flight HTTP request gets folded into this snapshot's own reset rather than kept for the
|
|
42
|
+
# next one) that class already lives with, for the same reason: simple and consistent beats a
|
|
43
|
+
# cleverer dedup that would only trade one rare edge case for a subtler one.
|
|
44
|
+
def flush
|
|
45
|
+
snapshot = nil
|
|
46
|
+
|
|
47
|
+
@mutex.synchronize do
|
|
48
|
+
return if @entries.empty?
|
|
49
|
+
snapshot = @entries
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
return unless client.deliver_metrics(snapshot)
|
|
53
|
+
|
|
54
|
+
@mutex.synchronize { @entries = [] }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
attr_reader :configuration, :client
|
|
59
|
+
|
|
60
|
+
def ensure_worker_started
|
|
61
|
+
return if @thread&.alive?
|
|
62
|
+
|
|
63
|
+
@mutex.synchronize do
|
|
64
|
+
return if @thread&.alive?
|
|
65
|
+
|
|
66
|
+
@thread = Thread.new { run }
|
|
67
|
+
@thread.abort_on_exception = false
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def run
|
|
72
|
+
loop do
|
|
73
|
+
sleep configuration.metric_flush_interval
|
|
74
|
+
flush
|
|
75
|
+
rescue StandardError => e
|
|
76
|
+
configuration.logger&.debug { "[ForgeOpsTracker] metric flush thread error: #{e.class}: #{e.message}" }
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Middleware
|
|
3
|
+
# Wraps every request: counts it, and separately counts it as crashed only when an exception
|
|
4
|
+
# actually escapes to here, never inferred from whether ErrorSubscriber happened to report
|
|
5
|
+
# anything, since ErrorSubscriber reports handled errors too (see Rails.error.handle), so "an
|
|
6
|
+
# event was reported" and "this request crashed" aren't the same signal. Always re-raises,
|
|
7
|
+
# so the host app's own error handling and ErrorSubscriber both run completely unaffected by
|
|
8
|
+
# this middleware's presence; it only ever observes, never intercepts.
|
|
9
|
+
class SessionTracking
|
|
10
|
+
def initialize(app, configuration: ForgeOpsTracker.configuration, flusher: nil)
|
|
11
|
+
@app = app
|
|
12
|
+
@configuration = configuration
|
|
13
|
+
@flusher = flusher
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call(env)
|
|
17
|
+
return app.call(env) unless configuration.track_sessions && configuration.enabled?
|
|
18
|
+
|
|
19
|
+
begin
|
|
20
|
+
response = app.call(env)
|
|
21
|
+
rescue StandardError
|
|
22
|
+
flusher_instance.record_session(crashed: true)
|
|
23
|
+
raise
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
flusher_instance.record_session(crashed: false)
|
|
27
|
+
response
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
attr_reader :app, :configuration
|
|
32
|
+
|
|
33
|
+
def flusher_instance
|
|
34
|
+
@flusher ||= SessionFlusher.new(configuration)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
module Middleware
|
|
3
|
+
# Wraps every request: if Warden is mounted (env["warden"] present, whether or not the app
|
|
4
|
+
# uses Devise specifically, since Devise just mounts Warden automatically, but any Warden-
|
|
5
|
+
# based auth setup works here, confirmed directly against the installed warden gem's own
|
|
6
|
+
# Warden::Manager/Warden::Proxy source) and it has a signed-in user, stashes a serialized
|
|
7
|
+
# version of that user in a thread-local for the duration of this request so
|
|
8
|
+
# ErrorSubscriber#report (which never receives the Rack env at all: Rails.error.subscribe's
|
|
9
|
+
# own interface hands it error/context/severity, nothing request-shaped) can read it back and
|
|
10
|
+
# attach it to whatever gets reported. A sibling to SessionTracking, not folded into it:
|
|
11
|
+
# distinct concern, same "own file, own class" shape every middleware in this gem already
|
|
12
|
+
# takes.
|
|
13
|
+
#
|
|
14
|
+
# The ensure-clear is load-bearing, not optional: Puma reuses threads across requests, so
|
|
15
|
+
# leaving this set would leak one request's user into a later, unrelated request handled on
|
|
16
|
+
# the same thread.
|
|
17
|
+
class UserContext
|
|
18
|
+
def initialize(app, configuration: ForgeOpsTracker.configuration)
|
|
19
|
+
@app = app
|
|
20
|
+
@configuration = configuration
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def call(env)
|
|
24
|
+
return app.call(env) unless configuration.track_current_user && configuration.enabled?
|
|
25
|
+
|
|
26
|
+
if env.key?("warden") && (user = env["warden"].user)
|
|
27
|
+
serialized = serialize(user)
|
|
28
|
+
Thread.current[:forge_ops_tracker_current_user] = serialized unless serialized.empty?
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
app.call(env)
|
|
32
|
+
ensure
|
|
33
|
+
Thread.current[:forge_ops_tracker_current_user] = nil
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
attr_reader :app, :configuration
|
|
38
|
+
|
|
39
|
+
# Duck-types rather than assuming a Devise-shaped model: id if the object responds to it,
|
|
40
|
+
# email if it responds to it, username else name if either responds to it. A custom
|
|
41
|
+
# Warden setup with a differently-shaped user object just gets whatever subset applies
|
|
42
|
+
# here; ForgeOpsTracker.set_user covers anything this can't infer.
|
|
43
|
+
def serialize(user)
|
|
44
|
+
{
|
|
45
|
+
id: (user.id if user.respond_to?(:id)),
|
|
46
|
+
email: (user.email if user.respond_to?(:email)),
|
|
47
|
+
username: (user.username if user.respond_to?(:username)) || (user.name if user.respond_to?(:name))
|
|
48
|
+
}.compact
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
require "thread"
|
|
2
|
+
|
|
3
|
+
module ForgeOpsTracker
|
|
4
|
+
# Times every request/job/query/outbound call in-process, bucketed by [transaction_name, kind]
|
|
5
|
+
# (see Railtie's several ActiveSupport::Notifications subscriptions, each a different kind), and
|
|
6
|
+
# periodically flushes each distinct bucket as one small aggregate report, rather than one
|
|
7
|
+
# network call per event. One shared flusher for every kind, not one per kind: the aggregation
|
|
8
|
+
# shape (count + duration sum + max over a period) is identical regardless of what's actually
|
|
9
|
+
# being timed. Same "lazily start a background thread on first use, not at load time" pattern
|
|
10
|
+
# SessionFlusher already uses, so each forked Puma/Passenger worker gets its own fresh thread
|
|
11
|
+
# instead of inheriting a dead one across fork.
|
|
12
|
+
class PerformanceFlusher
|
|
13
|
+
def initialize(configuration, client: Client.new(configuration))
|
|
14
|
+
@configuration = configuration
|
|
15
|
+
@client = client
|
|
16
|
+
@mutex = Mutex.new
|
|
17
|
+
@buckets = {}
|
|
18
|
+
@period_started_at = Time.now.utc
|
|
19
|
+
@thread = nil
|
|
20
|
+
|
|
21
|
+
# Flushes whatever's already tallied on a normal process exit, so the last partial window
|
|
22
|
+
# (anything shorter than a full performance_flush_interval) isn't silently dropped; same
|
|
23
|
+
# reasoning SessionFlusher's own at_exit hook documents.
|
|
24
|
+
at_exit { flush }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# kind ("controller"/"job"/"query"/"http", see Railtie's own subscriptions) is part of the
|
|
28
|
+
# bucket key alongside transaction_name, not folded into transaction_name itself: a job and a
|
|
29
|
+
# controller action (or, less likely but still real, a query and a job) can share a name
|
|
30
|
+
# without colliding into the same bucket.
|
|
31
|
+
def record(transaction_name:, duration_ms:, kind: "controller")
|
|
32
|
+
ensure_worker_started
|
|
33
|
+
|
|
34
|
+
@mutex.synchronize do
|
|
35
|
+
bucket = (@buckets[[ transaction_name, kind ]] ||= { count: 0, duration_sum_ms: 0.0, max_duration_ms: 0.0 })
|
|
36
|
+
bucket[:count] += 1
|
|
37
|
+
bucket[:duration_sum_ms] += duration_ms
|
|
38
|
+
bucket[:max_duration_ms] = duration_ms if duration_ms > bucket[:max_duration_ms]
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Snapshots and resets the in-process buckets, then delivers them as one batch. A failed
|
|
43
|
+
# delivery keeps every bucket where it is rather than resetting, so the next flush's window
|
|
44
|
+
# just grows instead of losing what was already tallied; same reasoning SessionFlusher#flush
|
|
45
|
+
# already documents, and the same reason PerformanceSamplesController accepts a batch rather
|
|
46
|
+
# than a single row: a failed flush shouldn't have to re-deliver by transaction one at a time.
|
|
47
|
+
def flush
|
|
48
|
+
snapshot = nil
|
|
49
|
+
period_started_at = nil
|
|
50
|
+
|
|
51
|
+
@mutex.synchronize do
|
|
52
|
+
return if @buckets.empty?
|
|
53
|
+
|
|
54
|
+
period_started_at = @period_started_at
|
|
55
|
+
snapshot = @buckets.map do |(transaction_name, kind), bucket|
|
|
56
|
+
{
|
|
57
|
+
transaction_name: transaction_name,
|
|
58
|
+
kind: kind,
|
|
59
|
+
environment: configuration.environment.to_s,
|
|
60
|
+
release: configuration.release,
|
|
61
|
+
period_started_at: period_started_at.iso8601,
|
|
62
|
+
period_ended_at: Time.now.utc.iso8601,
|
|
63
|
+
request_count: bucket[:count],
|
|
64
|
+
duration_sum_ms: bucket[:duration_sum_ms],
|
|
65
|
+
max_duration_ms: bucket[:max_duration_ms]
|
|
66
|
+
}
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
return unless snapshot && client.deliver_performance_samples(snapshot)
|
|
71
|
+
|
|
72
|
+
@mutex.synchronize do
|
|
73
|
+
@buckets = {}
|
|
74
|
+
@period_started_at = Time.now.utc
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
attr_reader :configuration, :client
|
|
80
|
+
|
|
81
|
+
def ensure_worker_started
|
|
82
|
+
return if @thread&.alive?
|
|
83
|
+
|
|
84
|
+
@mutex.synchronize do
|
|
85
|
+
return if @thread&.alive?
|
|
86
|
+
|
|
87
|
+
@thread = Thread.new { run }
|
|
88
|
+
@thread.abort_on_exception = false
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def run
|
|
93
|
+
loop do
|
|
94
|
+
sleep configuration.performance_flush_interval
|
|
95
|
+
flush
|
|
96
|
+
rescue StandardError => e
|
|
97
|
+
configuration.logger&.debug { "[ForgeOpsTracker] performance flush thread error: #{e.class}: #{e.message}" }
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
# The actual "what do we call this, and should we even record it" decisions Railtie's
|
|
3
|
+
# ActiveSupport::Notifications subscriptions make, pulled out into plain module-function methods
|
|
4
|
+
# rather than left inline in each subscribe block. Railtie itself only ever loads when
|
|
5
|
+
# Rails::Railtie is already defined (see lib/forge_ops_tracker.rb's own final line), so nothing
|
|
6
|
+
# in that file can be exercised by this gem's own test suite without adding Rails itself as a
|
|
7
|
+
# development dependency just to fire a notification event by hand; every method here takes a
|
|
8
|
+
# plain payload Hash instead of a real ActiveSupport::Notifications::Event, so the decision logic
|
|
9
|
+
# itself (the SCHEMA/cached skip, the ActiveJob-wrapper dedup) is fully unit-testable with no
|
|
10
|
+
# Rails/Sidekiq/ActiveJob dependency at all. Railtie's own subscribe blocks stay thin, calling
|
|
11
|
+
# straight into these.
|
|
12
|
+
module PerformanceInstrumentation
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def controller_transaction_name(payload)
|
|
16
|
+
"#{payload[:controller]}##{payload[:action]}"
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# nil means "don't record this one": Rails' own sql.active_record fires for two kinds of
|
|
20
|
+
# noise that would drown out real query timing otherwise. "SCHEMA" is Rails' own internal
|
|
21
|
+
# introspection (column lookups, etc.), never something a customer's own code triggered;
|
|
22
|
+
# payload[:cached] is a genuinely instant hash lookup off Rails' own query cache, not a real
|
|
23
|
+
# round trip to the database, so timing it would understate every cached query's own real
|
|
24
|
+
# cost the next time it isn't cached.
|
|
25
|
+
#
|
|
26
|
+
# payload[:name] (Rails' own auto-generated label, e.g. "User Load", "Order Create"), not
|
|
27
|
+
# payload[:sql] (the literal query text): a customer's own query text could carry a literal
|
|
28
|
+
# value in some edge cases despite ActiveRecord's own parameterization, and even where it
|
|
29
|
+
# doesn't, raw SQL text is far higher cardinality than this system's transaction_name is
|
|
30
|
+
# designed for everywhere else. Falls back to the literal string "SQL" for the rare query
|
|
31
|
+
# with no name at all, rather than skipping it outright.
|
|
32
|
+
def query_transaction_name(payload)
|
|
33
|
+
return nil if payload[:name] == "SCHEMA" || payload[:cached]
|
|
34
|
+
|
|
35
|
+
# A plain nil-or-empty check, not ActiveSupport's #presence: this module is unit-tested
|
|
36
|
+
# without Rails/ActiveSupport loaded at all (see this file's own top comment), so it can't
|
|
37
|
+
# rely on a method ActiveSupport adds to String/NilClass.
|
|
38
|
+
name = payload[:name]
|
|
39
|
+
name.nil? || name.empty? ? "SQL" : name
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def job_transaction_name(payload)
|
|
43
|
+
payload[:job].class.name
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# True for a job dispatched through ActiveJob but actually run on Sidekiq: Sidekiq's own
|
|
47
|
+
# server middleware chain wraps every job it runs, ActiveJob-dispatched or not, so without
|
|
48
|
+
# this check a job on the Sidekiq/ActiveJob combination would be recorded twice: once here
|
|
49
|
+
# (kind: "job", via the raw Sidekiq middleware) and once already by the perform.active_job
|
|
50
|
+
# subscription that already covers every ActiveJob job regardless of which queue backend
|
|
51
|
+
# actually runs it (Solid Queue, Sidekiq, or anything else). The class name itself is
|
|
52
|
+
# ActiveJob's own, not this gem's: confirmed directly against how ActiveJob wraps a job for
|
|
53
|
+
# Sidekiq, not assumed from either project's docs alone.
|
|
54
|
+
def sidekiq_worker_is_active_job_wrapper?(worker)
|
|
55
|
+
return false unless defined?(::ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper)
|
|
56
|
+
|
|
57
|
+
worker.is_a?(::ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
module ForgeOpsTracker
|
|
2
|
+
# A small shared background-thread loop for the Sidekiq/Solid Queue/Puma stats reporters (see
|
|
3
|
+
# integrations/): unlike PerformanceFlusher and the two metric buffers, which genuinely differ
|
|
4
|
+
# in what they buffer and are kept as separate sibling classes on purpose (see
|
|
5
|
+
# InfrastructureMetricBuffer's own comment), these three reporters only differ in *what they
|
|
6
|
+
# poll*, not *how they poll it*: lazily start a thread on first use (so a forked worker gets
|
|
7
|
+
# its own live thread regardless of when it forked relative to load, same reasoning
|
|
8
|
+
# PerformanceFlusher's own lazy start documents), sleep the interval, call the block, rescue and
|
|
9
|
+
# log so one bad tick never kills the loop, forever. Genuinely mechanical enough across all
|
|
10
|
+
# three to share, unlike those.
|
|
11
|
+
#
|
|
12
|
+
# No at_exit flush: unlike the flushers/buffers, which hold real accumulated data that would
|
|
13
|
+
# otherwise be lost, a gauge reading skipped on process exit is simply the same as one skipped
|
|
14
|
+
# because the interval hadn't ticked yet: nothing to lose.
|
|
15
|
+
class PeriodicPoller
|
|
16
|
+
def self.start(interval, logger: nil, &block)
|
|
17
|
+
new(interval, logger: logger, &block).tap(&:start)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def initialize(interval, logger: nil, &block)
|
|
21
|
+
@interval = interval
|
|
22
|
+
@logger = logger
|
|
23
|
+
@block = block
|
|
24
|
+
@mutex = Mutex.new
|
|
25
|
+
@thread = nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def start
|
|
29
|
+
return if @thread&.alive?
|
|
30
|
+
|
|
31
|
+
@mutex.synchronize do
|
|
32
|
+
return if @thread&.alive?
|
|
33
|
+
|
|
34
|
+
@thread = Thread.new { run }
|
|
35
|
+
@thread.abort_on_exception = false
|
|
36
|
+
end
|
|
37
|
+
self
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
def run
|
|
42
|
+
loop do
|
|
43
|
+
sleep @interval
|
|
44
|
+
@block.call
|
|
45
|
+
rescue StandardError => e
|
|
46
|
+
@logger&.debug { "[ForgeOpsTracker] periodic poller error: #{e.class}: #{e.message}" }
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
module ForgeOpsTracker
|
|
2
2
|
# Redacts likely-sensitive content out of a payload before it ever leaves
|
|
3
|
-
# this process
|
|
3
|
+
# this process; the same patterns ForgeOps itself applies again on
|
|
4
4
|
# arrival (defense in depth: this layer keeps the data off the wire and
|
|
5
5
|
# out of any request logging in between; the server-side layer is what
|
|
6
6
|
# actually protects the database, and doesn't depend on every reporting
|
|
7
7
|
# app running an up-to-date version of this gem). See the main
|
|
8
|
-
# application's PiiScrubber for the shared design rationale
|
|
8
|
+
# application's PiiScrubber for the shared design rationale; kept as a
|
|
9
9
|
# separate, dependency-free implementation here rather than requiring the
|
|
10
10
|
# private app's code, since this gem has to work standalone in any host
|
|
11
11
|
# app regardless of what's reporting into it.
|