prometheus_exporter 0.5.0 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +42 -0
  3. data/.gitignore +2 -0
  4. data/.rubocop.yml +7 -1
  5. data/Appraisals +10 -0
  6. data/CHANGELOG +27 -2
  7. data/README.md +257 -5
  8. data/bin/prometheus_exporter +21 -0
  9. data/gemfiles/.bundle/config +2 -0
  10. data/gemfiles/ar_60.gemfile +5 -0
  11. data/gemfiles/ar_61.gemfile +7 -0
  12. data/lib/prometheus_exporter.rb +2 -0
  13. data/lib/prometheus_exporter/client.rb +32 -4
  14. data/lib/prometheus_exporter/instrumentation.rb +1 -0
  15. data/lib/prometheus_exporter/instrumentation/active_record.rb +16 -7
  16. data/lib/prometheus_exporter/instrumentation/process.rb +2 -0
  17. data/lib/prometheus_exporter/instrumentation/sidekiq.rb +44 -3
  18. data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb +50 -0
  19. data/lib/prometheus_exporter/metric/base.rb +4 -0
  20. data/lib/prometheus_exporter/metric/counter.rb +4 -0
  21. data/lib/prometheus_exporter/metric/gauge.rb +4 -0
  22. data/lib/prometheus_exporter/metric/histogram.rb +6 -0
  23. data/lib/prometheus_exporter/metric/summary.rb +7 -0
  24. data/lib/prometheus_exporter/middleware.rb +26 -8
  25. data/lib/prometheus_exporter/server.rb +1 -0
  26. data/lib/prometheus_exporter/server/active_record_collector.rb +1 -0
  27. data/lib/prometheus_exporter/server/collector.rb +1 -0
  28. data/lib/prometheus_exporter/server/delayed_job_collector.rb +11 -0
  29. data/lib/prometheus_exporter/server/hutch_collector.rb +6 -0
  30. data/lib/prometheus_exporter/server/runner.rb +24 -2
  31. data/lib/prometheus_exporter/server/shoryuken_collector.rb +8 -0
  32. data/lib/prometheus_exporter/server/sidekiq_collector.rb +11 -2
  33. data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb +46 -0
  34. data/lib/prometheus_exporter/server/web_collector.rb +7 -5
  35. data/lib/prometheus_exporter/server/web_server.rb +29 -17
  36. data/lib/prometheus_exporter/version.rb +1 -1
  37. data/prometheus_exporter.gemspec +9 -5
  38. metadata +65 -17
  39. data/.travis.yml +0 -12
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require 'optparse'
5
+ require 'json'
5
6
 
6
7
  require_relative "./../lib/prometheus_exporter"
7
8
  require_relative "./../lib/prometheus_exporter/server"
@@ -34,6 +35,9 @@ def run
34
35
  opt.on('--prefix METRIC_PREFIX', "Prefix to apply to all metrics (default: #{PrometheusExporter::DEFAULT_PREFIX})") do |o|
35
36
  options[:prefix] = o.to_s
36
37
  end
38
+ opt.on('--label METRIC_LABEL', "Label to apply to all metrics (default: #{PrometheusExporter::DEFAULT_LABEL})") do |o|
39
+ options[:label] = JSON.parse(o.to_s)
40
+ end
37
41
  opt.on('-c', '--collector FILE', String, "(optional) Custom collector to run") do |o|
38
42
  custom_collector_filename = o.to_s
39
43
  end
@@ -43,6 +47,12 @@ def run
43
47
  opt.on('-v', '--verbose') do |o|
44
48
  options[:verbose] = true
45
49
  end
50
+ opt.on('--auth FILE', String, "(optional) enable basic authentication using a htpasswd FILE") do |o|
51
+ options[:auth] = o
52
+ end
53
+ opt.on('--realm REALM', String, "(optional) Use REALM for basic authentication (default: \"#{PrometheusExporter::DEFAULT_REALM}\")") do |o|
54
+ options[:realm] = o
55
+ end
46
56
 
47
57
  opt.on('--unicorn-listen-address ADDRESS', String, '(optional) Address where unicorn listens on (unix or TCP address)') do |o|
48
58
  options[:unicorn_listen_address] = o
@@ -53,6 +63,17 @@ def run
53
63
  end
54
64
  end.parse!
55
65
 
66
+ if options.has_key?(:realm) && !options.has_key?(:auth)
67
+ STDERR.puts "[Warn] Providing REALM without AUTH has no effect"
68
+ end
69
+
70
+ if options.has_key?(:auth)
71
+ unless File.exist?(options[:auth]) && File.readable?(options[:auth])
72
+ STDERR.puts "[Error] The AUTH file either doesn't exist or we don't have access to it"
73
+ exit 1
74
+ end
75
+ end
76
+
56
77
  if custom_collector_filename
57
78
  eval File.read(custom_collector_filename), nil, File.expand_path(custom_collector_filename)
58
79
  found = false
@@ -0,0 +1,2 @@
1
+ ---
2
+ BUNDLE_RETRY: "1"
@@ -0,0 +1,5 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "activerecord", "~> 6.1.0"
6
+
7
+ gemspec path: "../"
@@ -9,7 +9,9 @@ module PrometheusExporter
9
9
  DEFAULT_PORT = 9394
10
10
  DEFAULT_BIND_ADDRESS = 'localhost'
11
11
  DEFAULT_PREFIX = 'ruby_'
12
+ DEFAULT_LABEL = {}
12
13
  DEFAULT_TIMEOUT = 2
14
+ DEFAULT_REALM = 'Prometheus Exporter'
13
15
 
14
16
  class OjCompat
15
17
  def self.parse(obj)
@@ -53,12 +53,21 @@ module PrometheusExporter
53
53
  MAX_SOCKET_AGE = 25
54
54
  MAX_QUEUE_SIZE = 10_000
55
55
 
56
- def initialize(host: 'localhost', port: PrometheusExporter::DEFAULT_PORT, max_queue_size: nil, thread_sleep: 0.5, json_serializer: nil, custom_labels: nil)
56
+ def initialize(
57
+ host: ENV.fetch('PROMETHEUS_EXPORTER_HOST', 'localhost'),
58
+ port: ENV.fetch('PROMETHEUS_EXPORTER_PORT', PrometheusExporter::DEFAULT_PORT),
59
+ max_queue_size: nil,
60
+ thread_sleep: 0.5,
61
+ json_serializer: nil,
62
+ custom_labels: nil
63
+ )
57
64
  @metrics = []
58
65
 
59
66
  @queue = Queue.new
67
+
60
68
  @socket = nil
61
69
  @socket_started = nil
70
+ @socket_pid = nil
62
71
 
63
72
  max_queue_size ||= MAX_QUEUE_SIZE
64
73
  max_queue_size = max_queue_size.to_i
@@ -100,7 +109,16 @@ module PrometheusExporter
100
109
  end
101
110
 
102
111
  def send_json(obj)
103
- payload = @custom_labels.nil? ? obj : obj.merge(custom_labels: @custom_labels)
112
+ payload =
113
+ if @custom_labels
114
+ if obj[:custom_labels]
115
+ obj.merge(custom_labels: @custom_labels.merge(obj[:custom_labels]))
116
+ else
117
+ obj.merge(custom_labels: @custom_labels)
118
+ end
119
+ else
120
+ obj
121
+ end
104
122
  send(@json_serializer.dump(payload))
105
123
  end
106
124
 
@@ -170,7 +188,7 @@ module PrometheusExporter
170
188
 
171
189
  def close_socket!
172
190
  begin
173
- if @socket
191
+ if @socket && !@socket.closed?
174
192
  @socket.write("0\r\n")
175
193
  @socket.write("\r\n")
176
194
  @socket.flush
@@ -184,12 +202,20 @@ module PrometheusExporter
184
202
  end
185
203
 
186
204
  def close_socket_if_old!
187
- if @socket && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
205
+ if @socket_pid == Process.pid && @socket && @socket_started && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
188
206
  close_socket!
189
207
  end
190
208
  end
191
209
 
192
210
  def ensure_socket!
211
+ # if process was forked socket may be owned by parent
212
+ # leave it alone and reset
213
+ if @socket_pid != Process.pid
214
+ @socket = nil
215
+ @socket_started = nil
216
+ @socket_pid = nil
217
+ end
218
+
193
219
  close_socket_if_old!
194
220
  if !@socket
195
221
  @socket = TCPSocket.new @host, @port
@@ -200,12 +226,14 @@ module PrometheusExporter
200
226
  @socket.write("Content-Type: application/octet-stream\r\n")
201
227
  @socket.write("\r\n")
202
228
  @socket_started = Time.now.to_f
229
+ @socket_pid = Process.pid
203
230
  end
204
231
 
205
232
  nil
206
233
  rescue
207
234
  @socket = nil
208
235
  @socket_started = nil
236
+ @socket_pid = nil
209
237
  raise
210
238
  end
211
239
 
@@ -4,6 +4,7 @@ require_relative "client"
4
4
  require_relative "instrumentation/process"
5
5
  require_relative "instrumentation/method_profiler"
6
6
  require_relative "instrumentation/sidekiq"
7
+ require_relative "instrumentation/sidekiq_queue"
7
8
  require_relative "instrumentation/delayed_job"
8
9
  require_relative "instrumentation/puma"
9
10
  require_relative "instrumentation/hutch"
@@ -67,21 +67,30 @@ module PrometheusExporter::Instrumentation
67
67
  ObjectSpace.each_object(::ActiveRecord::ConnectionAdapters::ConnectionPool) do |pool|
68
68
  next if pool.connections.nil?
69
69
 
70
- labels_from_config = pool.spec.config
71
- .select { |k, v| @config_labels.include? k }
72
- .map { |k, v| [k.to_s.prepend("dbconfig_"), v] }
73
-
74
- labels = @metric_labels.merge(pool_name: pool.spec.name).merge(Hash[labels_from_config])
75
-
76
70
  metric = {
77
71
  pid: pid,
78
72
  type: "active_record",
79
73
  hostname: ::PrometheusExporter.hostname,
80
- metric_labels: labels
74
+ metric_labels: labels(pool)
81
75
  }
82
76
  metric.merge!(pool.stat)
83
77
  metrics << metric
84
78
  end
85
79
  end
80
+
81
+ private
82
+
83
+ def labels(pool)
84
+ if pool.respond_to?(:spec) # ActiveRecord <= 6.0
85
+ @metric_labels.merge(pool_name: pool.spec.name).merge(pool.spec.config
86
+ .select { |k, v| @config_labels.include? k }
87
+ .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }.to_h)
88
+ elsif pool.respond_to?(:db_config) # ActiveRecord >= 6.1.rc1
89
+ @metric_labels.merge(pool_name: pool.db_config.name).merge(
90
+ @config_labels.each_with_object({}) { |l, acc| acc["dbconfig_#{l}"] = pool.db_config.public_send(l) })
91
+ else
92
+ raise "Unsupported connection pool"
93
+ end
94
+ end
86
95
  end
87
96
  end
@@ -3,6 +3,8 @@
3
3
  # collects stats from currently running process
4
4
  module PrometheusExporter::Instrumentation
5
5
  class Process
6
+ @thread = nil if !defined?(@thread)
7
+
6
8
  def self.start(client: nil, type: "ruby", frequency: 30, labels: nil)
7
9
 
8
10
  metric_labels =
@@ -1,6 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'yaml'
4
+
3
5
  module PrometheusExporter::Instrumentation
6
+ JOB_WRAPPER_CLASS_NAME = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'
7
+ DELAYED_CLASS_NAMES = [
8
+ 'Sidekiq::Extensions::DelayedClass',
9
+ 'Sidekiq::Extensions::DelayedModel',
10
+ 'Sidekiq::Extensions::DelayedMailer',
11
+ ]
12
+
4
13
  class Sidekiq
5
14
  def self.death_handler
6
15
  -> (job, ex) do
@@ -32,15 +41,47 @@ module PrometheusExporter::Instrumentation
32
41
  raise e
33
42
  ensure
34
43
  duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
35
- class_name = worker.class.to_s == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' ?
36
- msg['wrapped'] : worker.class.to_s
37
44
  @client.send_json(
38
45
  type: "sidekiq",
39
- name: class_name,
46
+ name: get_name(worker, msg),
47
+ queue: queue,
40
48
  success: success,
41
49
  shutdown: shutdown,
42
50
  duration: duration
43
51
  )
44
52
  end
53
+
54
+ private
55
+
56
+ def get_name(worker, msg)
57
+ class_name = worker.class.to_s
58
+ if class_name == JOB_WRAPPER_CLASS_NAME
59
+ get_job_wrapper_name(msg)
60
+ elsif DELAYED_CLASS_NAMES.include?(class_name)
61
+ get_delayed_name(msg, class_name)
62
+ else
63
+ class_name
64
+ end
65
+ end
66
+
67
+ def get_job_wrapper_name(msg)
68
+ msg['wrapped']
69
+ end
70
+
71
+ def get_delayed_name(msg, class_name)
72
+ # fallback to class_name since we're relying on the internal implementation
73
+ # of the delayed extensions
74
+ # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
75
+ begin
76
+ (target, method_name, _args) = YAML.load(msg['args'].first)
77
+ if target.class == Class
78
+ "#{target.name}##{method_name}"
79
+ else
80
+ "#{target.class.name}##{method_name}"
81
+ end
82
+ rescue
83
+ class_name
84
+ end
85
+ end
45
86
  end
46
87
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PrometheusExporter::Instrumentation
4
+ class SidekiqQueue
5
+ def self.start(client: nil, frequency: 30)
6
+ client ||= PrometheusExporter::Client.default
7
+ sidekiq_queue_collector = new
8
+
9
+ Thread.new do
10
+ loop do
11
+ begin
12
+ client.send_json(sidekiq_queue_collector.collect)
13
+ rescue StandardError => e
14
+ STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
15
+ ensure
16
+ sleep frequency
17
+ end
18
+ end
19
+ end
20
+ end
21
+
22
+ def collect
23
+ {
24
+ type: 'sidekiq_queue',
25
+ queues: collect_queue_stats
26
+ }
27
+ end
28
+
29
+ def collect_queue_stats
30
+ hostname = Socket.gethostname
31
+ pid = ::Process.pid
32
+ ps = ::Sidekiq::ProcessSet.new
33
+
34
+ process = ps.find do |sp|
35
+ sp['hostname'] == hostname && sp['pid'] == pid
36
+ end
37
+
38
+ queues = process.nil? ? [] : process['queues']
39
+
40
+ ::Sidekiq::Queue.all.map do |queue|
41
+ next unless queues.include? queue.name
42
+ {
43
+ backlog_total: queue.size,
44
+ latency_seconds: queue.latency.to_i,
45
+ labels: { queue: queue.name }
46
+ }
47
+ end.compact
48
+ end
49
+ end
50
+ end
@@ -2,6 +2,10 @@
2
2
 
3
3
  module PrometheusExporter::Metric
4
4
  class Base
5
+
6
+ @default_prefix = nil if !defined?(@default_prefix)
7
+ @default_labels = nil if !defined?(@default_labels)
8
+
5
9
  # prefix applied to all metrics
6
10
  def self.default_prefix=(name)
7
11
  @default_prefix = name
@@ -27,6 +27,10 @@ module PrometheusExporter::Metric
27
27
  @data.dup
28
28
  end
29
29
 
30
+ def remove(labels)
31
+ @data.delete(labels)
32
+ end
33
+
30
34
  def observe(increment = 1, labels = {})
31
35
  @data[labels] ||= 0
32
36
  @data[labels] += increment
@@ -27,6 +27,10 @@ module PrometheusExporter::Metric
27
27
  @data.dup
28
28
  end
29
29
 
30
+ def remove(labels)
31
+ @data.delete(labels)
32
+ end
33
+
30
34
  def observe(value, labels = {})
31
35
  if value.nil?
32
36
  data.delete(labels)
@@ -27,6 +27,12 @@ module PrometheusExporter::Metric
27
27
  data
28
28
  end
29
29
 
30
+ def remove(labels)
31
+ @observations.delete(labels)
32
+ @counts.delete(labels)
33
+ @sums.delete(labels)
34
+ end
35
+
30
36
  def type
31
37
  "histogram"
32
38
  end
@@ -32,6 +32,13 @@ module PrometheusExporter::Metric
32
32
  data
33
33
  end
34
34
 
35
+ def remove(labels)
36
+ @counts.delete(labels)
37
+ @sums.delete(labels)
38
+ @buffers[0].delete(labels)
39
+ @buffers[1].delete(labels)
40
+ end
41
+
35
42
  def type
36
43
  "summary"
37
44
  end
@@ -36,22 +36,40 @@ class PrometheusExporter::Middleware
36
36
 
37
37
  result
38
38
  ensure
39
+
40
+ obj = {
41
+ type: "web",
42
+ timings: info,
43
+ queue_time: queue_time,
44
+ default_labels: default_labels(env, result)
45
+ }
46
+ labels = custom_labels(env)
47
+ if labels
48
+ obj = obj.merge(custom_labels: labels)
49
+ end
50
+
51
+ @client.send_json(obj)
52
+ end
53
+
54
+ def default_labels(env, result)
39
55
  status = (result && result[0]) || -1
40
56
  params = env["action_dispatch.request.parameters"]
41
- action, controller = nil
57
+ action = controller = nil
42
58
  if params
43
59
  action = params["action"]
44
60
  controller = params["controller"]
45
61
  end
46
62
 
47
- @client.send_json(
48
- type: "web",
49
- timings: info,
50
- queue_time: queue_time,
51
- action: action,
52
- controller: controller,
63
+ {
64
+ action: action || "other",
65
+ controller: controller || "other",
53
66
  status: status
54
- )
67
+ }
68
+ end
69
+
70
+ # allows subclasses to add custom labels based on env
71
+ def custom_labels(env)
72
+ nil
55
73
  end
56
74
 
57
75
  private