prometheus_exporter 0.3.4 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7725cb2b5d8846c874cd69d2ce92ca7a7fd2859ce9e6babe8cc86e96db9e3f2b
4
- data.tar.gz: c08f9d1a4b18733c9a1c8882453bc40ffb7347ac5a71ccc972500334245aecba
3
+ metadata.gz: e41c3872a56ccec50aa5442ede71eb37509761e1a53c065ff3b877f8ef3732a0
4
+ data.tar.gz: 921f95d6a0e28bc17b212fda81984d28327d31e364d9a8efc34718a32d2c4053
5
5
  SHA512:
6
- metadata.gz: 436ad9dcd8248147fb13ba1f0fc3d2e0b1badc113c41bc7ead07c560f984f32b7caa72fd4cfa1162209b525319574cfb962dff9dd092c2557fca39df2fbe1a8c
7
- data.tar.gz: 9365bea75c02bc742514058cd7d66c077a8b0d49ce941a5067887cacd6a051ca28f86870c4456579f16101e98ea4d70733c08e8f9924ff8dead52be4b2c01d91
6
+ metadata.gz: e208fd0be610e8a268c0a1dcf7559b07dbdf8745bff9cedf6fb04967d132d36d324b8d76c85cad19f74ec4e623eeb356f90c624f4918d7e264839d55eebb8815
7
+ data.tar.gz: e1d6287fa0ca432e59ad48be400e375f75448f539a56117f8fde8c9a32314af5ee37fdaf989229c8d1000115d802012082fc641d098426354b9eccef46622237
data/CHANGELOG CHANGED
@@ -1,3 +1,10 @@
1
+ 0.4.0 - 23-10-2018
2
+
3
+ - Feature: histogram support
4
+ - Feature: custom quantile support for summary
5
+ - Feature: Puma metrics
6
+ - Fix: delayed job metrics
7
+
1
8
  0.3.4 - 02-10-2018
2
9
 
3
10
  - Fix: custom collector via CLI was not working correctly
data/README.md CHANGED
@@ -8,11 +8,13 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
8
8
  * [Installation](#installation)
9
9
  * [Usage](#usage)
10
10
  * [Single process mode](#single-process-mode)
11
+ * [Custom quantiles and buckets](#custom-quantiles-and-buckets)
11
12
  * [Multi process mode](#multi-process-mode)
12
13
  * [Rails integration](#rails-integration)
13
14
  * [Per-process stats](#per-process-stats)
14
15
  * [Sidekiq metrics](#sidekiq-metrics)
15
16
  * [Delayed Job plugin](#delayed-job-plugin)
17
+ * [Puma metrics](#puma-metrics)
16
18
  * [Custom type collectors](#custom-type-collectors)
17
19
  * [Multi process mode with custom collector](#multi-process-mode-with-custom-collector)
18
20
  * [GraphQL support](#graphql-support)
@@ -60,10 +62,12 @@ server.start
60
62
  gauge = PrometheusExporter::Metric::Gauge.new("rss", "used RSS for process")
61
63
  counter = PrometheusExporter::Metric::Counter.new("web_requests", "number of web requests")
62
64
  summary = PrometheusExporter::Metric::Summary.new("page_load_time", "time it took to load page")
65
+ histogram = PrometheusExporter::Metric::Histogram.new("api_access_time", "time it took to call api")
63
66
 
64
67
  server.collector.register_metric(gauge)
65
68
  server.collector.register_metric(counter)
66
69
  server.collector.register_metric(summary)
70
+ server.collector.register_metric(histogram)
67
71
 
68
72
  gauge.observe(get_rss)
69
73
  gauge.observe(get_rss)
@@ -75,10 +79,23 @@ summary.observe(1.1)
75
79
  summary.observe(1.12)
76
80
  summary.observe(0.12)
77
81
 
82
+ histogram.observe(0.2, api: 'twitter')
83
+
78
84
  # http://localhost:12345/metrics now returns all your metrics
79
85
 
80
86
  ```
81
87
 
88
+ #### Custom quantiles and buckets
89
+
90
+ You can also choose custom quantiles for summaries and custom buckets for histograms.
91
+
92
+ ```ruby
93
+
94
+ summary = PrometheusExporter::Metric::Summary.new("load_time", "time to load page", quantiles: [0.99, 0.75, 0.5, 0.25])
95
+ histogram = PrometheusExporter::Metric::Histogram.new("api_time", "time to call api", buckets: [0.1, 0.5, 1])
96
+
97
+ ```
98
+
82
99
  ### Multi process mode
83
100
 
84
101
  In some cases (for example, unicorn or puma clusters) you may want to aggregate metrics across multiple processes.
@@ -211,6 +228,20 @@ Configure your HTTP server / load balancer to add a header `X-Request-Start: t=<
211
228
 
212
229
  Hint: we aim to be API-compatible with the big APM solutions, so if you've got requests queueing time configured for them, it should be expected to also work with `prometheus_exporter`.
213
230
 
231
+ ### Puma metrics
232
+
233
+ The puma metrics are using the `Puma.stats` method and hence need to be started after the
234
+ workers has been booted and from a Puma thread otherwise the metrics won't be accessible.
235
+ The easiest way to gather this metrics is to put the following in your `puma.rb` config:
236
+
237
+ ```ruby
238
+ # puma.rb config
239
+ after_worker_boot do
240
+ require 'prometheus_exporter/instrumentation'
241
+ PrometheusExporter::Instrumentation::Puma.start
242
+ end
243
+ ```
244
+
214
245
  ### Custom type collectors
215
246
 
216
247
  In some cases you may have custom metrics you want to ship the collector in a batch. In this case you may still be interested in the base collector behavior, but would like to add your own special messages.
@@ -0,0 +1,62 @@
1
+ require 'json'
2
+
3
+ # collects stats from puma
4
+ module PrometheusExporter::Instrumentation
5
+ class Puma
6
+ def self.start(client: nil, frequency: 30)
7
+ puma_collector = new
8
+ client ||= PrometheusExporter::Client.default
9
+ Thread.new do
10
+ while true
11
+ begin
12
+ metric = puma_collector.collect
13
+ client.send_json metric
14
+ rescue => e
15
+ STDERR.puts("Prometheus Exporter Failed To Collect Puma Stats #{e}")
16
+ ensure
17
+ sleep frequency
18
+ end
19
+ end
20
+ end
21
+ end
22
+
23
+ def collect
24
+ metric = {}
25
+ metric[:type] = "puma"
26
+ collect_puma_stats(metric)
27
+ metric
28
+ end
29
+
30
+ def collect_puma_stats(metric)
31
+ stats = JSON.parse(::Puma.stats)
32
+
33
+ if stats.key? 'workers'
34
+ metric[:phase] = stats["phase"]
35
+ metric[:workers_total] = stats["workers"]
36
+ metric[:booted_workers_total] = stats["booted_workers"]
37
+ metric[:old_workers_total] = stats["old_workers"]
38
+
39
+ stats["worker_status"].each do |worker|
40
+ next if worker["last_status"].empty?
41
+ collect_worker_status(metric, worker["last_status"])
42
+ end
43
+ else
44
+ collect_worker_status(metric, stats)
45
+ end
46
+ end
47
+
48
+ private
49
+
50
+ def collect_worker_status(metric, status)
51
+ metric[:request_backlog_total] ||= 0
52
+ metric[:running_threads_total] ||= 0
53
+ metric[:thread_pool_capacity_total] ||= 0
54
+ metric[:max_threads_total] ||= 0
55
+
56
+ metric[:request_backlog_total] += status["backlog"]
57
+ metric[:running_threads_total] += status["running"]
58
+ metric[:thread_pool_capacity_total] += status["pool_capacity"]
59
+ metric[:max_threads_total] += status["max_threads"]
60
+ end
61
+ end
62
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module PrometheusExporter::Instrumentation
2
4
  class Sidekiq
3
5
 
@@ -13,10 +15,15 @@ module PrometheusExporter::Instrumentation
13
15
  result
14
16
  ensure
15
17
  duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
18
+ class_name = if worker.class.to_s == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'
19
+ msg['wrapped']
20
+ else
21
+ worker.class.to_s
22
+ end
16
23
 
17
24
  @client.send_json(
18
25
  type: "sidekiq",
19
- name: worker.class.to_s,
26
+ name: class_name,
20
27
  success: success,
21
28
  duration: duration
22
29
  )
@@ -3,3 +3,4 @@ require_relative "instrumentation/method_profiler"
3
3
  require_relative "instrumentation/sidekiq"
4
4
  require_relative "instrumentation/delayed_job"
5
5
  require_relative "instrumentation/global"
6
+ require_relative "instrumentation/puma"
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PrometheusExporter::Metric
4
+ class Histogram < Base
5
+
6
+ DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5.0, 10.0].freeze
7
+
8
+ def initialize(name, help, opts = {})
9
+ super(name, help)
10
+ @buckets = (opts[:buckets] || DEFAULT_BUCKETS).sort.reverse
11
+ @sums = {}
12
+ @counts = {}
13
+ @observations = {}
14
+ end
15
+
16
+ def type
17
+ "histogram"
18
+ end
19
+
20
+ def metric_text
21
+ text = +""
22
+ first = true
23
+ @observations.each do |labels, buckets|
24
+ text << "\n" unless first
25
+ first = false
26
+ count = @counts[labels]
27
+ sum = @sums[labels]
28
+ text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, "+Inf"))} #{count}\n"
29
+ @buckets.each do |bucket|
30
+ value = @observations[labels][bucket]
31
+ text << "#{prefix(@name)}_bucket#{labels_text(with_bucket(labels, bucket.to_s))} #{value}\n"
32
+ end
33
+ text << "#{prefix(@name)}_count#{labels_text(labels)} #{count}\n"
34
+ text << "#{prefix(@name)}_sum#{labels_text(labels)} #{sum}"
35
+ end
36
+ text
37
+ end
38
+
39
+ def observe(value, labels = nil)
40
+ labels ||= {}
41
+ buckets = ensure_histogram(labels)
42
+
43
+ value = value.to_f
44
+ @sums[labels] += value
45
+ @counts[labels] += 1
46
+
47
+ fill_buckets(value, buckets)
48
+ end
49
+
50
+ def ensure_histogram(labels)
51
+ @sums[labels] ||= 0.0
52
+ @counts[labels] ||= 0
53
+ buckets = @observations[labels]
54
+ if buckets.nil?
55
+ buckets = @buckets.map{|b| [b, 0]}.to_h
56
+ @observations[labels] = buckets
57
+ end
58
+ buckets
59
+ end
60
+
61
+ def fill_buckets(value, buckets)
62
+ @buckets.each do |b|
63
+ break if value > b
64
+ buckets[b] += 1
65
+ end
66
+ end
67
+
68
+ def with_bucket(labels, bucket)
69
+ labels.merge({"le" => bucket})
70
+ end
71
+
72
+ end
73
+ end
@@ -3,18 +3,19 @@
3
3
  module PrometheusExporter::Metric
4
4
  class Summary < Base
5
5
 
6
- QUANTILES = [0.99, 0.9, 0.5, 0.1, 0.01]
6
+ DEFAULT_QUANTILES = [0.99, 0.9, 0.5, 0.1, 0.01]
7
7
  ROTATE_AGE = 120
8
8
 
9
9
  attr_reader :estimators, :count, :total
10
10
 
11
- def initialize(name, help)
12
- super
11
+ def initialize(name, help, opts = {})
12
+ super(name, help)
13
13
  @buffers = [{}, {}]
14
14
  @last_rotated = Process.clock_gettime(Process::CLOCK_MONOTONIC)
15
15
  @current_buffer = 0
16
16
  @counts = {}
17
17
  @sums = {}
18
+ @quantiles = opts[:quantiles] || DEFAULT_QUANTILES
18
19
  end
19
20
 
20
21
  def type
@@ -27,7 +28,7 @@ module PrometheusExporter::Metric
27
28
  result = {}
28
29
 
29
30
  if length > 0
30
- QUANTILES.each do |quantile|
31
+ @quantiles.each do |quantile|
31
32
  result[quantile] = sorted[(length * quantile).ceil - 1]
32
33
  end
33
34
  end
@@ -48,7 +49,7 @@ module PrometheusExporter::Metric
48
49
  end
49
50
 
50
51
  def metric_text
51
- text = String.new
52
+ text = +""
52
53
  first = true
53
54
  calculate_all_quantiles.each do |labels, quantiles|
54
55
  text << "\n" unless first
@@ -1,4 +1,5 @@
1
1
  require_relative "metric/base"
2
2
  require_relative "metric/counter"
3
3
  require_relative "metric/gauge"
4
+ require_relative "metric/histogram"
4
5
  require_relative "metric/summary"
@@ -30,6 +30,7 @@ module PrometheusExporter::Server
30
30
  register_collector(ProcessCollector.new)
31
31
  register_collector(SidekiqCollector.new)
32
32
  register_collector(DelayedJobCollector.new)
33
+ register_collector(PumaCollector.new)
33
34
  end
34
35
 
35
36
  def register_collector(collector)
@@ -85,6 +86,8 @@ module PrometheusExporter::Server
85
86
  PrometheusExporter::Metric::Counter.new(name, help)
86
87
  when "summary"
87
88
  PrometheusExporter::Metric::Summary.new(name, help)
89
+ when "histogram"
90
+ PrometheusExporter::Metric::Histogram.new(name, help)
88
91
  end
89
92
 
90
93
  if metric
@@ -0,0 +1,48 @@
1
+ module PrometheusExporter::Server
2
+ class PumaCollector < TypeCollector
3
+ PUMA_GAUGES = {
4
+ workers_total: "Number of puma workers.",
5
+ booted_workers_total: "Number of puma workers booted.",
6
+ old_workers_total: "Number of old puma workers.",
7
+ running_threads_total: "Number of puma threads currently running.",
8
+ request_backlog_total: "Number of requests waiting to be processed by a puma thread.",
9
+ thread_pool_capacity_total: "Number of puma threads available at current scale.",
10
+ max_threads_total: "Number of puma threads at available at max scale.",
11
+ }
12
+
13
+ def initialize
14
+ @puma_metrics = []
15
+ end
16
+
17
+ def type
18
+ "puma"
19
+ end
20
+
21
+ def metrics
22
+ return [] if @puma_metrics.length == 0
23
+
24
+ metrics = {}
25
+
26
+ @puma_metrics.map do |m|
27
+ labels = {}
28
+ if m["phase"]
29
+ labels.merge(phase: m["phase"])
30
+ end
31
+
32
+ PUMA_GAUGES.map do |k, help|
33
+ k = k.to_s
34
+ if v = m[k]
35
+ g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new("puma_#{k}", help)
36
+ g.observe(v, labels)
37
+ end
38
+ end
39
+ end
40
+
41
+ metrics.values
42
+ end
43
+
44
+ def collect(obj)
45
+ @puma_metrics << obj
46
+ end
47
+ end
48
+ end
@@ -8,3 +8,4 @@ require_relative "server/collector_base"
8
8
  require_relative "server/collector"
9
9
  require_relative "server/web_server"
10
10
  require_relative "server/runner"
11
+ require_relative "server/puma_collector"
@@ -1,3 +1,3 @@
1
1
  module PrometheusExporter
2
- VERSION = "0.3.4"
2
+ VERSION = "0.4.0"
3
3
  end
@@ -30,5 +30,6 @@ Gem::Specification.new do |spec|
30
30
  spec.add_development_dependency "guard-minitest", "~> 2.0"
31
31
  spec.add_development_dependency "oj", "~> 3.0"
32
32
  spec.add_development_dependency "rack-test", "~> 0.8.3"
33
+ spec.add_development_dependency "minitest-stub-const", "~> 0.6"
33
34
  spec.required_ruby_version = '>= 2.3.0'
34
35
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: prometheus_exporter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sam Saffron
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-10-01 00:00:00.000000000 Z
11
+ date: 2018-10-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -122,6 +122,20 @@ dependencies:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
124
  version: 0.8.3
125
+ - !ruby/object:Gem::Dependency
126
+ name: minitest-stub-const
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '0.6'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '0.6'
125
139
  description: Prometheus metric collector and exporter for Ruby
126
140
  email:
127
141
  - sam.saffron@gmail.com
@@ -151,11 +165,13 @@ files:
151
165
  - lib/prometheus_exporter/instrumentation/global.rb
152
166
  - lib/prometheus_exporter/instrumentation/method_profiler.rb
153
167
  - lib/prometheus_exporter/instrumentation/process.rb
168
+ - lib/prometheus_exporter/instrumentation/puma.rb
154
169
  - lib/prometheus_exporter/instrumentation/sidekiq.rb
155
170
  - lib/prometheus_exporter/metric.rb
156
171
  - lib/prometheus_exporter/metric/base.rb
157
172
  - lib/prometheus_exporter/metric/counter.rb
158
173
  - lib/prometheus_exporter/metric/gauge.rb
174
+ - lib/prometheus_exporter/metric/histogram.rb
159
175
  - lib/prometheus_exporter/metric/summary.rb
160
176
  - lib/prometheus_exporter/middleware.rb
161
177
  - lib/prometheus_exporter/server.rb
@@ -163,6 +179,7 @@ files:
163
179
  - lib/prometheus_exporter/server/collector_base.rb
164
180
  - lib/prometheus_exporter/server/delayed_job_collector.rb
165
181
  - lib/prometheus_exporter/server/process_collector.rb
182
+ - lib/prometheus_exporter/server/puma_collector.rb
166
183
  - lib/prometheus_exporter/server/runner.rb
167
184
  - lib/prometheus_exporter/server/sidekiq_collector.rb
168
185
  - lib/prometheus_exporter/server/type_collector.rb