prometheus_exporter 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f949348bbafc06b8f7a3dd3927693de8fef54090d85a7889afbfbf7dcb167b6
-  data.tar.gz: adeca8000ebb59c7225c1453135900aeab5c1760902edd5b4fecf9fa4b879a57
+  metadata.gz: 562647915cd81e672056a83fc79a031d2a0ebb2b4a7a5e32f5786ae5c7d480a7
+  data.tar.gz: 525e28d0cbe4e853c91bb463970fd231b134dba6b5a190c61a2ff7710074fb7b
 SHA512:
-  metadata.gz: 7d3829ad57f03a19f6081c3444f9b13906aedd4fa1c5b99237d454096e0d993c27c2df0993482c421b6034a58a55c9603620bcdf7698248e4ac23a5063d63718
-  data.tar.gz: fb915716acbbb4f24a152c8ebf9eb35bdc97185a7144eb986e50aafad07a36d5cc83cbbbf7c16e8b399207e9c66d90973d2e6f9ddcdc8a17cab797840d13ed8b
+  metadata.gz: 324b3b0acd1c97e8b5535dff8f57500b4dc3f2433456372c609accbc78d1baf839d82a1e38eabb543f043bbc9fefb401ab1460ba252ee8b9330fe0a91acffc1d
+  data.tar.gz: b2a0082d4e2431cc1f24f11a6af86d6965e2db7c93df1021ed58b38ff8651c84d1e8939b42a13b1b7cac60c9237435ff838a24918ed0ee18d13c3df2aece76d4

data/Appraisals

@@ -6,5 +6,5 @@ appraise "ar-60" do
 end
 
 appraise "ar-61" do
-  gem "activerecord", "~> 6.1.0.rc2"
+  gem "activerecord", "~> 6.1.1"
 end

data/CHANGELOG

@@ -1,3 +1,15 @@
+0.8.0 - 05-07-2021
+
+- FIX: handle ThreadError more gracefully in cases where process shuts down
+- FEATURE: add job_name and queue_name labels to delayed job metrics
+- FEATURE: always scope puma metrics on hostname in collector
+- FEATURE: add customizable labels option to puma collector
+- FEATURE: support for Rescue
+- DEV: Remove support for EOL ruby 2.5
+- FIX: Add source location to MethodProfiler patches
+- FEATURE: Improve Active Record instrumentation
+- FEATURE: Support HTTP_X_AMZN_TRACE_ID when supplied
+
 0.7.0 - 29-12-2020
 
 - Dev: Removed support from EOL rubies, only 2.5, 2.6, 2.7 and 3.0 are supported now.

data/README.md

@@ -19,6 +19,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a
     * [Hutch metrics](#hutch-message-processing-tracer)
   * [Puma metrics](#puma-metrics)
   * [Unicorn metrics](#unicorn-process-metrics)
+  * [Resque metrics](#resque-metrics)
   * [Custom type collectors](#custom-type-collectors)
   * [Multi process mode with custom collector](#multi-process-mode-with-custom-collector)
   * [GraphQL support](#graphql-support)
@@ -478,6 +479,8 @@ end
 | Summary | `delayed_job_duration_seconds_summary`    | Summary of the time it takes jobs to execute                       | `status`   |
 | Summary | `delayed_job_attempts_summary`            | Summary of the amount of attempts it takes delayed jobs to succeed | -          |
 
+All metrics have labels for `job_name` and `queue_name`.
+
 #### Hutch Message Processing Tracer
 
 Capture [Hutch](https://github.com/gocardless/hutch) metrics (how many jobs ran? how many failed? how long did they take?)
@@ -505,7 +508,7 @@ Request Queueing is defined as the time it takes for a request to reach your app
 
 As this metric starts before `prometheus_exporter` can handle the request, you must add a specific HTTP header as early in your infrastructure as possible (we recommend your load balancer or reverse proxy).
 
-Configure your HTTP server / load balancer to add a header `X-Request-Start: t=<MSEC>` when passing the request upstream. For more information, please consult your software manual.
+The Amazon Application Load Balancer [request tracing header](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html) is natively supported. If you are using another upstream entrypoint, you may configure your HTTP server / load balancer to add a header `X-Request-Start: t=<MSEC>` when passing the request upstream. For more information, please consult your software manual.
 
 Hint: we aim to be API-compatible with the big APM solutions, so if you've got requests queueing time configured for them, it should be expected to also work with `prometheus_exporter`.
 
@@ -535,7 +538,29 @@ end
 | Gauge | `puma_thread_pool_capacity_total` | Number of puma threads available at current scale           |
 | Gauge | `puma_max_threads_total`          | Number of puma threads at available at max scale            |
 
-All metrics may have a `phase` label.
+All metrics may have a `phase` label and all custom labels provided with the `labels` option.
+
+### Resque metrics
+
+The resque metrics are using the `Resque.info` method, which queries Redis internally. To start monitoring your resque
+installation, you'll need to start the instrumentation:
+
+```ruby
+# e.g. config/initializers/resque.rb
+require 'prometheus_exporter/instrumentation'
+PrometheusExporter::Instrumentation::Resque.start
+```
+
+#### Metrics collected by Resque Instrumentation
+
+| Type  | Name                   | Description                            |
+| ---   | ---                    | ---                                    |
+| Gauge | `processed_jobs_total` | Total number of processed Resque jobs  |
+| Gauge | `failed_jobs_total`    | Total number of failed Resque jobs     |
+| Gauge | `pending_jobs_total`   | Total number of pending Resque jobs    |
+| Gauge | `queues_total`         | Total number of Resque queues          |
+| Gauge | `workers_total`        | Total number of Resque workers running |
+| Gauge | `working_total`        | Total number of Resque workers working |
 
 ### Unicorn process metrics
 

data/lib/prometheus_exporter/client.rb

@@ -184,6 +184,9 @@ module PrometheusExporter
           end
         end
       end
+    rescue ThreadError => e
+      raise unless e.message =~ /can't alloc thread/
+      STDERR.puts "Prometheus Exporter, failed to send message ThreadError #{e}"
     end
 
     def close_socket!

data/lib/prometheus_exporter/instrumentation.rb

@@ -11,3 +11,4 @@ require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
 require_relative "instrumentation/active_record"
 require_relative "instrumentation/shoryuken"
+require_relative "instrumentation/resque"

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -81,15 +81,13 @@ module PrometheusExporter::Instrumentation
     private
 
     def labels(pool)
-      if pool.respond_to?(:spec) # ActiveRecord <= 6.0
+      if ::ActiveRecord.version < Gem::Version.new("6.1.0.rc1")
         @metric_labels.merge(pool_name: pool.spec.name).merge(pool.spec.config
           .select { |k, v| @config_labels.include? k }
           .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }.to_h)
-      elsif pool.respond_to?(:db_config) # ActiveRecord >= 6.1.rc1
+      else
         @metric_labels.merge(pool_name: pool.db_config.name).merge(
           @config_labels.each_with_object({}) { |l, acc| acc["dbconfig_#{l}"] = pool.db_config.public_send(l) })
-      else
-        raise "Unsupported connection pool"
       end
     end
   end

data/lib/prometheus_exporter/instrumentation/delayed_job.rb

@@ -13,8 +13,8 @@ module PrometheusExporter::Instrumentation
           callbacks do |lifecycle|
             lifecycle.around(:invoke_job) do |job, *args, &block|
               max_attempts = Delayed::Worker.max_attempts
-              enqueued_count = Delayed::Job.count
-              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil).count
+              enqueued_count = Delayed::Job.where(queue: job.queue).count
+              pending_count = Delayed::Job.where(attempts: 0, locked_at: nil, queue: job.queue).count
               instrumenter.call(job, max_attempts, enqueued_count, pending_count, *args, &block)
             end
           end
@@ -41,6 +41,7 @@ module PrometheusExporter::Instrumentation
       @client.send_json(
         type: "delayed_job",
         name: job.handler.to_s.match(JOB_CLASS_REGEXP).to_a[1].to_s,
+        queue_name: job.queue,
         success: success,
         duration: duration,
         attempts: attempts,

data/lib/prometheus_exporter/instrumentation/method_profiler.rb

@@ -5,6 +5,7 @@ module PrometheusExporter::Instrumentation; end
 
 class PrometheusExporter::Instrumentation::MethodProfiler
   def self.patch(klass, methods, name)
+    patch_source_line = __LINE__ + 3
     patches = methods.map do |method_name|
       <<~RUBY
       unless defined?(#{method_name}__mp_unpatched)
@@ -26,7 +27,7 @@ class PrometheusExporter::Instrumentation::MethodProfiler
       RUBY
     end.join("\n")
 
-    klass.class_eval patches
+    klass.class_eval patches, __FILE__, patch_source_line
   end
 
   def self.transfer

data/lib/prometheus_exporter/instrumentation/puma.rb

@@ -5,8 +5,8 @@ require "json"
 # collects stats from puma
 module PrometheusExporter::Instrumentation
   class Puma
-    def self.start(client: nil, frequency: 30)
-      puma_collector = new
+    def self.start(client: nil, frequency: 30, labels: {})
+      puma_collector = new(labels)
       client ||= PrometheusExporter::Client.default
       Thread.new do
         while true
@@ -22,13 +22,25 @@ module PrometheusExporter::Instrumentation
       end
     end
 
+    def initialize(metric_labels = {})
+      @metric_labels = metric_labels
+    end
+
     def collect
-      metric = {}
-      metric[:type] = "puma"
+      metric = {
+        pid: pid,
+        type: "puma",
+        hostname: ::PrometheusExporter.hostname,
+        metric_labels: @metric_labels
+      }
       collect_puma_stats(metric)
       metric
     end
 
+    def pid
+      @pid = ::Process.pid
+    end
+
     def collect_puma_stats(metric)
       stats = JSON.parse(::Puma.stats)
 

data/lib/prometheus_exporter/instrumentation/resque.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# collects stats from resque
+module PrometheusExporter::Instrumentation
+  class Resque
+    def self.start(client: nil, frequency: 30)
+      resque_collector = new
+      client ||= PrometheusExporter::Client.default
+      Thread.new do
+        while true
+          begin
+            client.send_json(resque_collector.collect)
+          rescue => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Resque Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def collect
+      metric = {}
+      metric[:type] = "resque"
+      collect_resque_stats(metric)
+      metric
+    end
+
+    def collect_resque_stats(metric)
+      info = ::Resque.info
+
+      metric[:processed_jobs_total] = info[:processed]
+      metric[:failed_jobs_total] = info[:failed]
+      metric[:pending_jobs_total] = info[:pending]
+      metric[:queues_total] = info[:queues]
+      metric[:worker_total] = info[:workers]
+      metric[:working_total] = info[:working]
+    end
+  end
+end

data/lib/prometheus_exporter/middleware.rb

@@ -90,19 +90,24 @@ class PrometheusExporter::Middleware
     Process.clock_gettime(Process::CLOCK_REALTIME)
   end
 
-  # get the content of the x-queue-start or x-request-start header
+  # determine queue start from well-known trace headers
   def queue_start(env)
+
+    # get the content of the x-queue-start or x-request-start header
     value = env['HTTP_X_REQUEST_START'] || env['HTTP_X_QUEUE_START']
     unless value.nil? || value == ''
-      convert_header_to_ms(value.to_s)
+      # nginx returns time as milliseconds with 3 decimal places
+      # apache returns time as microseconds without decimal places
+      # this method takes care to convert both into a proper second + fractions timestamp
+      value = value.to_s.gsub(/t=|\./, '')
+      return "#{value[0, 10]}.#{value[10, 13]}".to_f
     end
-  end
 
-  # nginx returns time as milliseconds with 3 decimal places
-  # apache returns time as microseconds without decimal places
-  # this method takes care to convert both into a proper second + fractions timestamp
-  def convert_header_to_ms(str)
-    str = str.gsub(/t=|\./, '')
-    "#{str[0, 10]}.#{str[10, 13]}".to_f
+    # get the content of the x-amzn-trace-id header
+    # see also: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html
+    value = env['HTTP_X_AMZN_TRACE_ID']
+    value&.split('Root=')&.last&.split('-')&.fetch(1)&.to_i(16)
+
   end
+
 end

data/lib/prometheus_exporter/server.rb

@@ -16,3 +16,4 @@ require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
 require_relative "server/active_record_collector"
 require_relative "server/shoryuken_collector"
+require_relative "server/resque_collector"

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -47,7 +47,8 @@ module PrometheusExporter::Server
       obj["created_at"] = now
 
       @active_record_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"] &&
+         obj["metric_labels"]["pool_name"] == current["metric_labels"]["pool_name"]) ||
           (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
       end
 

data/lib/prometheus_exporter/server/collector.rb

@@ -20,6 +20,7 @@ module PrometheusExporter::Server
       register_collector(UnicornCollector.new)
       register_collector(ActiveRecordCollector.new)
       register_collector(ShoryukenCollector.new)
+      register_collector(ResqueCollector.new)
     end
 
     def register_collector(collector)

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -19,21 +19,22 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
+      default_labels = { job_name: obj['name'], queue_name: obj['queue_name'] }
       custom_labels = obj['custom_labels']
+
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
       ensure_delayed_job_metrics
       @delayed_job_duration_seconds.observe(obj["duration"], labels)
       @delayed_jobs_total.observe(1, labels)
       @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"])
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"])
-      @delayed_jobs_pending.observe(obj["pending"])
+      @delayed_jobs_max_attempts_reached_total.observe(1, labels) if obj["attempts"] >= obj["max_attempts"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels)
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "success")) if obj["success"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "failed"))  if !obj["success"]
+      @delayed_job_attempts_summary.observe(obj["attempts"], labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], labels)
+      @delayed_jobs_pending.observe(obj["pending"], labels)
     end
 
     def metrics

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -34,6 +34,9 @@ module PrometheusExporter::Server
         if m["custom_labels"]
           labels.merge!(m["custom_labels"])
         end
+        if m["metric_labels"]
+          labels.merge!(m["metric_labels"])
+        end
 
         PUMA_GAUGES.map do |k, help|
           k = k.to_s
@@ -51,7 +54,12 @@ module PrometheusExporter::Server
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
 
       obj["created_at"] = now
-      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
+
+      @puma_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_PUMA_METRIC_AGE < now)
+      end
+
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ResqueCollector < TypeCollector
+    MAX_RESQUE_METRIC_AGE = 30
+    RESQUE_GAUGES = {
+      processed_jobs_total: "Total number of processed Resque jobs.",
+      failed_jobs_total: "Total number of failed Resque jobs.",
+      pending_jobs_total: "Total number of pending Resque jobs.",
+      queues_total: "Total number of Resque queues.",
+      workers_total: "Total number of Resque workers running.",
+      working_total: "Total number of Resque workers working."
+    }
+
+    def initialize
+      @resque_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      "resque"
+    end
+
+    def metrics
+      return [] if resque_metrics.length == 0
+
+      resque_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        RESQUE_GAUGES.map do |name, help|
+          name = name.to_s
+          if value = metric[name]
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("resque_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      object["created_at"] = now
+      resque_metrics.delete_if { |metric| metric["created_at"] + MAX_RESQUE_METRIC_AGE < now }
+      resque_metrics << object
+    end
+
+    private
+
+    attr_reader :resque_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

data/prometheus_exporter.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "webrick"
 
   spec.add_development_dependency "rubocop", ">= 0.69"
-  spec.add_development_dependency "bundler", ">= 2.2.2"
+  spec.add_development_dependency "bundler", ">= 2.1.4"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
@@ -42,5 +42,5 @@ Gem::Specification.new do |spec|
   if !RUBY_ENGINE == 'jruby'
     spec.add_development_dependency "raindrops", "~> 0.19"
   end
-  spec.required_ruby_version = '>= 2.5.0'
+  spec.required_ruby_version = '>= 2.6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Sam Saffron
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-28 00:00:00.000000000 Z
+date: 2021-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webrick
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.1.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -240,6 +240,7 @@ files:
 - lib/prometheus_exporter/instrumentation/method_profiler.rb
 - lib/prometheus_exporter/instrumentation/process.rb
 - lib/prometheus_exporter/instrumentation/puma.rb
+- lib/prometheus_exporter/instrumentation/resque.rb
 - lib/prometheus_exporter/instrumentation/shoryuken.rb
 - lib/prometheus_exporter/instrumentation/sidekiq.rb
 - lib/prometheus_exporter/instrumentation/sidekiq_queue.rb
@@ -259,6 +260,7 @@ files:
 - lib/prometheus_exporter/server/hutch_collector.rb
 - lib/prometheus_exporter/server/process_collector.rb
 - lib/prometheus_exporter/server/puma_collector.rb
+- lib/prometheus_exporter/server/resque_collector.rb
 - lib/prometheus_exporter/server/runner.rb
 - lib/prometheus_exporter/server/shoryuken_collector.rb
 - lib/prometheus_exporter/server/sidekiq_collector.rb
@@ -282,14 +284,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.2
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Prometheus Exporter