prometheus_exporter 0.5.3 → 0.8.1

data/lib/prometheus_exporter/instrumentation/unicorn.rb

@@ -18,7 +18,7 @@ module PrometheusExporter::Instrumentation
             metric = unicorn_collector.collect
             client.send_json metric
           rescue StandardError => e
-            STDERR.puts("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
+            client.logger.error("Prometheus Exporter Failed To Collect Unicorn Stats #{e}")
           ensure
             sleep frequency
           end

data/lib/prometheus_exporter/middleware.rb

@@ -36,22 +36,40 @@ class PrometheusExporter::Middleware
 
     result
   ensure
+
+    obj = {
+      type: "web",
+      timings: info,
+      queue_time: queue_time,
+      default_labels: default_labels(env, result)
+    }
+    labels = custom_labels(env)
+    if labels
+      obj = obj.merge(custom_labels: labels)
+    end
+
+    @client.send_json(obj)
+  end
+
+  def default_labels(env, result)
     status = (result && result[0]) || -1
     params = env["action_dispatch.request.parameters"]
-    action, controller = nil
+    action = controller = nil
     if params
       action = params["action"]
       controller = params["controller"]
     end
 
-    @client.send_json(
-      type: "web",
-      timings: info,
-      queue_time: queue_time,
-      action: action,
-      controller: controller,
+    {
+      action: action || "other",
+      controller: controller || "other",
       status: status
-    )
+    }
+  end
+
+  # allows subclasses to add custom labels based on env
+  def custom_labels(env)
+    nil
   end
 
   private
@@ -72,19 +90,24 @@ class PrometheusExporter::Middleware
     Process.clock_gettime(Process::CLOCK_REALTIME)
   end
 
-  # get the content of the x-queue-start or x-request-start header
+  # determine queue start from well-known trace headers
   def queue_start(env)
+
+    # get the content of the x-queue-start or x-request-start header
     value = env['HTTP_X_REQUEST_START'] || env['HTTP_X_QUEUE_START']
     unless value.nil? || value == ''
-      convert_header_to_ms(value.to_s)
+      # nginx returns time as milliseconds with 3 decimal places
+      # apache returns time as microseconds without decimal places
+      # this method takes care to convert both into a proper second + fractions timestamp
+      value = value.to_s.gsub(/t=|\./, '')
+      return "#{value[0, 10]}.#{value[10, 13]}".to_f
     end
-  end
 
-  # nginx returns time as milliseconds with 3 decimal places
-  # apache returns time as microseconds without decimal places
-  # this method takes care to convert both into a proper second + fractions timestamp
-  def convert_header_to_ms(str)
-    str = str.gsub(/t=|\./, '')
-    "#{str[0, 10]}.#{str[10, 13]}".to_f
+    # get the content of the x-amzn-trace-id header
+    # see also: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-request-tracing.html
+    value = env['HTTP_X_AMZN_TRACE_ID']
+    value&.split('Root=')&.last&.split('-')&.fetch(1)&.to_i(16)
+
   end
+
 end

data/lib/prometheus_exporter/server.rb

@@ -16,3 +16,4 @@ require_relative "server/hutch_collector"
 require_relative "server/unicorn_collector"
 require_relative "server/active_record_collector"
 require_relative "server/shoryuken_collector"
+require_relative "server/resque_collector"

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -47,7 +47,8 @@ module PrometheusExporter::Server
       obj["created_at"] = now
 
       @active_record_metrics.delete_if do |current|
-        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"] &&
+         obj["metric_labels"]["pool_name"] == current["metric_labels"]["pool_name"]) ||
           (current["created_at"] + MAX_ACTIVERECORD_METRIC_AGE < now)
       end
 

data/lib/prometheus_exporter/server/collector.rb

@@ -20,6 +20,7 @@ module PrometheusExporter::Server
       register_collector(UnicornCollector.new)
       register_collector(ActiveRecordCollector.new)
       register_collector(ShoryukenCollector.new)
+      register_collector(ResqueCollector.new)
     end
 
     def register_collector(collector)

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -19,21 +19,22 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
+      default_labels = { job_name: obj['name'], queue_name: obj['queue_name'] }
       custom_labels = obj['custom_labels']
+
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
       ensure_delayed_job_metrics
       @delayed_job_duration_seconds.observe(obj["duration"], labels)
       @delayed_jobs_total.observe(1, labels)
       @delayed_failed_jobs_total.observe(1, labels) if !obj["success"]
-      @delayed_jobs_max_attempts_reached_total.observe(1) if obj["attempts"] >= obj["max_attempts"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"])
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "success") if obj["success"]
-      @delayed_job_duration_seconds_summary.observe(obj["duration"], status: "failed")  if !obj["success"]
-      @delayed_job_attempts_summary.observe(obj["attempts"]) if obj["success"]
-      @delayed_jobs_enqueued.observe(obj["enqueued"])
-      @delayed_jobs_pending.observe(obj["pending"])
+      @delayed_jobs_max_attempts_reached_total.observe(1, labels) if obj["attempts"] >= obj["max_attempts"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels)
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "success")) if obj["success"]
+      @delayed_job_duration_seconds_summary.observe(obj["duration"], labels.merge(status: "failed"))  if !obj["success"]
+      @delayed_job_attempts_summary.observe(obj["attempts"], labels) if obj["success"]
+      @delayed_jobs_enqueued.observe(obj["enqueued"], labels)
+      @delayed_jobs_pending.observe(obj["pending"], labels)
     end
 
     def metrics

data/lib/prometheus_exporter/server/puma_collector.rb

@@ -34,6 +34,9 @@ module PrometheusExporter::Server
         if m["custom_labels"]
           labels.merge!(m["custom_labels"])
         end
+        if m["metric_labels"]
+          labels.merge!(m["metric_labels"])
+        end
 
         PUMA_GAUGES.map do |k, help|
           k = k.to_s
@@ -51,7 +54,12 @@ module PrometheusExporter::Server
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
 
       obj["created_at"] = now
-      @puma_metrics.delete_if { |m| m["created_at"] + MAX_PUMA_METRIC_AGE < now }
+
+      @puma_metrics.delete_if do |current|
+        (obj["pid"] == current["pid"] && obj["hostname"] == current["hostname"]) ||
+          (current["created_at"] + MAX_PUMA_METRIC_AGE < now)
+      end
+
       @puma_metrics << obj
     end
   end

data/lib/prometheus_exporter/server/resque_collector.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class ResqueCollector < TypeCollector
+    MAX_RESQUE_METRIC_AGE = 30
+    RESQUE_GAUGES = {
+      processed_jobs_total: "Total number of processed Resque jobs.",
+      failed_jobs_total: "Total number of failed Resque jobs.",
+      pending_jobs_total: "Total number of pending Resque jobs.",
+      queues_total: "Total number of Resque queues.",
+      workers_total: "Total number of Resque workers running.",
+      working_total: "Total number of Resque workers working."
+    }
+
+    def initialize
+      @resque_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      "resque"
+    end
+
+    def metrics
+      return [] if resque_metrics.length == 0
+
+      resque_metrics.map do |metric|
+        labels = metric.fetch("custom_labels", {})
+
+        RESQUE_GAUGES.map do |name, help|
+          name = name.to_s
+          if value = metric[name]
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("resque_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      object["created_at"] = now
+      resque_metrics.delete_if { |metric| metric["created_at"] + MAX_RESQUE_METRIC_AGE < now }
+      resque_metrics << object
+    end
+
+    private
+
+    attr_reader :resque_metrics, :gauges
+  end
+end

data/lib/prometheus_exporter/server/runner.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'prometheus_exporter/client'
+require_relative '../client'
 require_relative '../instrumentation/unicorn'
 
 module PrometheusExporter::Server
@@ -15,6 +15,8 @@ module PrometheusExporter::Server
       @collector_class = nil
       @type_collectors = nil
       @prefix = nil
+      @auth = nil
+      @realm = nil
 
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
@@ -40,12 +42,20 @@ module PrometheusExporter::Server
         )
       end
 
-      server = server_class.new port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose
+      server = server_class.new(port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose, auth: auth, realm: realm)
       server.start
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+
+    def auth
+      @auth || nil
+    end
+
+    def realm
+      @realm || PrometheusExporter::DEFAULT_REALM
+    end
 
     def prefix
       @prefix || PrometheusExporter::DEFAULT_PREFIX

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -17,7 +17,7 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
+      default_labels = { job_name: obj['name'], queue: obj['queue'] }
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
@@ -52,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Counter.new(
+        PrometheusExporter::Metric::Summary.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =

data/lib/prometheus_exporter/server/web_collector.rb

@@ -56,14 +56,11 @@ module PrometheusExporter::Server
     end
 
     def observe(obj)
-      default_labels = {
-        controller: obj['controller'] || 'other',
-        action: obj['action'] || 'other'
-      }
+      default_labels = obj['default_labels']
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
-      @http_requests_total.observe(1, labels.merge(status: obj["status"]))
+      @http_requests_total.observe(1, labels)
 
       if timings = obj["timings"]
         @http_duration_seconds.observe(timings["total_duration"], labels)

data/lib/prometheus_exporter/server/web_server.rb

@@ -9,9 +9,14 @@ module PrometheusExporter::Server
   class WebServer
     attr_reader :collector
 
-    def initialize(port: , bind: nil, collector: nil, timeout: PrometheusExporter::DEFAULT_TIMEOUT, verbose: false)
-
-      @verbose = verbose
+    def initialize(opts)
+      @port = opts[:port] || PrometheusExporter::DEFAULT_PORT
+      @bind = opts[:bind] || PrometheusExporter::DEFAULT_BIND_ADDRESS
+      @collector = opts[:collector] || Collector.new
+      @timeout = opts[:timeout] || PrometheusExporter::DEFAULT_TIMEOUT
+      @verbose = opts[:verbose] || false
+      @auth = opts[:auth]
+      @realm = opts[:realm] || PrometheusExporter::DEFAULT_REALM
 
       @metrics_total = PrometheusExporter::Metric::Counter.new("collector_metrics_total", "Total metrics processed by exporter web.")
 
@@ -23,33 +28,34 @@ module PrometheusExporter::Server
       @sessions_total.observe(0)
       @bad_metrics_total.observe(0)
 
-      access_log, logger = nil
+      @access_log, @logger = nil
+      log_target = opts[:log_target]
 
-      if verbose
-        access_log = [
+      if @verbose
+        @access_log = [
           [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT],
           [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT],
         ]
-        logger = WEBrick::Log.new($stderr)
+        @logger = WEBrick::Log.new(log_target || $stderr)
       else
-        access_log = []
-        logger = WEBrick::Log.new("/dev/null")
+        @access_log = []
+        @logger = WEBrick::Log.new(log_target || "/dev/null")
       end
 
+      @logger.info "Using Basic Authentication via #{@auth}" if @verbose && @auth
+
       @server = WEBrick::HTTPServer.new(
-        Port: port,
-        BindAddress: bind,
-        Logger: logger,
-        AccessLog: access_log,
+        Port: @port,
+        BindAddress: @bind,
+        Logger: @logger,
+        AccessLog: @access_log,
       )
 
-      @collector = collector || Collector.new
-      @port = port
-      @timeout = timeout
-
       @server.mount_proc '/' do |req, res|
         res['Content-Type'] = 'text/plain; charset=utf-8'
         if req.path == '/metrics'
+          authenticate(req, res) if @auth
+
           res.status = 200
           if req.header["accept-encoding"].to_s.include?("gzip")
             sio = StringIO.new
@@ -82,10 +88,7 @@ module PrometheusExporter::Server
           @collector.process(block)
         rescue => e
           if @verbose
-            STDERR.puts
-            STDERR.puts e.inspect
-            STDERR.puts e.backtrace
-            STDERR.puts
+            logger.error "\n\n#{e.inspect}\n#{e.backtrace}\n\n"
           end
           @bad_metrics_total.observe
           res.body = "Bad Metrics #{e}"
@@ -103,7 +106,7 @@ module PrometheusExporter::Server
         begin
           @server.start
         rescue => e
-          STDERR.puts "Failed to start prometheus collector web on port #{@port}: #{e}"
+          logger.error "Failed to start prometheus collector web on port #{@port}: #{e}"
         end
       end
     end
@@ -120,7 +123,7 @@ module PrometheusExporter::Server
         end
       rescue Timeout::Error
         # we timed out ... bummer
-        STDERR.puts "Generating Prometheus metrics text timed out"
+        logger.error "Generating Prometheus metrics text timed out"
       end
 
       metrics = []
@@ -159,5 +162,12 @@ module PrometheusExporter::Server
       gauge
     end
 
+    def authenticate(req, res)
+      htpasswd = WEBrick::HTTPAuth::Htpasswd.new(@auth)
+      basic_auth = WEBrick::HTTPAuth::BasicAuth.new({ Realm: @realm, UserDB: htpasswd, Logger: @logger })
+
+      basic_auth.authenticate(req, res)
+    end
+
   end
 end

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '0.5.3'
+  VERSION = '0.8.1'
 end

data/prometheus_exporter.gemspec

@@ -24,19 +24,23 @@ Gem::Specification.new do |spec|
   spec.executables          = ["prometheus_exporter"]
   spec.require_paths        = ["lib"]
 
+  spec.add_dependency "webrick"
+
   spec.add_development_dependency "rubocop", ">= 0.69"
-  spec.add_development_dependency "bundler", "> 1.16"
+  spec.add_development_dependency "bundler", ">= 2.1.4"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
-  spec.add_development_dependency "mini_racer", "~> 0.1"
+  spec.add_development_dependency "mini_racer", "~> 0.3.1"
   spec.add_development_dependency "guard-minitest", "~> 2.0"
   spec.add_development_dependency "oj", "~> 3.0"
   spec.add_development_dependency "rack-test", "~> 0.8.3"
   spec.add_development_dependency "minitest-stub-const", "~> 0.6"
   spec.add_development_dependency "rubocop-discourse", ">2"
+  spec.add_development_dependency "appraisal", "~> 2.3"
+  spec.add_development_dependency "activerecord", "~> 6.0.0"
   if !RUBY_ENGINE == 'jruby'
     spec.add_development_dependency "raindrops", "~> 0.19"
   end
-  spec.required_ruby_version = '>= 2.3.0'
+  spec.required_ruby_version = '>= 2.6.0'
 end