prometheus_exporter 0.5.0 → 0.7.0

data/lib/prometheus_exporter/server.rb

@@ -5,6 +5,7 @@ require_relative "server/type_collector"
 require_relative "server/web_collector"
 require_relative "server/process_collector"
 require_relative "server/sidekiq_collector"
+require_relative "server/sidekiq_queue_collector"
 require_relative "server/delayed_job_collector"
 require_relative "server/collector_base"
 require_relative "server/collector"

data/lib/prometheus_exporter/server/active_record_collector.rb

@@ -27,6 +27,7 @@ module PrometheusExporter::Server
 
       @active_record_metrics.map do |m|
         metric_key = (m["metric_labels"] || {}).merge("pid" => m["pid"])
+        metric_key.merge!(m["custom_labels"]) if m["custom_labels"]
 
         ACTIVE_RECORD_GAUGES.map do |k, help|
           k = k.to_s

data/lib/prometheus_exporter/server/collector.rb

@@ -13,6 +13,7 @@ module PrometheusExporter::Server
       register_collector(WebCollector.new)
       register_collector(ProcessCollector.new)
       register_collector(SidekiqCollector.new)
+      register_collector(SidekiqQueueCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)

data/lib/prometheus_exporter/server/delayed_job_collector.rb

@@ -2,6 +2,17 @@
 
 module PrometheusExporter::Server
   class DelayedJobCollector < TypeCollector
+    def initialize
+      @delayed_jobs_total = nil
+      @delayed_job_duration_seconds = nil
+      @delayed_jobs_total = nil
+      @delayed_failed_jobs_total = nil
+      @delayed_jobs_max_attempts_reached_total = nil
+      @delayed_job_duration_seconds_summary = nil
+      @delayed_job_attempts_summary = nil
+      @delayed_jobs_enqueued = nil
+      @delayed_jobs_pending = nil
+    end
 
     def type
       "delayed_job"

data/lib/prometheus_exporter/server/hutch_collector.rb

@@ -2,6 +2,12 @@
 
 module PrometheusExporter::Server
   class HutchCollector < TypeCollector
+    def initialize
+      @hutch_jobs_total = nil
+      @hutch_job_duration_seconds = nil
+      @hutch_jobs_total = nil
+      @hutch_failed_jobs_total = nil
+    end
 
     def type
       "hutch"

data/lib/prometheus_exporter/server/runner.rb

@@ -9,6 +9,15 @@ module PrometheusExporter::Server
 
   class Runner
     def initialize(options = {})
+      @timeout = nil
+      @port = nil
+      @bind = nil
+      @collector_class = nil
+      @type_collectors = nil
+      @prefix = nil
+      @auth = nil
+      @realm = nil
+
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
       end
@@ -16,6 +25,7 @@ module PrometheusExporter::Server
 
     def start
       PrometheusExporter::Metric::Base.default_prefix = prefix
+      PrometheusExporter::Metric::Base.default_labels = label
 
       register_type_collectors
 
@@ -32,12 +42,20 @@ module PrometheusExporter::Server
         )
       end
 
-      server = server_class.new port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose
+      server = server_class.new(port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose, auth: auth, realm: realm)
       server.start
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+
+    def auth
+      @auth || nil
+    end
+
+    def realm
+      @realm || PrometheusExporter::DEFAULT_REALM
+    end
 
     def prefix
       @prefix || PrometheusExporter::DEFAULT_PREFIX
@@ -76,6 +94,10 @@ module PrometheusExporter::Server
       @_collector ||= collector_class.new
     end
 
+    def label
+      @label ||= PrometheusExporter::DEFAULT_LABEL
+    end
+
     private
 
     def register_type_collectors

data/lib/prometheus_exporter/server/shoryuken_collector.rb

@@ -3,6 +3,14 @@
 module PrometheusExporter::Server
   class ShoryukenCollector < TypeCollector
 
+    def initialize
+      @shoryuken_jobs_total = nil
+      @shoryuken_job_duration_seconds = nil
+      @shoryuken_jobs_total = nil
+      @shoryuken_restarted_jobs_total = nil
+      @shoryuken_failed_jobs_total = nil
+    end
+
     def type
       "shoryuken"
     end

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -3,12 +3,21 @@
 module PrometheusExporter::Server
   class SidekiqCollector < TypeCollector
 
+    def initialize
+      @sidekiq_jobs_total = nil
+      @sidekiq_job_duration_seconds = nil
+      @sidekiq_jobs_total = nil
+      @sidekiq_restarted_jobs_total = nil
+      @sidekiq_failed_jobs_total = nil
+      @sidekiq_dead_jobs_total = nil
+    end
+
     def type
       "sidekiq"
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
+      default_labels = { job_name: obj['name'], queue: obj['queue'] }
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
@@ -43,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Counter.new(
+        PrometheusExporter::Metric::Summary.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =

data/lib/prometheus_exporter/server/sidekiq_queue_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+module PrometheusExporter::Server
+  class SidekiqQueueCollector < TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_QUEUE_GAUGES = {
+      'backlog_total' => 'Size of the sidekiq queue.',
+      'latency_seconds' => 'Latency of the sidekiq queue.',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_queue'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        labels = metric.fetch("labels", {})
+        SIDEKIQ_QUEUE_GAUGES.map do |name, help|
+          if (value = metric[name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_queue_#{name}", help)
+            gauge.observe(value, labels)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      object['queues'].each do |queue|
+        queue["created_at"] = now
+        queue["labels"].merge!(object['custom_labels']) if object['custom_labels']
+        sidekiq_metrics.delete_if { |metric| metric['created_at'] + MAX_SIDEKIQ_METRIC_AGE < now }
+        sidekiq_metrics << queue
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/server/web_collector.rb

@@ -4,6 +4,11 @@ module PrometheusExporter::Server
   class WebCollector < TypeCollector
     def initialize
       @metrics = {}
+      @http_requests_total = nil
+      @http_duration_seconds = nil
+      @http_redis_duration_seconds = nil
+      @http_sql_duration_seconds = nil
+      @http_queue_duration_seconds = nil
     end
 
     def type
@@ -51,14 +56,11 @@ module PrometheusExporter::Server
     end
 
     def observe(obj)
-      default_labels = {
-        controller: obj['controller'] || 'other',
-        action: obj['action'] || 'other'
-      }
+      default_labels = obj['default_labels']
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
-      @http_requests_total.observe(1, labels.merge(status: obj["status"]))
+      @http_requests_total.observe(1, labels)
 
       if timings = obj["timings"]
         @http_duration_seconds.observe(timings["total_duration"], labels)

data/lib/prometheus_exporter/server/web_server.rb

@@ -9,9 +9,14 @@ module PrometheusExporter::Server
   class WebServer
     attr_reader :collector
 
-    def initialize(port: , bind: nil, collector: nil, timeout: PrometheusExporter::DEFAULT_TIMEOUT, verbose: false)
-
-      @verbose = verbose
+    def initialize(opts)
+      @port = opts[:port] || PrometheusExporter::DEFAULT_PORT
+      @bind = opts[:bind] || PrometheusExporter::DEFAULT_BIND_ADDRESS
+      @collector = opts[:collector] || Collector.new
+      @timeout = opts[:timeout] || PrometheusExporter::DEFAULT_TIMEOUT
+      @verbose = opts[:verbose] || false
+      @auth = opts[:auth]
+      @realm = opts[:realm] || PrometheusExporter::DEFAULT_REALM
 
       @metrics_total = PrometheusExporter::Metric::Counter.new("collector_metrics_total", "Total metrics processed by exporter web.")
 
@@ -23,33 +28,33 @@ module PrometheusExporter::Server
       @sessions_total.observe(0)
       @bad_metrics_total.observe(0)
 
-      access_log, logger = nil
+      @access_log, @logger = nil
 
-      if verbose
-        access_log = [
+      if @verbose
+        @access_log = [
           [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT],
           [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT],
         ]
-        logger = WEBrick::Log.new($stderr)
+        @logger = WEBrick::Log.new($stderr)
       else
-        access_log = []
-        logger = WEBrick::Log.new("/dev/null")
+        @access_log = []
+        @logger = WEBrick::Log.new("/dev/null")
       end
 
+      @logger.info "Using Basic Authentication via #{@auth}" if @verbose && @auth
+
       @server = WEBrick::HTTPServer.new(
-        Port: port,
-        BindAddress: bind,
-        Logger: logger,
-        AccessLog: access_log,
+        Port: @port,
+        BindAddress: @bind,
+        Logger: @logger,
+        AccessLog: @access_log,
       )
 
-      @collector = collector || Collector.new
-      @port = port
-      @timeout = timeout
-
       @server.mount_proc '/' do |req, res|
         res['Content-Type'] = 'text/plain; charset=utf-8'
         if req.path == '/metrics'
+          authenticate(req, res) if @auth
+
           res.status = 200
           if req.header["accept-encoding"].to_s.include?("gzip")
             sio = StringIO.new
@@ -159,5 +164,12 @@ module PrometheusExporter::Server
       gauge
     end
 
+    def authenticate(req, res)
+      htpasswd = WEBrick::HTTPAuth::Htpasswd.new(@auth)
+      basic_auth = WEBrick::HTTPAuth::BasicAuth.new({ Realm: @realm, UserDB: htpasswd, Logger: @logger })
+
+      basic_auth.authenticate(req, res)
+    end
+
   end
 end

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '0.5.0'
+  VERSION = '0.7.0'
 end

data/prometheus_exporter.gemspec

@@ -24,19 +24,23 @@ Gem::Specification.new do |spec|
   spec.executables          = ["prometheus_exporter"]
   spec.require_paths        = ["lib"]
 
+  spec.add_dependency "webrick"
+
   spec.add_development_dependency "rubocop", ">= 0.69"
-  spec.add_development_dependency "bundler", "> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", ">= 2.2.2"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
-  spec.add_development_dependency "mini_racer", "~> 0.1"
+  spec.add_development_dependency "mini_racer", "~> 0.3.1"
   spec.add_development_dependency "guard-minitest", "~> 2.0"
   spec.add_development_dependency "oj", "~> 3.0"
   spec.add_development_dependency "rack-test", "~> 0.8.3"
   spec.add_development_dependency "minitest-stub-const", "~> 0.6"
-  spec.add_development_dependency 'rubocop-discourse', '~> 1.0'
+  spec.add_development_dependency "rubocop-discourse", ">2"
+  spec.add_development_dependency "appraisal", "~> 2.3"
+  spec.add_development_dependency "activerecord", "~> 6.0.0"
   if !RUBY_ENGINE == 'jruby'
     spec.add_development_dependency "raindrops", "~> 0.19"
   end
-  spec.required_ruby_version = '>= 2.3.0'
+  spec.required_ruby_version = '>= 2.5.0'
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Sam Saffron
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-14 00:00:00.000000000 Z
+date: 2020-12-28 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -28,30 +42,30 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.2.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.2.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.3.1
 - !ruby/object:Gem::Dependency
   name: guard-minitest
   requirement: !ruby/object:Gem::Requirement
@@ -152,18 +166,46 @@ dependencies:
         version: '0.6'
 - !ruby/object:Gem::Dependency
   name: rubocop-discourse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 6.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 6.0.0
 description: Prometheus metric collector and exporter for Ruby
 email:
 - sam.saffron@gmail.com
@@ -172,9 +214,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
+- Appraisals
 - CHANGELOG
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -185,6 +228,9 @@ files:
 - bench/bench.rb
 - bin/prometheus_exporter
 - examples/custom_collector.rb
+- gemfiles/.bundle/config
+- gemfiles/ar_60.gemfile
+- gemfiles/ar_61.gemfile
 - lib/prometheus_exporter.rb
 - lib/prometheus_exporter/client.rb
 - lib/prometheus_exporter/instrumentation.rb
@@ -196,6 +242,7 @@ files:
 - lib/prometheus_exporter/instrumentation/puma.rb
 - lib/prometheus_exporter/instrumentation/shoryuken.rb
 - lib/prometheus_exporter/instrumentation/sidekiq.rb
+- lib/prometheus_exporter/instrumentation/sidekiq_queue.rb
 - lib/prometheus_exporter/instrumentation/unicorn.rb
 - lib/prometheus_exporter/metric.rb
 - lib/prometheus_exporter/metric/base.rb
@@ -215,6 +262,7 @@ files:
 - lib/prometheus_exporter/server/runner.rb
 - lib/prometheus_exporter/server/shoryuken_collector.rb
 - lib/prometheus_exporter/server/sidekiq_collector.rb
+- lib/prometheus_exporter/server/sidekiq_queue_collector.rb
 - lib/prometheus_exporter/server/type_collector.rb
 - lib/prometheus_exporter/server/unicorn_collector.rb
 - lib/prometheus_exporter/server/web_collector.rb
@@ -234,15 +282,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.2
+signing_key:
 specification_version: 4
 summary: Prometheus Exporter
 test_files: []