prometheus_exporter 0.5.3 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a284b77cbd17e80457e94520d18976dc3dd6fb8c238d8a90cda6513e291f01ad
-  data.tar.gz: 4464419fc9c67d387edd9d6d5bdfac1411d0864d29c0cd3a6f10d2f02b3d7e24
+  metadata.gz: f516c39448418a2216851149f0479f3f9e0f5feadae5a5468e7953fc0551d318
+  data.tar.gz: dcc937b79e05d4cd74a64ab2135425c75c4bba1be053bec99e8db0695a3ac998
 SHA512:
-  metadata.gz: 84c2e2af95a8839cd9deecca9fbc26bacff590ae5cb3677862f1eb1d476401114406130cc78d2248cec0380e70c1b74b306e1265b3343bbac9c09d8400e692a3
-  data.tar.gz: 41898546588ef7eafa772252ef5ed1a8c222e4009e64ccec35d463217336597a94f1c844f3e2174e030c19a7c1b90d1028e5729cc393cf797654362f5d66b7f2
+  metadata.gz: 5664f24c1c4a1520bafe789e5df8a9f1c40b118933cab9b73baa953f633ec6e81c3a11bbc9fc63097a2a2acdd2390daeade9449e6451cc83207fc2162575b359
+  data.tar.gz: ca4aeedbc6e211818569257e8a3e9fde93be363a2da246abae45749bc9cc232052ac52f0850cd54d0f600904bed193a10db1f76e4fc9567b837b20a78264158b

data/.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: Test Exporter
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 0 * * 0' # weekly
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }}
+    strategy:
+      matrix:
+        ruby: ["2.7", "2.6", "2.5", "2.4"]
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.ruby }}-gems-
+      - name: Setup gems
+        run: |
+          bundle config path vendor/bundle
+          bundle install --jobs 4
+      - name: Rubocop
+        run: bundle exec rubocop
+      - name: Run tests
+        run: bundle exec rake

data/CHANGELOG

@@ -1,3 +1,7 @@
+0.6.0 - 10-11-2020
+
+- FEATURE: add support for basic-auth in the prometheus_exporter web server
+
 0.5.3 - 29-07-2020
 
 - FEATURE: added #remove to all metric types so users can remove specific labels if needed

data/README.md

@@ -200,12 +200,27 @@ $ bundle exec prometheus_exporter
 | Summary | `http_sql_duration_seconds`²    | Time spent in HTTP reqs in SQL in seconds                   |
 | Summary | `http_queue_duration_seconds`³  | Time spent queueing the request in load balancer in seconds |
 
-All metrics have a `controller` and an `action` label.  
-`http_requests_total` additionally has a (HTTP response) `status` label.  
+All metrics have a `controller` and an `action` label.
+`http_requests_total` additionally has a (HTTP response) `status` label.
 
-¹) Only available when Redis is used.  
-²) Only available when Mysql or PostgreSQL are used.  
-³) Only available when [Instrumenting Request Queueing Time](#instrumenting-request-queueing-time) is set up.  
+To add your own labels to the default metrics, create a subclass of `PrometheusExporter::Middleware`, override `custom_labels`, and use it in your initializer.
+```ruby
+class MyMiddleware < PrometheusExporter::Middleware
+  def custom_labels(env)
+    labels = {}
+
+    if env['HTTP_X_PLATFORM']
+      labels['platform'] = env['HTTP_X_PLATFORM']
+    end
+
+    labels
+  end
+end
+```
+
+¹) Only available when Redis is used.
+²) Only available when Mysql or PostgreSQL are used.
+³) Only available when [Instrumenting Request Queueing Time](#instrumenting-request-queueing-time) is set up.
 
 #### Activerecord Connection Pool Metrics
 
@@ -365,19 +380,19 @@ Sometimes the Sidekiq server shuts down before it can send metrics, that were ge
 **PrometheusExporter::Instrumentation::Sidekiq**
 | Type    | Name                           | Description                                                                  |
 | ---     | ---                            | ---                                                                          |
-| Counter | `sidekiq_job_duration_seconds` | Total time spent in sidekiq jobs                                             |
+| Summary | `sidekiq_job_duration_seconds` | Time spent in sidekiq jobs                                                   |
 | Counter | `sidekiq_jobs_total`           | Total number of sidekiq jobs executed                                        |
 | Counter | `sidekiq_restarted_jobs_total` | Total number of sidekiq jobs that we restarted because of a sidekiq shutdown |
 | Counter | `sidekiq_failed_jobs_total`    | Total number of failed sidekiq jobs                                          |
 
-All metrics have a `job_name` label.
+All metrics have a `job_name` label and a `queue` label.
 
 **PrometheusExporter::Instrumentation::Sidekiq.death_handler**
 | Type    | Name                      | Description                       |
 | ---     | ---                       | ---                               |
 | Counter | `sidekiq_dead_jobs_total` | Total number of dead sidekiq jobs |
 
-This metric also has a `job_name` label.
+This metric has a `job_name` label and a `queue` label.
 
 **PrometheusExporter::Instrumentation::SidekiqQueue**
 | Type  | Name                            | Description                  |
@@ -387,7 +402,7 @@ This metric also has a `job_name` label.
 
 Both metrics will have a `queue` label with the name of the queue.
 
-_See [Metrics collected by Process Instrumentation](#metrics-collected-by-process-instrumentation) for a list of metrics the Process instrumentation will produce._  
+_See [Metrics collected by Process Instrumentation](#metrics-collected-by-process-instrumentation) for a list of metrics the Process instrumentation will produce._
 
 #### Shoryuken metrics
 
@@ -411,7 +426,7 @@ end
 | Counter | `shoryuken_restarted_jobs_total` | Total number of shoryuken jobs that we restarted because of a shoryuken shutdown |
 | Counter | `shoryuken_failed_jobs_total`    | Total number of failed shoryuken jobs                                            |
 
-All metrics have labels for `job_name` and `queue_name`.  
+All metrics have labels for `job_name` and `queue_name`.
 
 #### Delayed Job plugin
 
@@ -456,7 +471,7 @@ end
 | Counter | `hutch_jobs_total`           | Total number of hutch jobs executed     |
 | Counter | `hutch_failed_jobs_total`    | Total number failed hutch jobs executed |
 
-All metrics have a `job_name` label. 
+All metrics have a `job_name` label.
 
 #### Instrumenting Request Queueing Time
 
@@ -494,7 +509,7 @@ end
 | Gauge | `puma_thread_pool_capacity_total` | Number of puma threads available at current scale           |
 | Gauge | `puma_max_threads_total`          | Number of puma threads at available at max scale            |
 
-All metrics may have a `phase` label.   
+All metrics may have a `phase` label.
 
 ### Unicorn process metrics
 
@@ -577,8 +592,8 @@ Then you can collect the metrics you need on demand:
 
 ```ruby
 def metrics
-  user_count_gague = PrometheusExporter::Metric::Gauge.new('user_count', 'number of users in the app')
-  user_count_gague.observe User.count
+  user_count_gauge = PrometheusExporter::Metric::Gauge.new('user_count', 'number of users in the app')
+  user_count_gauge.observe User.count
   [user_count_gauge]
 end
 ```
@@ -692,6 +707,25 @@ ruby_web_requests{hostname="app-server-01"} 1
 When running the process for `prometheus_exporter` using `bin/prometheus_exporter`, there are several configurations that
 can be passed in:
 
+```
+Usage: prometheus_exporter [options]
+    -p, --port INTEGER               Port exporter should listen on (default: 9394)
+    -b, --bind STRING                IP address exporter should listen on (default: localhost)
+    -t, --timeout INTEGER            Timeout in seconds for metrics endpoint (default: 2)
+        --prefix METRIC_PREFIX       Prefix to apply to all metrics (default: ruby_)
+        --label METRIC_LABEL         Label to apply to all metrics (default: {})
+    -c, --collector FILE             (optional) Custom collector to run
+    -a, --type-collector FILE        (optional) Custom type collectors to run in main collector
+    -v, --verbose
+        --auth FILE                  (optional) enable basic authentication using a htpasswd FILE
+        --realm REALM                (optional) Use REALM for basic authentication (default: "Prometheus Exporter")
+        --unicorn-listen-address ADDRESS
+                                     (optional) Address where unicorn listens on (unix or TCP address)
+        --unicorn-master PID_FILE    (optional) PID file of unicorn master process to monitor unicorn
+```
+
+#### Example
+
 The following will run the process at
 - Port `8080` (default `9394`)
 - Bind to `0.0.0.0` (default `localhost`)
@@ -707,6 +741,29 @@ prometheus_exporter -p 8080 \
                     --prefix 'foo_'
 ```
 
+#### Enabling Basic Authentication
+
+If you desire authentication on your `/metrics` route, you can enable basic authentication with the `--auth` option.
+
+```
+$ prometheus_exporter --auth my-htpasswd-file
+```
+
+Additionally, the `--realm` option may be used to provide a customized realm for the challenge request.
+
+Notes:
+
+* You will need to create a `htpasswd` formatted file before hand which contains one or more user:password entries
+* Only the basic `crypt` encryption is currently supported
+
+A simple `htpasswd` file can be created with the Apache `htpasswd` utility; e.g:
+
+```
+$ htpasswd -cdb my-htpasswd-file my-user my-unencrypted-password
+```
+
+This will create a file named `my-htpasswd-file` which is suitable for use the `--auth` option.
+
 ### Client default labels
 
 You can specify a default label for instrumentation metrics sent by a specific client. For example:

data/bin/prometheus_exporter

@@ -47,6 +47,12 @@ def run
     opt.on('-v', '--verbose') do |o|
       options[:verbose] = true
     end
+    opt.on('--auth FILE', String, "(optional) enable basic authentication using a htpasswd FILE") do |o|
+      options[:auth] = o
+    end
+    opt.on('--realm REALM', String, "(optional) Use REALM for basic authentication (default: \"#{PrometheusExporter::DEFAULT_REALM}\")") do |o|
+      options[:realm] = o
+    end
 
     opt.on('--unicorn-listen-address ADDRESS', String, '(optional) Address where unicorn listens on (unix or TCP address)') do |o|
       options[:unicorn_listen_address] = o
@@ -57,6 +63,17 @@ def run
     end
   end.parse!
 
+  if options.has_key?(:realm) && !options.has_key?(:auth)
+    STDERR.puts "[Warn] Providing REALM without AUTH has no effect"
+  end
+
+  if options.has_key?(:auth)
+    unless File.exist?(options[:auth]) && File.readable?(options[:auth])
+      STDERR.puts "[Error] The AUTH file either doesn't exist or we don't have access to it"
+      exit 1
+    end
+  end
+
   if custom_collector_filename
     eval File.read(custom_collector_filename), nil, File.expand_path(custom_collector_filename)
     found = false

data/lib/prometheus_exporter.rb

@@ -11,6 +11,7 @@ module PrometheusExporter
   DEFAULT_PREFIX = 'ruby_'
   DEFAULT_LABEL = {}
   DEFAULT_TIMEOUT = 2
+  DEFAULT_REALM = 'Prometheus Exporter'
 
   class OjCompat
     def self.parse(obj)

data/lib/prometheus_exporter/client.rb

@@ -64,8 +64,10 @@ module PrometheusExporter
       @metrics = []
 
       @queue = Queue.new
+
       @socket = nil
       @socket_started = nil
+      @socket_pid = nil
 
       max_queue_size ||= MAX_QUEUE_SIZE
       max_queue_size = max_queue_size.to_i
@@ -107,7 +109,16 @@ module PrometheusExporter
     end
 
     def send_json(obj)
-      payload = @custom_labels.nil? ? obj : obj.merge(custom_labels: @custom_labels)
+      payload =
+        if @custom_labels
+          if obj[:custom_labels]
+            obj.merge(custom_labels: @custom_labels.merge(obj[:custom_labels]))
+          else
+            obj.merge(custom_labels: @custom_labels)
+          end
+        else
+          obj
+        end
       send(@json_serializer.dump(payload))
     end
 
@@ -191,12 +202,20 @@ module PrometheusExporter
     end
 
     def close_socket_if_old!
-      if @socket && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
+      if @socket_pid == Process.pid && @socket && @socket_started && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
         close_socket!
       end
     end
 
     def ensure_socket!
+      # if process was forked socket may be owned by parent
+      # leave it alone and reset
+      if @socket_pid != Process.pid
+        @socket = nil
+        @socket_started = nil
+        @socket_pid = nil
+      end
+
       close_socket_if_old!
       if !@socket
         @socket = TCPSocket.new @host, @port
@@ -207,12 +226,14 @@ module PrometheusExporter
         @socket.write("Content-Type: application/octet-stream\r\n")
         @socket.write("\r\n")
         @socket_started = Time.now.to_f
+        @socket_pid = Process.pid
       end
 
       nil
     rescue
       @socket = nil
       @socket_started = nil
+      @socket_pid = nil
       raise
     end
 

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -69,7 +69,7 @@ module PrometheusExporter::Instrumentation
 
         labels_from_config = pool.spec.config
           .select { |k, v| @config_labels.include? k }
-          .map { |k, v| [k.to_s.prepend("dbconfig_"), v] }
+          .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }
 
         labels = @metric_labels.merge(pool_name: pool.spec.name).merge(Hash[labels_from_config])
 

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -1,6 +1,15 @@
 # frozen_string_literal: true
 
+require 'yaml'
+
 module PrometheusExporter::Instrumentation
+  JOB_WRAPPER_CLASS_NAME = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'
+  DELAYED_CLASS_NAMES = [
+    'Sidekiq::Extensions::DelayedClass',
+    'Sidekiq::Extensions::DelayedModel',
+    'Sidekiq::Extensions::DelayedMailer',
+  ]
+
   class Sidekiq
     def self.death_handler
       -> (job, ex) do
@@ -32,15 +41,47 @@ module PrometheusExporter::Instrumentation
       raise e
     ensure
       duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
-      class_name = worker.class.to_s == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' ?
-                     msg['wrapped'] : worker.class.to_s
       @client.send_json(
         type: "sidekiq",
-        name: class_name,
+        name: get_name(worker, msg),
+        queue: queue,
         success: success,
         shutdown: shutdown,
         duration: duration
       )
     end
+
+    private
+
+    def get_name(worker, msg)
+      class_name = worker.class.to_s
+      if class_name == JOB_WRAPPER_CLASS_NAME
+        get_job_wrapper_name(msg)
+      elsif DELAYED_CLASS_NAMES.include?(class_name)
+        get_delayed_name(msg, class_name)
+      else
+        class_name
+      end
+    end
+
+    def get_job_wrapper_name(msg)
+      msg['wrapped']
+    end
+
+    def get_delayed_name(msg, class_name)
+      # fallback to class_name since we're relying on the internal implementation
+      # of the delayed extensions
+      # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+      begin
+        (target, method_name, _args) = YAML.load(msg['args'].first)
+        if target.class == Class
+          "#{target.name}##{method_name}"
+        else
+          "#{target.class.name}##{method_name}"
+        end
+      rescue
+        class_name
+      end
+    end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -27,13 +27,24 @@ module PrometheusExporter::Instrumentation
     end
 
     def collect_queue_stats
+      hostname = Socket.gethostname
+      pid = ::Process.pid
+      ps = ::Sidekiq::ProcessSet.new
+
+      process = ps.find do |sp|
+        sp['hostname'] == hostname && sp['pid'] == pid
+      end
+
+      queues = process.nil? ? [] : process['queues']
+
       ::Sidekiq::Queue.all.map do |queue|
+        next unless queues.include? queue.name
         {
           backlog_total: queue.size,
           latency_seconds: queue.latency.to_i,
           labels: { queue: queue.name }
         }
-      end
+      end.compact
     end
   end
 end

data/lib/prometheus_exporter/middleware.rb

@@ -44,14 +44,25 @@ class PrometheusExporter::Middleware
       controller = params["controller"]
     end
 
-    @client.send_json(
+    obj = {
       type: "web",
       timings: info,
       queue_time: queue_time,
       action: action,
       controller: controller,
       status: status
-    )
+    }
+    labels = custom_labels(env)
+    if labels
+      obj = obj.merge(custom_labels: labels)
+    end
+
+    @client.send_json(obj)
+  end
+
+  # allows subclasses to add custom labels based on env
+  def custom_labels(env)
+    nil
   end
 
   private

data/lib/prometheus_exporter/server/runner.rb

@@ -15,6 +15,8 @@ module PrometheusExporter::Server
       @collector_class = nil
       @type_collectors = nil
       @prefix = nil
+      @auth = nil
+      @realm = nil
 
       options.each do |k, v|
         send("#{k}=", v) if self.class.method_defined?("#{k}=")
@@ -40,12 +42,20 @@ module PrometheusExporter::Server
         )
       end
 
-      server = server_class.new port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose
+      server = server_class.new(port: port, bind: bind, collector: collector, timeout: timeout, verbose: verbose, auth: auth, realm: realm)
       server.start
     end
 
     attr_accessor :unicorn_listen_address, :unicorn_pid_file
-    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label
+    attr_writer :prefix, :port, :bind, :collector_class, :type_collectors, :timeout, :verbose, :server_class, :label, :auth, :realm
+
+    def auth
+      @auth || nil
+    end
+
+    def realm
+      @realm || PrometheusExporter::DEFAULT_REALM
+    end
 
     def prefix
       @prefix || PrometheusExporter::DEFAULT_PREFIX

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -17,7 +17,7 @@ module PrometheusExporter::Server
     end
 
     def collect(obj)
-      default_labels = { job_name: obj['name'] }
+      default_labels = { job_name: obj['name'], queue: obj['queue'] }
       custom_labels = obj['custom_labels']
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
@@ -52,7 +52,7 @@ module PrometheusExporter::Server
       if !@sidekiq_jobs_total
 
         @sidekiq_job_duration_seconds =
-        PrometheusExporter::Metric::Counter.new(
+        PrometheusExporter::Metric::Summary.new(
           "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs.")
 
         @sidekiq_jobs_total =

data/lib/prometheus_exporter/server/web_server.rb

@@ -9,9 +9,14 @@ module PrometheusExporter::Server
   class WebServer
     attr_reader :collector
 
-    def initialize(port: , bind: nil, collector: nil, timeout: PrometheusExporter::DEFAULT_TIMEOUT, verbose: false)
-
-      @verbose = verbose
+    def initialize(opts)
+      @port = opts[:port] || PrometheusExporter::DEFAULT_PORT
+      @bind = opts[:bind] || PrometheusExporter::DEFAULT_BIND_ADDRESS
+      @collector = opts[:collector] || Collector.new
+      @timeout = opts[:timeout] || PrometheusExporter::DEFAULT_TIMEOUT
+      @verbose = opts[:verbose] || false
+      @auth = opts[:auth]
+      @realm = opts[:realm] || PrometheusExporter::DEFAULT_REALM
 
       @metrics_total = PrometheusExporter::Metric::Counter.new("collector_metrics_total", "Total metrics processed by exporter web.")
 
@@ -23,33 +28,33 @@ module PrometheusExporter::Server
       @sessions_total.observe(0)
       @bad_metrics_total.observe(0)
 
-      access_log, logger = nil
+      @access_log, @logger = nil
 
-      if verbose
-        access_log = [
+      if @verbose
+        @access_log = [
           [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT],
           [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT],
         ]
-        logger = WEBrick::Log.new($stderr)
+        @logger = WEBrick::Log.new($stderr)
       else
-        access_log = []
-        logger = WEBrick::Log.new("/dev/null")
+        @access_log = []
+        @logger = WEBrick::Log.new("/dev/null")
       end
 
+      @logger.info "Using Basic Authentication via #{@auth}" if @verbose && @auth
+
       @server = WEBrick::HTTPServer.new(
-        Port: port,
-        BindAddress: bind,
-        Logger: logger,
-        AccessLog: access_log,
+        Port: @port,
+        BindAddress: @bind,
+        Logger: @logger,
+        AccessLog: @access_log,
       )
 
-      @collector = collector || Collector.new
-      @port = port
-      @timeout = timeout
-
       @server.mount_proc '/' do |req, res|
         res['Content-Type'] = 'text/plain; charset=utf-8'
         if req.path == '/metrics'
+          authenticate(req, res) if @auth
+
           res.status = 200
           if req.header["accept-encoding"].to_s.include?("gzip")
             sio = StringIO.new
@@ -159,5 +164,12 @@ module PrometheusExporter::Server
       gauge
     end
 
+    def authenticate(req, res)
+      htpasswd = WEBrick::HTTPAuth::Htpasswd.new(@auth)
+      basic_auth = WEBrick::HTTPAuth::BasicAuth.new({ Realm: @realm, UserDB: htpasswd, Logger: @logger })
+
+      basic_auth.authenticate(req, res)
+    end
+
   end
 end

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '0.5.3'
+  VERSION = '0.6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.0
 platform: ruby
 authors:
 - Sam Saffron
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-29 00:00:00.000000000 Z
+date: 2020-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubocop
@@ -172,9 +172,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - CHANGELOG
 - CODE_OF_CONDUCT.md
 - Gemfile

data/.travis.yml

@@ -1,12 +0,0 @@
-sudo: required
-language: ruby
-dist: trusty
-rvm:
-  - 2.3
-  - 2.4
-  - 2.5
-  - 2.6
-before_install: gem install bundler -v 1.16.1
-before_script: sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'
-
-script: bundle exec rubocop && bundle exec rake