prometheus_exporter 0.4.17 → 0.6.0

data/bin/prometheus_exporter

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'optparse'
+require 'json'
 
 require_relative "./../lib/prometheus_exporter"
 require_relative "./../lib/prometheus_exporter/server"
@@ -19,6 +20,12 @@ def run
            "Port exporter should listen on (default: #{PrometheusExporter::DEFAULT_PORT})") do |o|
       options[:port] = o.to_i
     end
+    opt.on('-b',
+           '--bind STRING',
+           String,
+           "IP address exporter should listen on (default: #{PrometheusExporter::DEFAULT_BIND_ADDRESS})") do |o|
+      options[:bind] = o.to_s
+    end
     opt.on('-t',
            '--timeout INTEGER',
            Integer,
@@ -28,6 +35,9 @@ def run
     opt.on('--prefix METRIC_PREFIX', "Prefix to apply to all metrics (default: #{PrometheusExporter::DEFAULT_PREFIX})") do |o|
       options[:prefix] = o.to_s
     end
+    opt.on('--label METRIC_LABEL', "Label to apply to all metrics (default: #{PrometheusExporter::DEFAULT_LABEL})") do |o|
+      options[:label] = JSON.parse(o.to_s)
+    end
     opt.on('-c', '--collector FILE', String, "(optional) Custom collector to run") do |o|
       custom_collector_filename = o.to_s
     end
@@ -37,6 +47,12 @@ def run
     opt.on('-v', '--verbose') do |o|
       options[:verbose] = true
     end
+    opt.on('--auth FILE', String, "(optional) enable basic authentication using a htpasswd FILE") do |o|
+      options[:auth] = o
+    end
+    opt.on('--realm REALM', String, "(optional) Use REALM for basic authentication (default: \"#{PrometheusExporter::DEFAULT_REALM}\")") do |o|
+      options[:realm] = o
+    end
 
     opt.on('--unicorn-listen-address ADDRESS', String, '(optional) Address where unicorn listens on (unix or TCP address)') do |o|
       options[:unicorn_listen_address] = o
@@ -47,6 +63,17 @@ def run
     end
   end.parse!
 
+  if options.has_key?(:realm) && !options.has_key?(:auth)
+    STDERR.puts "[Warn] Providing REALM without AUTH has no effect"
+  end
+
+  if options.has_key?(:auth)
+    unless File.exist?(options[:auth]) && File.readable?(options[:auth])
+      STDERR.puts "[Error] The AUTH file either doesn't exist or we don't have access to it"
+      exit 1
+    end
+  end
+
   if custom_collector_filename
     eval File.read(custom_collector_filename), nil, File.expand_path(custom_collector_filename)
     found = false
@@ -81,7 +108,7 @@ def run
 
   runner = PrometheusExporter::Server::Runner.new(options)
 
-  puts "#{Time.now} Starting prometheus exporter on port #{runner.port}"
+  puts "#{Time.now} Starting prometheus exporter on #{runner.bind}:#{runner.port}"
   runner.start
   sleep
 end

data/lib/prometheus_exporter.rb

@@ -7,8 +7,11 @@ require "thread"
 module PrometheusExporter
   # per: https://github.com/prometheus/prometheus/wiki/Default-port-allocations
   DEFAULT_PORT = 9394
+  DEFAULT_BIND_ADDRESS = 'localhost'
   DEFAULT_PREFIX = 'ruby_'
+  DEFAULT_LABEL = {}
   DEFAULT_TIMEOUT = 2
+  DEFAULT_REALM = 'Prometheus Exporter'
 
   class OjCompat
     def self.parse(obj)
@@ -19,6 +22,17 @@ module PrometheusExporter
     end
   end
 
+  def self.hostname
+    @hostname ||=
+      begin
+        require 'socket'
+        Socket.gethostname
+      rescue => e
+        STDERR.puts "Unable to lookup hostname #{e}"
+        "unknown-host"
+      end
+  end
+
   def self.detect_json_serializer(preferred)
     if preferred.nil?
       preferred = :oj if has_oj?

data/lib/prometheus_exporter/client.rb

@@ -53,12 +53,21 @@ module PrometheusExporter
     MAX_SOCKET_AGE = 25
     MAX_QUEUE_SIZE = 10_000
 
-    def initialize(host: 'localhost', port: PrometheusExporter::DEFAULT_PORT, max_queue_size: nil, thread_sleep: 0.5, json_serializer: nil, custom_labels: nil)
+    def initialize(
+      host: ENV.fetch('PROMETHEUS_EXPORTER_HOST', 'localhost'),
+      port: ENV.fetch('PROMETHEUS_EXPORTER_PORT', PrometheusExporter::DEFAULT_PORT),
+      max_queue_size: nil,
+      thread_sleep: 0.5,
+      json_serializer: nil,
+      custom_labels: nil
+    )
       @metrics = []
 
       @queue = Queue.new
+
       @socket = nil
       @socket_started = nil
+      @socket_pid = nil
 
       max_queue_size ||= MAX_QUEUE_SIZE
       max_queue_size = max_queue_size.to_i
@@ -100,7 +109,16 @@ module PrometheusExporter
     end
 
     def send_json(obj)
-      payload = @custom_labels.nil? ? obj : obj.merge(custom_labels: @custom_labels)
+      payload =
+        if @custom_labels
+          if obj[:custom_labels]
+            obj.merge(custom_labels: @custom_labels.merge(obj[:custom_labels]))
+          else
+            obj.merge(custom_labels: @custom_labels)
+          end
+        else
+          obj
+        end
       send(@json_serializer.dump(payload))
     end
 
@@ -184,12 +202,20 @@ module PrometheusExporter
     end
 
     def close_socket_if_old!
-      if @socket && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
+      if @socket_pid == Process.pid && @socket && @socket_started && ((@socket_started + MAX_SOCKET_AGE) < Time.now.to_f)
         close_socket!
       end
     end
 
     def ensure_socket!
+      # if process was forked socket may be owned by parent
+      # leave it alone and reset
+      if @socket_pid != Process.pid
+        @socket = nil
+        @socket_started = nil
+        @socket_pid = nil
+      end
+
       close_socket_if_old!
       if !@socket
         @socket = TCPSocket.new @host, @port
@@ -200,12 +226,14 @@ module PrometheusExporter
         @socket.write("Content-Type: application/octet-stream\r\n")
         @socket.write("\r\n")
         @socket_started = Time.now.to_f
+        @socket_pid = Process.pid
       end
 
       nil
     rescue
       @socket = nil
       @socket_started = nil
+      @socket_pid = nil
       raise
     end
 

data/lib/prometheus_exporter/instrumentation.rb

@@ -4,8 +4,10 @@ require_relative "client"
 require_relative "instrumentation/process"
 require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
+require_relative "instrumentation/sidekiq_queue"
 require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"
 require_relative "instrumentation/unicorn"
 require_relative "instrumentation/active_record"
+require_relative "instrumentation/shoryuken"

data/lib/prometheus_exporter/instrumentation/active_record.rb

@@ -51,17 +51,6 @@ module PrometheusExporter::Instrumentation
     def initialize(metric_labels, config_labels)
       @metric_labels = metric_labels
       @config_labels = config_labels
-      @hostname = nil
-    end
-
-    def hostname
-      @hostname ||=
-        begin
-          `hostname`.strip
-        rescue => e
-          STDERR.puts "Unable to lookup hostname #{e}"
-          "unknown-host"
-        end
     end
 
     def collect
@@ -80,14 +69,14 @@ module PrometheusExporter::Instrumentation
 
         labels_from_config = pool.spec.config
           .select { |k, v| @config_labels.include? k }
-          .map { |k, v| [k.to_s.prepend("dbconfig_"), v] }
+          .map { |k, v| [k.to_s.dup.prepend("dbconfig_"), v] }
 
         labels = @metric_labels.merge(pool_name: pool.spec.name).merge(Hash[labels_from_config])
 
         metric = {
           pid: pid,
           type: "active_record",
-          hostname: hostname,
+          hostname: ::PrometheusExporter.hostname,
           metric_labels: labels
         }
         metric.merge!(pool.stat)

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -3,6 +3,8 @@
 # collects stats from currently running process
 module PrometheusExporter::Instrumentation
   class Process
+    @thread = nil if !defined?(@thread)
+
     def self.start(client: nil, type: "ruby", frequency: 30, labels: nil)
 
       metric_labels =
@@ -42,24 +44,13 @@ module PrometheusExporter::Instrumentation
 
     def initialize(metric_labels)
       @metric_labels = metric_labels
-      @hostname = nil
-    end
-
-    def hostname
-      @hostname ||=
-        begin
-          `hostname`.strip
-        rescue => e
-          STDERR.puts "Unable to lookup hostname #{e}"
-          "unknown-host"
-        end
     end
 
     def collect
       metric = {}
       metric[:type] = "process"
       metric[:metric_labels] = @metric_labels
-      metric[:hostname] = hostname
+      metric[:hostname] = ::PrometheusExporter.hostname
       collect_gc_stats(metric)
       collect_v8_stats(metric)
       collect_process_stats(metric)

data/lib/prometheus_exporter/instrumentation/shoryuken.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class Shoryuken
+
+    def initialize(client: nil)
+      @client = client || PrometheusExporter::Client.default
+    end
+
+    def call(worker, queue, msg, body)
+      success = false
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      result = yield
+      success = true
+      result
+    rescue ::Shoryuken::Shutdown => e
+      shutdown = true
+      raise e
+    ensure
+      duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
+      @client.send_json(
+          type: "shoryuken",
+          queue: queue,
+          name: worker.class.name,
+          success: success,
+          shutdown: shutdown,
+          duration: duration
+      )
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -1,6 +1,15 @@
 # frozen_string_literal: true
 
+require 'yaml'
+
 module PrometheusExporter::Instrumentation
+  JOB_WRAPPER_CLASS_NAME = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'
+  DELAYED_CLASS_NAMES = [
+    'Sidekiq::Extensions::DelayedClass',
+    'Sidekiq::Extensions::DelayedModel',
+    'Sidekiq::Extensions::DelayedMailer',
+  ]
+
   class Sidekiq
     def self.death_handler
       -> (job, ex) do
@@ -32,15 +41,47 @@ module PrometheusExporter::Instrumentation
       raise e
     ensure
       duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
-      class_name = worker.class.to_s == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' ?
-                     msg['wrapped'] : worker.class.to_s
       @client.send_json(
         type: "sidekiq",
-        name: class_name,
+        name: get_name(worker, msg),
+        queue: queue,
         success: success,
         shutdown: shutdown,
         duration: duration
       )
     end
+
+    private
+
+    def get_name(worker, msg)
+      class_name = worker.class.to_s
+      if class_name == JOB_WRAPPER_CLASS_NAME
+        get_job_wrapper_name(msg)
+      elsif DELAYED_CLASS_NAMES.include?(class_name)
+        get_delayed_name(msg, class_name)
+      else
+        class_name
+      end
+    end
+
+    def get_job_wrapper_name(msg)
+      msg['wrapped']
+    end
+
+    def get_delayed_name(msg, class_name)
+      # fallback to class_name since we're relying on the internal implementation
+      # of the delayed extensions
+      # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
+      begin
+        (target, method_name, _args) = YAML.load(msg['args'].first)
+        if target.class == Class
+          "#{target.name}##{method_name}"
+        else
+          "#{target.class.name}##{method_name}"
+        end
+      rescue
+        class_name
+      end
+    end
   end
 end

data/lib/prometheus_exporter/instrumentation/sidekiq_queue.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqQueue
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_queue_collector = new
+
+      Thread.new do
+        loop do
+          begin
+            client.send_json(sidekiq_queue_collector.collect)
+          rescue StandardError => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Queue metrics #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def collect
+      {
+        type: 'sidekiq_queue',
+        queues: collect_queue_stats
+      }
+    end
+
+    def collect_queue_stats
+      hostname = Socket.gethostname
+      pid = ::Process.pid
+      ps = ::Sidekiq::ProcessSet.new
+
+      process = ps.find do |sp|
+        sp['hostname'] == hostname && sp['pid'] == pid
+      end
+
+      queues = process.nil? ? [] : process['queues']
+
+      ::Sidekiq::Queue.all.map do |queue|
+        next unless queues.include? queue.name
+        {
+          backlog_total: queue.size,
+          latency_seconds: queue.latency.to_i,
+          labels: { queue: queue.name }
+        }
+      end.compact
+    end
+  end
+end

data/lib/prometheus_exporter/metric/base.rb

@@ -2,6 +2,10 @@
 
 module PrometheusExporter::Metric
   class Base
+
+    @default_prefix = nil if !defined?(@default_prefix)
+    @default_labels = nil if !defined?(@default_labels)
+
     # prefix applied to all metrics
     def self.default_prefix=(name)
       @default_prefix = name

data/lib/prometheus_exporter/metric/counter.rb

@@ -27,6 +27,10 @@ module PrometheusExporter::Metric
       @data.dup
     end
 
+    def remove(labels)
+      @data.delete(labels)
+    end
+
     def observe(increment = 1, labels = {})
       @data[labels] ||= 0
       @data[labels] += increment

data/lib/prometheus_exporter/metric/gauge.rb

@@ -27,6 +27,10 @@ module PrometheusExporter::Metric
       @data.dup
     end
 
+    def remove(labels)
+      @data.delete(labels)
+    end
+
     def observe(value, labels = {})
       if value.nil?
         data.delete(labels)

data/lib/prometheus_exporter/metric/histogram.rb

@@ -27,6 +27,12 @@ module PrometheusExporter::Metric
       data
     end
 
+    def remove(labels)
+      @observations.delete(labels)
+      @counts.delete(labels)
+      @sums.delete(labels)
+    end
+
     def type
       "histogram"
     end