prometheus_exporter 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f983fdbff857366a4b30252ff817544e08993b15181812b597ebb281ded2fb30
-  data.tar.gz: 4d9c9bae239d2bfefa6633646806095edd69b81af6d2c59f18cbe36c91313525
+  metadata.gz: 6957162d157420b2b8fb6d2a4b9e08483ce846497fe6bb21f1695b0e70460dc8
+  data.tar.gz: 9a74c21c97efad9ef3720df426d14a5e1d879eb34bebcafc625de8d45f3b232b
 SHA512:
-  metadata.gz: 0d614ff763d234e0a738e9388b8d38f6e96832a0757c2ee02b0ddb7f67f4875904d624dc49b0257126a2fe255f47738c98299571ccfb694317af3c8ee63ad479
-  data.tar.gz: c810eefd3a240548cd4d4cde7374541cb940388907d1cac422f57feea224997114904a09f7037e921652012038f65f5ec929bb69a239d67feb10c54145f55df8
+  metadata.gz: 695472a08f8f9ede18d6f0fdaa8722d2e4efb01b20bf23269d26afae580cb559da4f409bb3499da7a80dcd5d47e6156a6e8c7e54f01c770485212e21369fb61d
+  data.tar.gz: af55f7e9d025a925f0add89a61628d236691df0a613d82d8c4e622fc9b83b7651187d9e78121113de872632459e95015772ed9af43a12b85ce4a1f820be9b7fc

data/README.md

@@ -61,20 +61,20 @@ In some cases, for example unicorn or puma clusters you may want to aggregate me
 
 Simplest way to acheive this is use the built-in collector.
 
-First, run an exporter on your desired port:
+First, run an exporter on your desired port, we use the default port of 9394:
 
 ```
-# prometheus_exporter 12345
+# prometheus_exporter
 ```
 
-At this point an exporter is running on port 12345
+At this point an exporter is running on port 9394
 
 In your application:
 
 ```ruby
 require 'prometheus_exporter/client'
 
-client = PrometheusExporter::Client.new(host: 'localhost', port: 12345)
+client = PrometheusExporter::Client.default
 gauge = client.register(:gauge, "awesome", "amount of awesome")
 
 gauge.observe(10)
@@ -85,7 +85,7 @@ gauge.observe(99, day: "friday")
 Then you will get the metrics:
 
 ```bash
-% curl localhost:12345/metrics
+% curl localhost:9394/metrics
 # HELP collector_working Is the master process collector able to collect metrics
 # TYPE collector_working gauge
 collector_working 1
@@ -97,6 +97,50 @@ awesome 10
 
 ```
 
+### Easy integration into Rails
+
+You can easily integrate into any Rack application:
+
+In your Gemfile:
+
+```
+gem 'prometheus_exporter'
+```
+
+
+```
+# in an initializer
+
+unless Rails.env == "test"
+  require 'prometheus_exporter/middleware'
+  # insert in position 1
+  # instrument means method profiler will be injected in Redis and PG
+  Rails.application.middleware.unshift PrometheusExporter::Middleware
+end
+```
+
+You may also be interested in per-process stats, this collects memory and GC stats
+
+```
+# in an initializer
+unless Rails.env == "test"
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::Process.start(type: "master")
+end
+
+after_fork do
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::Process.start(type:"web")
+end
+
+```
+
+Ensure you run the exporter via
+
+```
+% bundle exec prometheus_exporter
+```
+
 ### Multi process mode with custom collector
 
 You can opt for custom collector logic in a multi process environment.
@@ -108,7 +152,7 @@ The standard collector ships "help", "type" and "name" for every metric, in some
 First, define a custom collector, it is critical you inherit off `PrometheusExporter::Server::Collector`, also it is critical you have custom implementations for #process and #prometheus_metrics_text
 
 ```ruby
-class MyCustomCollector < PrometheusExporter::Server::Collector
+class MyCustomCollector < PrometheusExporter::Server::CollectorBase
   def initialize
     @gauge1 = PrometheusExporter::Metric::Gauge.new("thing1", "I am thing 1")
     @gauge2 = PrometheusExporter::Metric::Gauge.new("thing2", "I am thing 2")

data/bin/prometheus_exporter

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+require_relative "./../lib/prometheus_exporter"
+require_relative "./../lib/prometheus_exporter/server"
+
+def run
+  port = PrometheusExporter::DEFAULT_PORT
+  prefix = "ruby_"
+  collector = nil
+  verbose = false
+
+  OptionParser.new do |opt|
+    opt.on('-p',
+           '--port INTEGER',
+           Integer,
+           "Port exporter should listen on (default: #{port})") do |o|
+      port = o.to_i
+    end
+    opt.on('--prefix METRIC_PREFIX', String, "Prefix to apply to all metrics (default: #{prefix})") do |o|
+      prefix = o.to_s
+    end
+    opt.on('-c', '--collector CUSTOM_COLLECTOR', String, "(optional) Custom collector to run") do |o|
+      collector = o.to_s
+    end
+    opt.on('-v', '--verbose') do |o|
+      verbose = true
+    end
+
+  end.parse!
+
+  PrometheusExporter::Metric::Base.default_prefix = prefix
+
+  if collector
+    eval File.read(collector)
+
+    ObjectSpace.each_object(Class) do |klass|
+      if klass < PrometheusExporter::Server::CollectorBase
+        collector = klass
+      end
+    end
+
+    if !collector
+      STDERR.puts "Can not find a class inheriting off PrometheusExporter::Server::Collector"
+      usage
+      exit 1
+    end
+  end
+
+  puts "#{Time.now} Starting prometheus exporter on port #{port}"
+  server = PrometheusExporter::Server::WebServer.new port: port, collector: collector&.new, verbose: verbose
+  server.start
+  sleep
+
+end
+
+run

data/lib/prometheus_exporter.rb

@@ -3,4 +3,6 @@ require "json"
 require "thread"
 
 module PrometheusExporter
+  # per: https://github.com/prometheus/prometheus/wiki/Default-port-allocations
+  DEFAULT_PORT = 9394
 end

data/lib/prometheus_exporter/client.rb

@@ -24,10 +24,18 @@ class PrometheusExporter::Client
     end
   end
 
+  def self.default
+    @default ||= new
+  end
+
+  def self.default=(client)
+    @default = client
+  end
+
   MAX_SOCKET_AGE = 25
   MAX_QUEUE_SIZE = 10_000
 
-  def initialize(host: 'localhost', port:, max_queue_size: nil, thread_sleep: 0.5)
+  def initialize(host: 'localhost', port: PrometheusExporter::DEFAULT_PORT, max_queue_size: nil, thread_sleep: 0.5)
     @metrics = []
 
     @queue = Queue.new

data/lib/prometheus_exporter/instrumentation.rb

@@ -0,0 +1,2 @@
+require_relative "instrumentation/process"
+require_relative "instrumentation/method_profiler"

data/lib/prometheus_exporter/instrumentation/method_profiler.rb

@@ -0,0 +1,55 @@
+# see https://samsaffron.com/archive/2017/10/18/fastest-way-to-profile-a-method-in-ruby
+module PrometheusExporter::Instrumentation; end
+
+class PrometheusExporter::Instrumentation::MethodProfiler
+  def self.patch(klass, methods, name)
+    patches = methods.map do |method_name|
+      <<~RUBY
+      unless defined?(#{method_name}__mp_unpatched)
+        alias_method :#{method_name}__mp_unpatched, :#{method_name}
+        def #{method_name}(*args, &blk)
+          unless prof = Thread.current[:_method_profiler]
+            return #{method_name}__mp_unpatched(*args, &blk)
+          end
+          begin
+            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            #{method_name}__mp_unpatched(*args, &blk)
+          ensure
+            data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
+            data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+            data[:calls] += 1
+          end
+        end
+      end
+      RUBY
+    end.join("\n")
+
+    klass.class_eval patches
+  end
+
+  def self.transfer
+    result = Thread.current[:_method_profiler]
+    Thread.current[:_method_profiler] = nil
+    result
+  end
+
+  def self.start(transfer = nil)
+    Thread.current[:_method_profiler] = transfer || {
+      __start: Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    }
+  end
+
+  def self.clear
+    Thread.current[:_method_profiler] = nil
+  end
+
+  def self.stop
+    finish = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    if data = Thread.current[:_method_profiler]
+      Thread.current[:_method_profiler] = nil
+      start = data.delete(:__start)
+      data[:total_duration] = finish - start
+    end
+    data
+  end
+end

data/lib/prometheus_exporter/instrumentation/process.rb

@@ -0,0 +1,77 @@
+# collects stats from currently running process
+module PrometheusExporter::Instrumentation
+  class Process
+    def self.start(client: nil, type: "ruby", frequency: 30)
+      process_collector = new(type)
+      client ||= PrometheusExporter::Client.default
+      Thread.new do
+        while true
+          begin
+            metric = process_collector.collect
+            client.send_json metric
+          rescue => e
+            STDERR.puts("Prometheus Discoruse Failed To Collect Process Stats #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def initialize(type)
+      @type = type
+    end
+
+    def collect
+      metric = {}
+      metric[:type] = "process"
+      metric[:process_type] = @type
+      collect_gc_stats(metric)
+      collect_v8_stats(metric)
+      collect_process_stats(metric)
+      metric
+    end
+
+    def pid
+      @pid = ::Process.pid
+    end
+
+    def rss
+      @pagesize ||= `getconf PAGESIZE`.to_i rescue 4096
+      File.read("/proc/#{pid}/statm").split(' ')[1].to_i * @pagesize rescue 0
+    end
+
+    def collect_process_stats(metric)
+      metric[:pid] = pid
+      metric[:rss] = rss
+
+    end
+
+    def collect_gc_stats(metric)
+      stat = GC.stat
+      metric[:heap_live_slots] = stat[:heap_live_slots]
+      metric[:heap_free_slots] = stat[:heap_free_slots]
+      metric[:major_gc_count] = stat[:major_gc_count]
+      metric[:minor_gc_count] = stat[:minor_gc_count]
+      metric[:total_allocated_objects] = stat[:total_allocated_objects]
+    end
+
+    def collect_v8_stats(metric)
+      return if !defined? MiniRacer
+
+      metric[:v8_heap_count] = metric[:v8_heap_size] = 0
+      metric[:v8_heap_size] = metric[:v8_physical_size] = 0
+      metric[:v8_used_heap_size] = 0
+
+      ObjectSpace.each_object(MiniRacer::Context) do |context|
+        stats = context.heap_stats
+        if stats
+          metric[:v8_heap_count] += 1
+          metric[:v8_heap_size] += stats[:total_heap_size].to_i
+          metric[:v8_used_heap_size] += stats[:used_heap_size].to_i
+          metric[:v8_physical_size] += stats[:total_physical_size].to_i
+        end
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/middleware.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'prometheus_exporter/instrumentation/method_profiler'
+require 'prometheus_exporter/client'
+
+class PrometheusExporter::Middleware
+  MethodProfiler = PrometheusExporter::Instrumentation::MethodProfiler
+
+  def initialize(app, config = { instrument: true, client: nil })
+    @app = app
+    @client = config[:client] || PrometheusExporter::Client.default
+
+    if config[:instrument]
+      if defined? Redis::Client
+        MethodProfiler.patch(Redis::Client, [:call, :call_pipeline], :redis)
+      end
+      if defined? PG::Connection
+        MethodProfiler.patch(PG::Connection, [
+          :exec, :async_exec, :exec_prepared, :send_query_prepared, :query
+        ], :sql)
+      end
+    end
+  end
+
+  def call(env)
+    MethodProfiler.start
+    result = @app.call(env)
+    info = MethodProfiler.stop
+    result
+  ensure
+    status = (result && result[0]) || -1
+    params = env["action_dispatch.request.parameters"]
+    action, controller = nil
+    if params
+      action = params["action"]
+      controller = params["controller"]
+    end
+
+    @client.send_json(
+      type: "web",
+      timings: info,
+      action: action,
+      controller: controller,
+      status: status
+    )
+  end
+end

data/lib/prometheus_exporter/server.rb

@@ -1,3 +1,4 @@
+require_relative "server/collector_base"
 require_relative "server/collector"
 require_relative "server/web_server"
 require_relative "metric"

data/lib/prometheus_exporter/server/collector.rb

@@ -2,9 +2,26 @@
 
 module PrometheusExporter::Server
 
-  class Collector
+  class Collector < CollectorBase
+    MAX_PROCESS_METRIC_AGE = 60
+    PROCESS_GAUGES = {
+      heap_free_slots: "Free ruby heap slots",
+      heap_live_slots: "Used ruby heap slots",
+      v8_heap_size: "Total JavaScript V8 heap size (bytes)",
+      v8_used_heap_size: "Total used JavaScript V8 heap size (bytes)",
+      v8_physical_size: "Physical size consumed by V8 heaps",
+      v8_heap_count: "Number of V8 contexts running",
+      rss: "Total RSS used by process",
+    }
+
+    PROCESS_COUNTERS = {
+      major_gc_count: "Major GC operations by process",
+      minor_gc_count: "Minor GC operations by process",
+      total_allocated_objects: "Total number of allocateds objects by process",
+    }
 
     def initialize
+      @process_metrics = []
       @metrics = {}
       @buffer = []
       @mutex = Mutex.new
@@ -13,27 +30,120 @@ module PrometheusExporter::Server
     def process(str)
       obj = JSON.parse(str)
       @mutex.synchronize do
-        metric = @metrics[obj["name"]]
-        if !metric
-          metric = register_metric_unsafe(obj)
+        if obj["type"] == "web"
+          observe_web(obj)
+        elsif obj["type"] == "process"
+          observe_process(obj)
+        else
+          metric = @metrics[obj["name"]]
+          if !metric
+            metric = register_metric_unsafe(obj)
+          end
+          metric.observe(obj["value"], obj["keys"])
         end
-        metric.observe(obj["value"], obj["keys"])
       end
     end
 
     def prometheus_metrics_text
       @mutex.synchronize do
-        @metrics.values.map(&:to_prometheus_text).join("\n")
+        val = @metrics.values.map(&:to_prometheus_text).join("\n")
+
+        metrics = {}
+
+        if @process_metrics.length > 0
+          val << "\n"
+
+          @process_metrics.map do |m|
+            metric_key = { pid: m["pid"], type: m["process_type"] }
+
+            PROCESS_GAUGES.map do |k, help|
+              k = k.to_s
+              if v = m[k]
+                g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new(k, help)
+                g.observe(v, metric_key)
+              end
+            end
+
+            PROCESS_COUNTERS.map do |k, help|
+              k = k.to_s
+              if v = m[k]
+                c = metrics[k] ||= PrometheusExporter::Metric::Counter.new(k, help)
+                c.observe(v, metric_key)
+              end
+            end
+
+          end
+
+          val << metrics.values.map(&:to_prometheus_text).join("\n")
+        end
+
+        val
       end
     end
 
+    protected
+
     def register_metric(metric)
       @mutex.synchronize do
         @metrics << metric
       end
     end
 
-    protected
+    def ensure_web_metrics
+      unless @http_requests
+        @metrics["http_requests"] = @http_requests = PrometheusExporter::Metric::Counter.new(
+          "http_requests",
+          "Total HTTP requests from web app"
+        )
+
+        @metrics["http_duration_seconds"] = @http_duration_seconds = PrometheusExporter::Metric::Summary.new(
+          "http_duration_seconds",
+          "Time spent in HTTP reqs in seconds"
+        )
+
+        @metrics["http_redis_duration_seconds"] = @http_redis_duration_seconds = PrometheusExporter::Metric::Summary.new(
+          "http_redis_duration_seconds",
+          "Time spent in HTTP reqs in redis seconds"
+        )
+
+        @metrics["http_sql_duration_seconds"] = @http_sql_duration_seconds = PrometheusExporter::Metric::Summary.new(
+          "http_sql_duration_seconds",
+          "Time spent in HTTP reqs in SQL in seconds"
+        )
+      end
+    end
+
+    def observe_web(obj)
+      ensure_web_metrics
+
+      labels = {
+        controller: obj["controller"] || "other",
+        action: obj["action"] || "other"
+      }
+
+      @http_requests.observe(1, labels.merge(status: obj["status"]))
+
+      if timings = obj["timings"]
+        @http_duration_seconds.observe(timings["total_duration"], labels)
+        if redis = timings["redis"]
+          @http_redis_duration_seconds.observe(redis["duration"], labels)
+        end
+        if sql = timings["sql"]
+          @http_sql_duration_seconds.observe(sql["duration"], labels)
+        end
+      end
+    end
+
+    def observe_process(obj)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      obj["created_at"] = now
+
+      @process_metrics.delete_if do |current|
+        obj["pid"] == current["pid"] || (current["created_at"] + MAX_PROCESS_METRIC_AGE < now)
+      end
+      @process_metrics << obj
+    end
 
     def register_metric_unsafe(obj)
       name = obj["name"]

data/lib/prometheus_exporter/server/collector_base.rb

@@ -0,0 +1,14 @@
+module PrometheusExporter::Server
+
+  # minimal interface to implement a customer collector
+  class CollectorBase
+
+    # called each time a string is delivered from the web
+    def process(str)
+    end
+
+    # a string denoting the metrics
+    def prometheus_metrics_text(str)
+    end
+  end
+end

data/lib/prometheus_exporter/server/web_server.rb

@@ -9,7 +9,9 @@ module PrometheusExporter::Server
   class WebServer
     attr_reader :collector
 
-    def initialize(port: , collector: nil)
+    def initialize(port: , collector: nil, verbose: false)
+
+      @verbose = verbose
 
       @total_metrics = PrometheusExporter::Metric::Counter.new("total_collector_metrics", "total metrics processed by exporter web")
 
@@ -21,10 +23,19 @@ module PrometheusExporter::Server
       @total_sessions.observe(0)
       @total_bad_metrics.observe(0)
 
+      access_log = []
+      if verbose
+        access_log = [
+          [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT],
+          [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT],
+        ]
+        logger = WEBrick::Log.new($stderr)
+      end
+
       @server = WEBrick::HTTPServer.new(
         Port: port,
-        AccessLog: [],
-        Logger: WEBrick::Log.new("/dev/null")
+        Logger: logger,
+        AccessLog: access_log
       )
 
       @collector = collector || Collector.new
@@ -52,7 +63,7 @@ module PrometheusExporter::Server
           handle_metrics(req, res)
         else
           res.status = 404
-          res.body = "Not Found! The Prometheus Discourse plugin only listens on /metrics and /send-metrics"
+          res.body = "Not Found! The Prometheus Ruby Exporter only listens on /metrics and /send-metrics"
         end
       end
     end
@@ -64,6 +75,12 @@ module PrometheusExporter::Server
           @total_metrics.observe
           @collector.process(block)
         rescue => e
+          if @verbose
+            STDERR.puts
+            STDERR.puts e.inspect
+            STDERR.puts e.backtrace
+            STDERR.puts
+          end
           @total_bad_metrics.observe
           res.body = "Bad Metrics #{e}"
           res.status = e.respond_to?(:status_code) ? e.status_code : 500

data/lib/prometheus_exporter/version.rb

@@ -1,3 +1,3 @@
 module PrometheusExporter
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/prometheus_exporter.gemspec

@@ -18,12 +18,13 @@ Gem::Specification.new do |spec|
     f.match(%r{^(test|spec|features|bin)/})
   end
   spec.bindir        = "bin"
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = ["prometheus_exporter"]
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
+  spec.add_development_dependency "mini_racer", "~> 0.1"
   spec.add_development_dependency "guard-minitest", "~> 2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Sam Saffron
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-14 00:00:00.000000000 Z
+date: 2018-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: mini_racer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: guard-minitest
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +97,8 @@ dependencies:
 description: Prometheus metric collector and exporter for Ruby
 email:
 - sam.saffron@gmail.com
-executables: []
+executables:
+- prometheus_exporter
 extensions: []
 extra_rdoc_files: []
 files:
@@ -97,16 +112,22 @@ files:
 - README.md
 - Rakefile
 - bench/bench.rb
+- bin/prometheus_exporter
 - examples/custom_collector.rb
 - lib/prometheus_exporter.rb
 - lib/prometheus_exporter/client.rb
+- lib/prometheus_exporter/instrumentation.rb
+- lib/prometheus_exporter/instrumentation/method_profiler.rb
+- lib/prometheus_exporter/instrumentation/process.rb
 - lib/prometheus_exporter/metric.rb
 - lib/prometheus_exporter/metric/base.rb
 - lib/prometheus_exporter/metric/counter.rb
 - lib/prometheus_exporter/metric/gauge.rb
 - lib/prometheus_exporter/metric/summary.rb
+- lib/prometheus_exporter/middleware.rb
 - lib/prometheus_exporter/server.rb
 - lib/prometheus_exporter/server/collector.rb
+- lib/prometheus_exporter/server/collector_base.rb
 - lib/prometheus_exporter/server/web_server.rb
 - lib/prometheus_exporter/version.rb
 - prometheus_exporter.gemspec