prometheus_exporter 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86a92ab30771272b941d1d453d9a48ec06b77f614ec966490702c270b90e369b
-  data.tar.gz: c606f0aacbcd0d79607ae3ea7f4bbe63b2632c4c431a8f9d19a86d8b072a8543
+  metadata.gz: '0670768fb402bef5d0e563cf3fb0a8299637f03f975f73ed7bf58ee8571bb682'
+  data.tar.gz: 1db696d86b7de07df16fd2abd3b35cc44d0d19e041d98cd6dc6738f245eccea4
 SHA512:
-  metadata.gz: 58ec62718537d203ebb9cfa2b2f52ce02ffd878c15845d4fba8a5f0e9c505719b5567af123d075f8e88ff6afed7efdff282178a67a552a335255376cf5799ecd
-  data.tar.gz: 2058b506f5485c86033a9534642a8740852dd351dc43a04cb5eb10fb78921bb7dc85a74505d0830b0c4d7b1abcc722ababcc0d3a2c20f6307ed608af9a38f706
+  metadata.gz: 062d0dbedc25eb1f4c267f434b4f4f1a76b9a70c3c1b0eb8d8c4d44afe63003737152afff75af41099a7e48b81c8c6206e9ace60cd26f92a8a3664030dfd13ee
+  data.tar.gz: 1a2afb4920a3cd2d15214b1b212d7828e62b022a804fba37fb1a8797a6ca93c6c90da8333524474e2dc32271e1a8a1abbb4e554be40d5df52c1ec4d65e723d7b

data/bin/prometheus_exporter

@@ -8,8 +8,10 @@ require_relative "./../lib/prometheus_exporter/server"
 def run
   port = PrometheusExporter::DEFAULT_PORT
   prefix = "ruby_"
-  collector = nil
+  collector_filename = nil
   verbose = false
+  type_collectors = []
+  collector_class = PrometheusExporter::Server::Collector
 
   OptionParser.new do |opt|
     opt.on('-p',
@@ -21,8 +23,11 @@ def run
     opt.on('--prefix METRIC_PREFIX', String, "Prefix to apply to all metrics (default: #{prefix})") do |o|
       prefix = o.to_s
     end
-    opt.on('-c', '--collector CUSTOM_COLLECTOR', String, "(optional) Custom collector to run") do |o|
-      collector = o.to_s
+    opt.on('-c', '--collector FILE', String, "(optional) Custom collector to run") do |o|
+      collector_filename = o.to_s
+    end
+    opt.on('-a', '--type-collector FILE', String, "(optional) Custom type collectors to run in main collector") do |o|
+      type_collectors << o
     end
     opt.on('-v', '--verbose') do |o|
       verbose = true
@@ -32,24 +37,42 @@ def run
 
   PrometheusExporter::Metric::Base.default_prefix = prefix
 
-  if collector
-    eval File.read(collector)
+  if collector_filename
+    eval File.read(collector_filename)
+
+    found = false
 
     ObjectSpace.each_object(Class) do |klass|
       if klass < PrometheusExporter::Server::CollectorBase
-        collector = klass
+        collector_class = klass
+        found = true
       end
     end
 
-    if !collector
-      STDERR.puts "Can not find a class inheriting off PrometheusExporter::Server::Collector"
+    if !found
+      STDERR.puts "Can not find a class inheriting off PrometheusExporter::Server::CollectorBase"
       usage
       exit 1
     end
   end
 
+  collector = collector_class.new
+
+  if type_collectors.length > 0
+    type_collectors.each do |t|
+      eval File.read(t)
+    end
+
+    ObjectSpace.each_object(Class) do |klass|
+      if klass < PrometheusExporter::Server::TypeCollector
+        collector.register_collector klass.new
+        STDERR.puts "Registered TypeCollector: #{klass}" if verbose
+      end
+    end
+  end
+
   puts "#{Time.now} Starting prometheus exporter on port #{port}"
-  server = PrometheusExporter::Server::WebServer.new port: port, collector: collector&.new, verbose: verbose
+  server = PrometheusExporter::Server::WebServer.new port: port, collector: collector, verbose: verbose
   server.start
   sleep
 

data/lib/prometheus_exporter/server.rb

@@ -1,3 +1,7 @@
+require_relative "server/type_collector"
+require_relative "server/web_collector"
+require_relative "server/process_collector"
+require_relative "server/sidekiq_collector"
 require_relative "server/collector_base"
 require_relative "server/collector"
 require_relative "server/web_server"

data/lib/prometheus_exporter/server/collector.rb

@@ -23,19 +23,22 @@ module PrometheusExporter::Server
     def initialize
       @process_metrics = []
       @metrics = {}
-      @buffer = []
       @mutex = Mutex.new
+      @collectors = {}
+      register_collector(WebCollector.new)
+      register_collector(ProcessCollector.new)
+      register_collector(SidekiqCollector.new)
+    end
+
+    def register_collector(collector)
+      @collectors[collector.type] = collector
     end
 
     def process(str)
       obj = JSON.parse(str)
       @mutex.synchronize do
-        if obj["type"] == "web"
-          observe_web(obj)
-        elsif obj["type"] == "process"
-          observe_process(obj)
-        elsif obj["type"] == "sidekiq"
-          observe_sidekiq(obj)
+        if collector = @collectors[obj["type"]]
+          collector.observe(obj)
         else
           metric = @metrics[obj["name"]]
           if !metric
@@ -48,132 +51,13 @@ module PrometheusExporter::Server
 
     def prometheus_metrics_text
       @mutex.synchronize do
-        val = @metrics.values.map(&:to_prometheus_text).join("\n")
-
-        metrics = {}
-
-        if @process_metrics.length > 0
-          val << "\n"
-
-          @process_metrics.map do |m|
-            metric_key = { pid: m["pid"], type: m["process_type"] }
-
-            PROCESS_GAUGES.map do |k, help|
-              k = k.to_s
-              if v = m[k]
-                g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new(k, help)
-                g.observe(v, metric_key)
-              end
-            end
-
-            PROCESS_COUNTERS.map do |k, help|
-              k = k.to_s
-              if v = m[k]
-                c = metrics[k] ||= PrometheusExporter::Metric::Counter.new(k, help)
-                c.observe(v, metric_key)
-              end
-            end
-
-          end
-
-          val << metrics.values.map(&:to_prometheus_text).join("\n")
-        end
-
-        val
+        (@metrics.values + @collectors.values.map(&:metrics).flatten)
+          .map(&:to_prometheus_text).join("\n")
       end
     end
 
     protected
 
-    def register_metric(metric)
-      @mutex.synchronize do
-        @metrics << metric
-      end
-    end
-
-    def ensure_web_metrics
-      unless @http_requests
-        @metrics["http_requests"] = @http_requests = PrometheusExporter::Metric::Counter.new(
-          "http_requests",
-          "Total HTTP requests from web app"
-        )
-
-        @metrics["http_duration_seconds"] = @http_duration_seconds = PrometheusExporter::Metric::Summary.new(
-          "http_duration_seconds",
-          "Time spent in HTTP reqs in seconds"
-        )
-
-        @metrics["http_redis_duration_seconds"] = @http_redis_duration_seconds = PrometheusExporter::Metric::Summary.new(
-          "http_redis_duration_seconds",
-          "Time spent in HTTP reqs in redis seconds"
-        )
-
-        @metrics["http_sql_duration_seconds"] = @http_sql_duration_seconds = PrometheusExporter::Metric::Summary.new(
-          "http_sql_duration_seconds",
-          "Time spent in HTTP reqs in SQL in seconds"
-        )
-      end
-    end
-
-    def observe_web(obj)
-      ensure_web_metrics
-
-      labels = {
-        controller: obj["controller"] || "other",
-        action: obj["action"] || "other"
-      }
-
-      @http_requests.observe(1, labels.merge(status: obj["status"]))
-
-      if timings = obj["timings"]
-        @http_duration_seconds.observe(timings["total_duration"], labels)
-        if redis = timings["redis"]
-          @http_redis_duration_seconds.observe(redis["duration"], labels)
-        end
-        if sql = timings["sql"]
-          @http_sql_duration_seconds.observe(sql["duration"], labels)
-        end
-      end
-    end
-
-    def observe_process(obj)
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-
-      obj["created_at"] = now
-
-      @process_metrics.delete_if do |current|
-        obj["pid"] == current["pid"] || (current["created_at"] + MAX_PROCESS_METRIC_AGE < now)
-      end
-      @process_metrics << obj
-    end
-
-    def observe_sidekiq(obj)
-      ensure_sidekiq_metrics
-      @sidekiq_job_duration_seconds.observe(obj["duration"], job_name: obj["name"])
-      @sidekiq_job_count.observe(1, job_name: obj["name"])
-      @sidekiq_failed_job_count.observe(1, job_name: obj["name"]) if !obj["success"]
-    end
-
-    def ensure_sidekiq_metrics
-      if !@sidekiq_job_count
-
-        @metrics["sidekiq_job_duration_seconds"] =
-          @sidekiq_job_duration_seconds =
-          PrometheusExporter::Metric::Counter.new(
-            "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs")
-
-        @metrics["sidekiq_job_count"] =
-          @sidekiq_job_count =
-          PrometheusExporter::Metric::Counter.new(
-            "sidekiq_job_count", "Total number of sidekiq jobs executed")
-
-        @metrics["sidekiq_failed_job_count"] =
-          @sidekiq_failed_job_count =
-          PrometheusExporter::Metric::Counter.new(
-            "sidekiq_failed_job_count", "Total number failed sidekiq jobs executed")
-      end
-    end
-
     def register_metric_unsafe(obj)
       name = obj["name"]
       help = obj["help"]

data/lib/prometheus_exporter/server/process_collector.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+
+  class ProcessCollector < TypeCollector
+    MAX_PROCESS_METRIC_AGE = 60
+    PROCESS_GAUGES = {
+      heap_free_slots: "Free ruby heap slots",
+      heap_live_slots: "Used ruby heap slots",
+      v8_heap_size: "Total JavaScript V8 heap size (bytes)",
+      v8_used_heap_size: "Total used JavaScript V8 heap size (bytes)",
+      v8_physical_size: "Physical size consumed by V8 heaps",
+      v8_heap_count: "Number of V8 contexts running",
+      rss: "Total RSS used by process",
+    }
+
+    PROCESS_COUNTERS = {
+      major_gc_count: "Major GC operations by process",
+      minor_gc_count: "Minor GC operations by process",
+      total_allocated_objects: "Total number of allocateds objects by process",
+    }
+
+    def initialize
+      @process_metrics = []
+    end
+
+    def type
+      "process"
+    end
+
+    def metrics
+      return [] if @process_metrics.length == 0
+
+      metrics = {}
+
+      @process_metrics.map do |m|
+        metric_key = { pid: m["pid"], type: m["process_type"] }
+
+        PROCESS_GAUGES.map do |k, help|
+          k = k.to_s
+          if v = m[k]
+            g = metrics[k] ||= PrometheusExporter::Metric::Gauge.new(k, help)
+            g.observe(v, metric_key)
+          end
+        end
+
+        PROCESS_COUNTERS.map do |k, help|
+          k = k.to_s
+          if v = m[k]
+            c = metrics[k] ||= PrometheusExporter::Metric::Counter.new(k, help)
+            c.observe(v, metric_key)
+          end
+        end
+      end
+
+      metrics.values
+    end
+
+    def observe(obj)
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+      obj["created_at"] = now
+
+      @process_metrics.delete_if do |current|
+        obj["pid"] == current["pid"] || (current["created_at"] + MAX_PROCESS_METRIC_AGE < now)
+      end
+      @process_metrics << obj
+    end
+  end
+end

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -0,0 +1,42 @@
+module PrometheusExporter::Server
+  class SidekiqCollector < TypeCollector
+
+    def type
+      "sidekiq"
+    end
+
+    def observe(obj)
+      ensure_sidekiq_metrics
+      @sidekiq_job_duration_seconds.observe(obj["duration"], job_name: obj["name"])
+      @sidekiq_job_count.observe(1, job_name: obj["name"])
+      @sidekiq_failed_job_count.observe(1, job_name: obj["name"]) if !obj["success"]
+    end
+
+    def metrics
+      if @sidekiq_job_count
+        [@sidekiq_job_duration_seconds, @sidekiq_job_count, @sidekiq_failed_job_count]
+      else
+        []
+      end
+    end
+
+    protected
+
+    def ensure_sidekiq_metrics
+      if !@sidekiq_job_count
+
+        @sidekiq_job_duration_seconds =
+        PrometheusExporter::Metric::Counter.new(
+          "sidekiq_job_duration_seconds", "Total time spent in sidekiq jobs")
+
+        @sidekiq_job_count =
+        PrometheusExporter::Metric::Counter.new(
+          "sidekiq_job_count", "Total number of sidekiq jobs executed")
+
+        @sidekiq_failed_job_count =
+        PrometheusExporter::Metric::Counter.new(
+          "sidekiq_failed_job_count", "Total number failed sidekiq jobs executed")
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/server/type_collector.rb

@@ -0,0 +1,15 @@
+module PrometheusExporter::Server
+  class TypeCollector
+    def type
+      raise "must implement type"
+    end
+
+    def collect(obj)
+      raise "must implement collect"
+    end
+
+    def metrics
+      raise "must implement metrics"
+    end
+  end
+end

data/lib/prometheus_exporter/server/web_collector.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class WebCollector < TypeCollector
+    def initialize
+      @metrics = {}
+    end
+
+    def type
+      "web"
+    end
+
+    def collect(obj)
+      ensure_metrics
+      observe(obj)
+    end
+
+    def metrics
+      @metrics.values
+    end
+
+    protected
+
+    def ensure_metrics
+      unless @http_requests
+        @metrics["http_requests"] = @http_requests = PrometheusExporter::Metric::Counter.new(
+          "http_requests",
+          "Total HTTP requests from web app"
+        )
+
+        @metrics["http_duration_seconds"] = @http_duration_seconds = PrometheusExporter::Metric::Summary.new(
+          "http_duration_seconds",
+          "Time spent in HTTP reqs in seconds"
+        )
+
+        @metrics["http_redis_duration_seconds"] = @http_redis_duration_seconds = PrometheusExporter::Metric::Summary.new(
+          "http_redis_duration_seconds",
+          "Time spent in HTTP reqs in redis seconds"
+        )
+
+        @metrics["http_sql_duration_seconds"] = @http_sql_duration_seconds = PrometheusExporter::Metric::Summary.new(
+          "http_sql_duration_seconds",
+          "Time spent in HTTP reqs in SQL in seconds"
+        )
+      end
+    end
+
+    def observe(obj)
+
+      labels = {
+        controller: obj["controller"] || "other",
+        action: obj["action"] || "other"
+      }
+
+      @http_requests.observe(1, labels.merge(status: obj["status"]))
+
+      if timings = obj["timings"]
+        @http_duration_seconds.observe(timings["total_duration"], labels)
+        if redis = timings["redis"]
+          @http_redis_duration_seconds.observe(redis["duration"], labels)
+        end
+        if sql = timings["sql"]
+          @http_sql_duration_seconds.observe(sql["duration"], labels)
+        end
+      end
+    end
+  end
+end

data/lib/prometheus_exporter/version.rb

@@ -1,3 +1,3 @@
 module PrometheusExporter
-  VERSION = "0.1.9"
+  VERSION = "0.1.10"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Sam Saffron
@@ -130,6 +130,10 @@ files:
 - lib/prometheus_exporter/server.rb
 - lib/prometheus_exporter/server/collector.rb
 - lib/prometheus_exporter/server/collector_base.rb
+- lib/prometheus_exporter/server/process_collector.rb
+- lib/prometheus_exporter/server/sidekiq_collector.rb
+- lib/prometheus_exporter/server/type_collector.rb
+- lib/prometheus_exporter/server/web_collector.rb
 - lib/prometheus_exporter/server/web_server.rb
 - lib/prometheus_exporter/version.rb
 - prometheus_exporter.gemspec