prometheus_exporter 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82048f57a2ee65b704b92d2e74c2e8033215e0331c8604c1e45a63d16f8c3a1d
-  data.tar.gz: fc18bcc997f15bbdb182d5199543cd11e6ccad6a774ee838586b313eaa2a6de6
+  metadata.gz: 194941c0e02ab01e7e35d9931b66d2403100eab66b7b8708b5e512a400293bb8
+  data.tar.gz: 4059f738ec94d58d91c676ac24da5f2d6abd006d4b38ac42ca18ba92af43ab6b
 SHA512:
-  metadata.gz: ec6a0e811258775d28e2de42713481ddde3bbd40a7c6ec9773aaf1f0a557facf5b85a6ca00646b17747381c9b39481d5f3cb6b53a66db676b0bc30e74549990f
-  data.tar.gz: c15fb3209e8045e113169aaa884de7bc7bfe96d2cb20dcc6a1c57e3a82b68b828e4b513c0fc96e7b156f1dcc721055f036a37fe747a684f70a2b01f6eb238061
+  metadata.gz: e1b835e597e67fbe25e3d8c61869f68e7c5d7d5dd51e6c0972508a3816dc81f3454ad7680b74245496a73e14e86ff43dc0569d50fe8488ece2637631f1f4c4cf
+  data.tar.gz: dcb886ecc1e568e93509fd73046cf2f498015791a41521ae567b14467ad4b2e33b93899a9145b997604bbc35916d45303e6845d31a2b284bd268bd685e4dea1e

data/CHANGELOG

@@ -1,8 +1,18 @@
-0.4.4 - 14-02-2019
+0.4.6 - 02-04-2019
+
+- Feature: Allow resetting a counter
+- Feature: Add sidekiq metrics: restarted, dead jobs counters
+- Fix: Client shutting down before sending metrics to collector
+
+0.4.5 - 14-02-2019
 
 - Feature: Allow process collector to ship custom labels for all process metrics
 - Fix: Always scope process metrics on hostname in collector
 
+0.4.4 - 13-02-2019
+
+- Feature: add support for local metric collection without using HTTP 
+
 0.4.3 - 11-02-2019
 
 - Feature: Add alias for Gauge #observe called #set, this makes it a bit easier to migrate from prom

data/README.md

@@ -196,7 +196,7 @@ end
 
 #### Sidekiq metrics
 
-Including Sidekiq metrics (how many jobs ran? how many failed? how long did they take?)
+Including Sidekiq metrics (how many jobs ran? how many failed? how long did they take? how many are dead? how many were restarted?)
 
 ```ruby
 Sidekiq.configure_server do |config|
@@ -204,6 +204,7 @@ Sidekiq.configure_server do |config|
       require 'prometheus_exporter/instrumentation'
       chain.add PrometheusExporter::Instrumentation::Sidekiq
    end
+   config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler
 end
 ```
 
@@ -218,6 +219,16 @@ Sidekiq.configure_server do |config|
 end
 ```
 
+Sometimes the Sidekiq server shuts down before it can send metrics, that were generated right before the shutdown, to the collector. Especially if you care about the `sidekiq_restarted_jobs_total` metric, it is a good idea to explicitly stop the client:
+
+```ruby
+  Sidekiq.configure_server do |config|
+    at_exit do
+      PrometheusExporter::Client.default.stop(wait_timeout_seconds: 10)
+    end
+  end
+```
+
 #### Delayed Job plugin
 
 In an initializer:

data/lib/prometheus_exporter/client.rb

@@ -120,16 +120,16 @@ module PrometheusExporter
       end
     end
 
-    def stop
+    def stop(wait_timeout_seconds: 0)
       @mutex.synchronize do
+        wait_for_empty_queue_with_timeout(wait_timeout_seconds)
         @worker_thread&.kill
         while @worker_thread.alive?
           sleep 0.001
         end
         @worker_thread = nil
       end
-
-      close_socket!
+        close_socket!
     end
 
     private
@@ -197,6 +197,13 @@ module PrometheusExporter
       raise
     end
 
+    def wait_for_empty_queue_with_timeout(timeout_seconds)
+      start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while @queue.length > 0
+        break if start_time + timeout_seconds < ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        sleep(0.05)
+      end
+    end
   end
 
   class LocalClient < Client
@@ -211,5 +218,4 @@ module PrometheusExporter
       @collector.process(json)
     end
   end
-
 end

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -2,6 +2,15 @@
 
 module PrometheusExporter::Instrumentation
   class Sidekiq
+    def self.death_handler
+      -> (job, ex) do
+        PrometheusExporter::Client.default.send_json(
+          type: "sidekiq",
+          name: job["class"],
+          dead: true,
+        )
+      end
+    end
 
     def initialize(client: nil)
       @client = client || PrometheusExporter::Client.default
@@ -9,19 +18,23 @@ module PrometheusExporter::Instrumentation
 
     def call(worker, msg, queue)
       success = false
+      shutdown = false
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       result = yield
       success = true
       result
+    rescue ::Sidekiq::Shutdown => e
+      shutdown = true
+      raise e
     ensure
       duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start
       class_name = worker.class.to_s == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' ?
                      msg['wrapped'] : worker.class.to_s
-
       @client.send_json(
         type: "sidekiq",
         name: class_name,
         success: success,
+        shutdown: shutdown,
         duration: duration
       )
     end

data/lib/prometheus_exporter/metric/counter.rb

@@ -33,5 +33,9 @@ module PrometheusExporter::Metric
       @data[labels] ||= 0
       @data[labels] -= value
     end
+
+    def reset(labels = {}, value = 0)
+      @data[labels] = value
+    end
   end
 end

data/lib/prometheus_exporter/server/sidekiq_collector.rb

@@ -11,14 +11,25 @@ module PrometheusExporter::Server
       labels = custom_labels.nil? ? default_labels : default_labels.merge(custom_labels)
 
       ensure_sidekiq_metrics
-      @sidekiq_job_duration_seconds.observe(obj["duration"], labels)
-      @sidekiq_jobs_total.observe(1, labels)
-      @sidekiq_failed_jobs_total.observe(1, labels) if !obj["success"]
+      if obj["dead"]
+        @sidekiq_dead_jobs_total.observe(1, labels)
+      else
+        @sidekiq_job_duration_seconds.observe(obj["duration"], labels)
+        @sidekiq_jobs_total.observe(1, labels)
+        @sidekiq_restarted_jobs_total.observe(1, labels) if obj["shutdown"]
+        @sidekiq_failed_jobs_total.observe(1, labels) if !obj["success"] && !obj["shutdown"]
+      end
     end
 
     def metrics
       if @sidekiq_jobs_total
-        [@sidekiq_job_duration_seconds, @sidekiq_jobs_total, @sidekiq_failed_jobs_total]
+        [
+          @sidekiq_job_duration_seconds,
+          @sidekiq_jobs_total,
+          @sidekiq_restarted_jobs_total,
+          @sidekiq_failed_jobs_total,
+          @sidekiq_dead_jobs_total,
+        ]
       else
         []
       end
@@ -37,9 +48,17 @@ module PrometheusExporter::Server
         PrometheusExporter::Metric::Counter.new(
           "sidekiq_jobs_total", "Total number of sidekiq jobs executed.")
 
+        @sidekiq_restarted_jobs_total =
+        PrometheusExporter::Metric::Counter.new(
+          "sidekiq_restarted_jobs_total", "Total number of sidekiq jobs that we restarted because of a sidekiq shutdown.")
+
         @sidekiq_failed_jobs_total =
         PrometheusExporter::Metric::Counter.new(
-          "sidekiq_failed_jobs_total", "Total number failed sidekiq jobs executed.")
+          "sidekiq_failed_jobs_total", "Total number of failed sidekiq jobs.")
+
+        @sidekiq_dead_jobs_total =
+        PrometheusExporter::Metric::Counter.new(
+          "sidekiq_dead_jobs_total", "Total number of dead sidekiq jobs.")
       end
     end
   end

data/lib/prometheus_exporter/version.rb

@@ -1,3 +1,3 @@
 module PrometheusExporter
-  VERSION = "0.4.5"
+  VERSION = "0.4.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
 platform: ruby
 authors:
 - Sam Saffron
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-13 00:00:00.000000000 Z
+date: 2019-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -207,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Prometheus Exporter