prometheus_exporter 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b9fae4f661a2154e78754266e14a491a3d4271a7f2bf308586d1d5c16975452
-  data.tar.gz: 3c12e51359809b26963be8d148ebf0775f83ad588ca6b7f352b855e73f553598
+  metadata.gz: 9a885aa655d0bd9d939bef77ec5d7c37027d19882a96b184d0351c49593393b9
+  data.tar.gz: bca2080108cb8169d5589ee481fc40b0edb39e890a8a0fba6ec301a987c35396
 SHA512:
-  metadata.gz: 866d3593b9d575451efec2a87dac5d0e4b03b9ed265b776c9ae3f65ce417e7cb04c8ee29769ba38d3fc5ad1abb5824c56c10f5ee7b8461d62629c95145f8c58d
-  data.tar.gz: 6dd42a1d78443807ca0b026617e55e23089e772620d0dfd88ccc3c2871a6096fb081ca0065060e943680158f7b78c88035bb8d64327f95e4756f3902add36fe3
+  metadata.gz: b21fe8ad31aec32b877ab50146759715c41968dd1edc510af7b986436983217fd7eb0c468faf2ceca7897ae855a36a88f3b6aef1e0dcf776846051e4cd35b822
+  data.tar.gz: 5a0a61bd914c269385bb6b46df97b2eca5e6fe4573092f10dfed7d92a4f052b62761faa53902bfdf93e2286dc96a13bc804dd56e820dec99c95a00550026b741

data/.github/workflows/ci.yml

@@ -1,7 +1,9 @@
-name: Test Exporter
+name: CI
 
 on:
   push:
+    branches:
+      - main
   pull_request:
   schedule:
     - cron: '0 0 * * 0' # weekly
@@ -9,41 +11,43 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: Ruby ${{ matrix.ruby }}
+    name: Ruby ${{ matrix.ruby }} AR ${{ matrix.activerecord }}
+    timeout-minutes: 10
+
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/ar_${{ matrix.activerecord }}.gemfile
+
     strategy:
+      fail-fast: false
       matrix:
-        ruby: ["3.0", "2.7", "2.6"]
+        ruby: [2.6, 2.7, 3.0]
+        activerecord: [60, 61]
+
     steps:
-      - uses: actions/checkout@master
-        with:
-          fetch-depth: 1
-      - uses: actions/setup-ruby@v1
+      - uses: actions/checkout@v2
+
+      - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-      - uses: actions/cache@v2
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-${{ matrix.ruby }}-gems-v2-${{ hashFiles('**/Gemfile.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.ruby }}-gems-v2-
-
-      - name: Downgrade rubygems
-        run: |
-          # for Ruby <= 2.6 , details https://github.com/rubygems/rubygems/issues/3284
-          gem update --system 3.0.8
-        if: ${{ matrix.ruby == '2.6' || matrix.ruby == '2.7' }}
-      - name: Upgrade rubygems
-        run: |
-          gem update --system
-      - name: Setup gems
-        run: |
-          gem install bundler
-          bundle config path vendor/bundle
-          bundle install --jobs 4
-          bundle exec appraisal install
+          bundler-cache: true
+
       - name: Rubocop
         run: bundle exec rubocop
-      - name: install gems
-        run: bundle exec appraisal bundle
+
       - name: Run tests
-        run: bundle exec appraisal rake
+        run: bundle exec rake
+
+  publish:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Release gem
+        uses: discourse/publish-rubygems-action@v2
+        env:
+          RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}
+          GIT_EMAIL: team@discourse.org
+          GIT_NAME: discoursebot

data/CHANGELOG

@@ -1,4 +1,10 @@
-1.0.0 - 23-11-2021
+1.0.1 - 2021-12-22
+
+- FEATURE: add labels to preflight requests
+- FEATURE: SidekiqStats metrics
+- FIX: mintor refactors to Sidekiq metrics
+
+1.0.0 - 2021-11-23
 
 - BREAKING: rename metrics to match prometheus official naming conventions (See https://prometheus.io/docs/practices/naming/#metric-names)
 - FEATURE: Sidekiq process metrics
@@ -6,11 +12,11 @@
 - FIX: logger improved for web server
 - FIX: Remove job labels from DelayedJob queues
 
-0.8.1 - 04-08-2021
+0.8.1 - 2021-08-04
 
 - FEATURE: swap from hardcoded STDERR to logger pattern (see README for details)
 
-0.8.0 - 05-07-2021
+0.8.0 - 2021-07-05
 
 - FIX: handle ThreadError more gracefully in cases where process shuts down
 - FEATURE: add job_name and queue_name labels to delayed job metrics
@@ -22,7 +28,7 @@
 - FEATURE: Improve Active Record instrumentation
 - FEATURE: Support HTTP_X_AMZN_TRACE_ID when supplied
 
-0.7.0 - 29-12-2020
+0.7.0 - 2020-12-29
 
 - Dev: Removed support from EOL rubies, only 2.5, 2.6, 2.7 and 3.0 are supported now.
 - Dev: Better support for Ruby 3.0, explicitly depending on webrick
@@ -30,111 +36,111 @@
 - FEATURE: clean pattern for overriding middleware labels was introduced (in README)
 - Fix: Better support for forking
 
-0.6.0 - 17-11-2020
+0.6.0 - 2020-11-17
 
 - FEATURE: add support for basic-auth in the prometheus_exporter web server
 
-0.5.3 - 29-07-2020
+0.5.3 - 2020-07-29
 
 - FEATURE: added #remove to all metric types so users can remove specific labels if needed
 
-0.5.2 - 01-07-2020
+0.5.2 - 2020-07-01
 
 - FEATURE: expanded instrumentation for sidekiq
 - FEATURE: configurable default labels
 
-0.5.1 - 25-02-2020
+0.5.1 - 2020-02-25
 
 - FEATURE: Allow configuring the default client's host and port via environment variables
 
-0.5.0 - 14-02-2020
+0.5.0 - 2020-02-14
 
 - Breaking change: listen only to localhost by default to prevent unintended insecure configuration
 - FIX: Avoid calling `hostname` aggressively, instead cache it on the exporter instance
 
-0.4.17 - 13-01-2020
+0.4.17 - 2020-01-13
 
 - FEATURE: add support for `to_h` on all metrics which can be used to query existing key/values
 
-0.4.16 - 04-11-2019
+0.4.16 - 2019-11-04
 
 - FEATURE: Support #reset! on all metric types to reset a metric to default
 
-0.4.15 - 04-11-2019
+0.4.15 - 2019-11-04
 
 - FEATURE: Improve delayed job collector, add pending counts
 - FEATURE: New ActiveRecord collector (documented in readme)
 - FEATURE: Allow passing in histogram and summary options
 - FEATURE: Allow custom labels for unicorn collector
 
-0.4.14 - 10-09-2019
+0.4.14 - 2019-09-10
 
 - FEATURE: allow finding metrics by name RemoteMetric #find_registered_metric
 - FIX: guard socket closing
 
-0.4.13 - 09-07-2019
+0.4.13 - 2019-07-09
 
 - Fix: Memory leak in unicorn and puma collectors
 
-0.4.12 - 30-05-2019
+0.4.12 - 2019-05-30
 
 - Fix: unicorn collector reporting incorrect number of unicorn workers
 
-0.4.11 - 15-05-2019
+0.4.11 - 2019-05-15
 
 - Fix: Handle stopping nil worker_threads in Client
 - Dev: add frozen string literals
 
-0.4.10 - 29-04-2019
+0.4.10 - 2019-04-29
 
 - Fix: Custom label support for puma collector
 - Fix: Raindrops socket collector not working correctly
 
-0.4.9 - 11-04-2019
+0.4.9 - 2019-04-11
 
 - Fix: Gem was not working correctly in Ruby 2.4 and below due to a syntax error
 
-0.4.8 - 10-04-2019
+0.4.8 - 2019-04-10
 
 - Feature: added helpers for instrumenting unicorn using raindrops
 
-0.4.7 - 08-04-2019
+0.4.7 - 2019-04-08
 
 - Fix: collector was not escaping " \ and \n correctly. This could lead
   to a corrupt payload in some cases.
 
-0.4.6 - 02-04-2019
+0.4.6 - 2019-04-02
 
 - Feature: Allow resetting a counter
 - Feature: Add sidekiq metrics: restarted, dead jobs counters
 - Fix: Client shutting down before sending metrics to collector
 
-0.4.5 - 14-02-2019
+0.4.5 - 2019-02-14
 
 - Feature: Allow process collector to ship custom labels for all process metrics
 - Fix: Always scope process metrics on hostname in collector
 
-0.4.4 - 13-02-2019
+0.4.4 - 2019-02-13
 
 - Feature: add support for local metric collection without using HTTP
 
-0.4.3 - 11-02-2019
+0.4.3 - 2019-02-11
 
 - Feature: Add alias for Gauge #observe called #set, this makes it a bit easier to migrate from prom
 - Feature: Add increment and decrement to Counter
 
-0.4.2 - 30-11-2018
+0.4.2 - 2018-11-30
 
 - Fix/Feature: setting a Gauge to nil will remove Gauge (setting to non numeric will raise)
 
-0.4.0 - 23-10-2018
+0.4.0 - 2018-10-23
 
 - Feature: histogram support
 - Feature: custom quantile support for summary
 - Feature: Puma metrics
 - Fix: delayed job metrics
 
-0.3.4 - 02-10-2018
+0.3.4 - 2018-10-02
 
 - Fix: custom collector via CLI was not working correctly
 

data/README.md

@@ -367,43 +367,49 @@ Metrics collected by Process instrumentation include labels `type` (as given wit
 
 #### Sidekiq metrics
 
-Including Sidekiq metrics (how many jobs ran? how many failed? how long did they take? how many are dead? how many were restarted?)
-
-```ruby
-Sidekiq.configure_server do |config|
-   config.server_middleware do |chain|
-      require 'prometheus_exporter/instrumentation'
-      chain.add PrometheusExporter::Instrumentation::Sidekiq
-   end
-   config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler
-end
-```
-
-To monitor Queue size and latency:
+There are different kinds of Sidekiq metrics that can be collected. A recommended setup looks like this:
 
 ```ruby
 Sidekiq.configure_server do |config|
+  require 'prometheus_exporter/instrumentation'
+  config.server_middleware do |chain|
+    chain.add PrometheusExporter::Instrumentation::Sidekiq
+  end
+  config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler
   config.on :startup do
-    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::Process.start type: 'sidekiq'
+    PrometheusExporter::Instrumentation::SidekiqProcess.start
     PrometheusExporter::Instrumentation::SidekiqQueue.start
+    PrometheusExporter::Instrumentation::SidekiqStats.start
   end
 end
 ```
 
-This will only monitor the queues that are consumed by the sidekiq process you are on.  You can pass an `all_queues` parameter to monitor metrics on all queues.
+* The middleware and death handler will generate job specific metrics (how many jobs ran? how many failed? how long did they take? how many are dead? how many were restarted?).
+* The [`Process`](#per-process-stats) metrics provide basic ruby metrics.
+* The `SidekiqProcess` metrics provide the concurrency and busy metrics for this process.
+* The `SidekiqQueue` metrics provides size and latency for the queues run by this process.
+* The `SidekiqStats` metrics provide general, global Sidekiq stats (size of Scheduled, Retries, Dead queues, total number of jobs, etc).
+
+For `SidekiqQueue`, if you run more than one process for the same queues, note that the same metrics will be exposed by all the processes, just like the `SidekiqStats` will if you run more than one process of any kind. You might want use `avg` or `max` when consuming their metrics.
 
-To monitor Sidekiq process info:
+An alternative would be to expose these metrics in lone, long-lived process. Using a rake task, for example:
 
 ```ruby
-Sidekiq.configure_server do |config|
-  config.on :startup do
-    require 'prometheus_exporter/instrumentation'
-    PrometheusExporter::Instrumentation::Process.start type: 'sidekiq'
-    PrometheusExporter::Instrumentation::SidekiqProcess.start
-  end
+task :sidekiq_metrics do
+  server = PrometheusExporter::Server::WebServer.new
+  server.start
+
+  PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(collector: server.collector)
+
+  PrometheusExporter::Instrumentation::SidekiqQueue.start(all_queues: true)
+  PrometheusExporter::Instrumentation::SidekiqStats.start
+  sleep
 end
 ```
 
+The `all_queues` parameter for `SidekiqQueue` will expose metrics for all queues.
+
 Sometimes the Sidekiq server shuts down before it can send metrics, that were generated right before the shutdown, to the collector. Especially if you care about the `sidekiq_restarted_jobs_total` metric, it is a good idea to explicitly stop the client:
 
 ```ruby
@@ -447,7 +453,21 @@ Both metrics will have a `queue` label with the name of the queue.
 | Gauge | `sidekiq_process_busy`        | Number of busy workers for this process |
 | Gauge | `sidekiq_process_concurrency` | Concurrency for this process            |
 
-Both metrics will include the labels `labels`, `queues`, `quiet`, `tag`, `hostname` and `identity`, as returned by the [Sidekiq API](https://github.com/mperham/sidekiq/wiki/API#processes).
+Both metrics will include the labels `labels`, `queues`, `quiet`, `tag`, `hostname` and `identity`, as returned by the [Sidekiq Processes API](https://github.com/mperham/sidekiq/wiki/API#processes).
+
+**PrometheusExporter::Instrumentation::SidekiqStats**
+| Type  | Name                            | Description                             |
+| ---   | ---                             | ---                                     |
+| Gauge | `sidekiq_stats_dead_size`       | Size of the dead queue                  |
+| Gauge | `sidekiq_stats_enqueued`        | Number of enqueued jobs                 |
+| Gauge | `sidekiq_stats_failed`          | Number of failed jobs                   |
+| Gauge | `sidekiq_stats_processed`       | Total number of processed jobs          |
+| Gauge | `sidekiq_stats_processes_size`  | Number of processes                     |
+| Gauge | `sidekiq_stats_retry_size`      | Size of the retries queue               |
+| Gauge | `sidekiq_stats_scheduled_size`  | Size of the scheduled queue             |
+| Gauge | `sidekiq_stats_workers_size`    | Number of jobs actively being processed |
+
+Based on the [Sidekiq Stats API](https://github.com/mperham/sidekiq/wiki/API#stats).
 
 _See [Metrics collected by Process Instrumentation](#metrics-collected-by-process-instrumentation) for a list of metrics the Process instrumentation will produce._
 

data/bin/prometheus_exporter

@@ -88,7 +88,7 @@ def run
   end
 
   if custom_collector_filename
-    eval File.read(custom_collector_filename), nil, File.expand_path(custom_collector_filename)
+    require File.expand_path(custom_collector_filename)
     found = false
 
     base_klass = PrometheusExporter::Server::CollectorBase
@@ -108,7 +108,7 @@ def run
 
   if custom_type_collectors_filenames.length > 0
     custom_type_collectors_filenames.each do |t|
-      eval File.read(t), nil, File.expand_path(t)
+      require File.expand_path(t)
     end
 
     ObjectSpace.each_object(Class) do |klass|

data/lib/prometheus_exporter/instrumentation/sidekiq.rb

@@ -73,7 +73,7 @@ module PrometheusExporter::Instrumentation
       # of the delayed extensions
       # https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/extensions/class_methods.rb
       begin
-        (target, method_name, _args) = YAML.load(msg['args'].first)
+        (target, method_name, _args) = YAML.load(msg['args'].first) # rubocop:disable Security/YAMLLoad
         if target.class == Class
           "#{target.name}##{method_name}"
         else

data/lib/prometheus_exporter/instrumentation/sidekiq_stats.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Instrumentation
+  class SidekiqStats
+    def self.start(client: nil, frequency: 30)
+      client ||= PrometheusExporter::Client.default
+      sidekiq_stats_collector = new
+
+      Thread.new do
+        loop do
+          begin
+            client.send_json(sidekiq_stats_collector.collect)
+          rescue StandardError => e
+            STDERR.puts("Prometheus Exporter Failed To Collect Sidekiq Stats metrics #{e}")
+          ensure
+            sleep frequency
+          end
+        end
+      end
+    end
+
+    def collect
+      {
+        type: 'sidekiq_stats',
+        stats: collect_stats
+      }
+    end
+
+    def collect_stats
+      stats = ::Sidekiq::Stats.new
+      {
+        'dead_size' => stats.dead_size,
+        'enqueued' => stats.enqueued,
+        'failed' => stats.failed,
+        'processed' => stats.processed,
+        'processes_size' => stats.processes_size,
+        'retry_size' => stats.retry_size,
+        'scheduled_size' => stats.scheduled_size,
+        'workers_size' => stats.workers_size,
+      }
+    end
+  end
+end

data/lib/prometheus_exporter/instrumentation.rb

@@ -6,6 +6,7 @@ require_relative "instrumentation/method_profiler"
 require_relative "instrumentation/sidekiq"
 require_relative "instrumentation/sidekiq_queue"
 require_relative "instrumentation/sidekiq_process"
+require_relative "instrumentation/sidekiq_stats"
 require_relative "instrumentation/delayed_job"
 require_relative "instrumentation/puma"
 require_relative "instrumentation/hutch"

data/lib/prometheus_exporter/middleware.rb

@@ -58,6 +58,11 @@ class PrometheusExporter::Middleware
     if params
       action = params["action"]
       controller = params["controller"]
+    elsif (cors = env["rack.cors"]) && cors.respond_to?(:preflight?) && cors.preflight?
+      # if the Rack CORS Middleware identifies the request as a preflight request,
+      # the stack doesn't get to the point where controllers/actions are defined
+      action = "preflight"
+      controller = "preflight"
     end
 
     {

data/lib/prometheus_exporter/server/collector.rb

@@ -15,6 +15,7 @@ module PrometheusExporter::Server
       register_collector(SidekiqCollector.new)
       register_collector(SidekiqQueueCollector.new)
       register_collector(SidekiqProcessCollector.new)
+      register_collector(SidekiqStatsCollector.new)
       register_collector(DelayedJobCollector.new)
       register_collector(PumaCollector.new)
       register_collector(HutchCollector.new)

data/lib/prometheus_exporter/server/sidekiq_process_collector.rb

@@ -26,7 +26,7 @@ module PrometheusExporter::Server
         SIDEKIQ_PROCESS_GAUGES.map do |name, help|
           if (value = metric[name])
             gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_process_#{name}", help)
-            gauges[name].observe(value, labels)
+            gauge.observe(value, labels)
           end
         end
       end

data/lib/prometheus_exporter/server/sidekiq_stats_collector.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module PrometheusExporter::Server
+  class SidekiqStatsCollector < TypeCollector
+    MAX_SIDEKIQ_METRIC_AGE = 60
+
+    SIDEKIQ_STATS_GAUGES = {
+      'dead_size' => 'Size of dead the queue',
+      'enqueued' => 'Number of enqueued jobs',
+      'failed' => 'Number of failed jobs',
+      'processed' => 'Total number of processed jobs',
+      'processes_size' => 'Number of processes',
+      'retry_size' => 'Size of the retries queue',
+      'scheduled_size' => 'Size of the scheduled queue',
+      'workers_size' => 'Number of jobs actively being processed',
+    }.freeze
+
+    attr_reader :sidekiq_metrics, :gauges
+
+    def initialize
+      @sidekiq_metrics = []
+      @gauges = {}
+    end
+
+    def type
+      'sidekiq_stats'
+    end
+
+    def metrics
+      sidekiq_metrics.map do |metric|
+        SIDEKIQ_STATS_GAUGES.map do |name, help|
+          if (value = metric['stats'][name])
+            gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("sidekiq_stats_#{name}", help)
+            gauge.observe(value)
+          end
+        end
+      end
+
+      gauges.values
+    end
+
+    def collect(object)
+      sidekiq_metrics << object
+    end
+  end
+end

data/lib/prometheus_exporter/server.rb

@@ -7,6 +7,7 @@ require_relative "server/process_collector"
 require_relative "server/sidekiq_collector"
 require_relative "server/sidekiq_queue_collector"
 require_relative "server/sidekiq_process_collector"
+require_relative "server/sidekiq_stats_collector"
 require_relative "server/delayed_job_collector"
 require_relative "server/collector_base"
 require_relative "server/collector"

data/lib/prometheus_exporter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PrometheusExporter
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

data/prometheus_exporter.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.0"
-  spec.add_development_dependency "mini_racer", "~> 0.3.1"
+  spec.add_development_dependency "mini_racer", "~> 0.5.0"
   spec.add_development_dependency "guard-minitest", "~> 2.0"
   spec.add_development_dependency "oj", "~> 3.0"
   spec.add_development_dependency "rack-test", "~> 0.8.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prometheus_exporter
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Sam Saffron
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-23 00:00:00.000000000 Z
+date: 2021-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webrick
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: guard-minitest
   requirement: !ruby/object:Gem::Requirement
@@ -245,6 +245,7 @@ files:
 - lib/prometheus_exporter/instrumentation/sidekiq.rb
 - lib/prometheus_exporter/instrumentation/sidekiq_process.rb
 - lib/prometheus_exporter/instrumentation/sidekiq_queue.rb
+- lib/prometheus_exporter/instrumentation/sidekiq_stats.rb
 - lib/prometheus_exporter/instrumentation/unicorn.rb
 - lib/prometheus_exporter/metric.rb
 - lib/prometheus_exporter/metric/base.rb
@@ -267,6 +268,7 @@ files:
 - lib/prometheus_exporter/server/sidekiq_collector.rb
 - lib/prometheus_exporter/server/sidekiq_process_collector.rb
 - lib/prometheus_exporter/server/sidekiq_queue_collector.rb
+- lib/prometheus_exporter/server/sidekiq_stats_collector.rb
 - lib/prometheus_exporter/server/type_collector.rb
 - lib/prometheus_exporter/server/unicorn_collector.rb
 - lib/prometheus_exporter/server/web_collector.rb