sidekiq_prometheus 0.8.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f2382deff51bcba928543262df3da7a6c1acd5c90b79a9aa427349d1d190cdf3
+  data.tar.gz: 905e723a9efa06ae33df67872e254af30a9f145fb2ef45dfd4bcf5bbe447c120
+SHA512:
+  metadata.gz: 35eb0aecd12715f0e2c29d500a89ee4ed33042f1a004fef4e8db62ddf942bcd85b1f41aac228f66e3d7e9bc29360871d7b0034d194a4057846a5afb5827093e4
+  data.tar.gz: a7f66949d6db7dde393c77a19872cb2384be079988f3d598bf7031510d30c766d28311223c56fa54353e72b5b1f12ce65f986317bc97e37ed1a143fd1a65b068

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.rspec_status
+Gemfile.lock

data/.rubocop.yml

@@ -0,0 +1,133 @@
+AllCops:
+  TargetRubyVersion: 2.5
+  DisplayCopNames: true
+  Include:
+    - '**/*.gemspec'
+    - '**/*.rake'
+    - '**/Gemfile'
+    - '**/Rakefile'
+    - '**/*.rb'
+  Exclude:
+    - db/schema.rb
+    - bin/setup
+    - bin/update
+    - vendor/**/*
+
+Style/RedundantReturn:
+  AllowMultipleReturnValues: true
+
+HashSyntax:
+  Enabled: false
+
+BlockNesting:
+  Enabled: true
+
+AssignmentInCondition:
+  Enabled: false
+
+CollectionMethods:
+  Enabled: false
+
+Lint/MissingCopEnableDirective:
+  Enabled: false
+
+Metrics/AbcSize:
+  Max: 75
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 50
+
+Metrics/ClassLength:
+  Max: 300
+
+Metrics/ModuleLength:
+  Max: 300
+
+Metrics/CyclomaticComplexity:
+  Max: 10
+
+Metrics/PerceivedComplexity:
+  Max: 10
+
+Metrics/BlockLength:
+  Enabled: true
+  ExcludedMethods:
+    - class_methods
+    - included
+
+  Exclude:
+    - Gemfile
+    - spec/**/*
+    - '*.gemspec'
+
+Naming/PredicateName:
+  Enabled: false
+
+Naming/FileName:
+  Exclude:
+    - Capfile
+    - Appraisals
+
+Performance/TimesMap:
+  Enabled: false
+
+HandleExceptions:
+  Enabled: false
+
+Documentation:
+  Enabled: false
+
+RegexpLiteral:
+  Enabled: false
+
+SignalException:
+  EnforcedStyle: only_raise
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/BlockDelimiters:
+  EnforcedStyle: braces_for_chaining
+
+Style/BracesAroundHashParameters:
+  EnforcedStyle: context_dependent
+
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/TrailingCommaInArguments:
+  Enabled: false
+
+Style/ClassAndModuleChildren:
+  EnforcedStyle: compact
+
+Style/TrivialAccessors:
+  Enabled: true
+  ExactNameMatch: true
+
+Style/LineEndConcatenation:
+  Enabled: false
+
+Style/EachWithObject:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Metrics/ParameterLists:
+  CountKeywordArgs: false
+
+Style/FormatStringToken:
+  Enabled: false
+
+Naming/MemoizedInstanceVariableName:
+  Enabled: false
+
+Style/RescueModifier:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5
+  - 2.4
+  - 2.3
+before_install: gem install bundler -v 1.16.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at leklund@fastly.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in sidekiq_prometheus.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Fastly, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,177 @@
+# SidekiqPrometheus
+
+Prometheus Instrumentation for Sidekiq.
+
+* Sidekiq server middleware for reporting job metrics
+* Global metrics reporter using the Sidekiq API for reporting Sidekiq cluster stats (requires Sidekiq::Enterprise)
+* Sidecar Rack server to provide scrape-able endpoint for Prometheus
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq_prometheus'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq_prometheus
+
+## Usage
+
+To run with the defaults add this to your Sidekiq initializer
+
+```
+SidekiqPrometheus.setup
+```
+
+This will register metrics, start the global reporter (if available), and start the Rack server for scraping. The default port is 9357 but this is easily configurable.
+
+If you are running multiple services that will be reporting Sidekiq metrics you will want to take advantage of the `base_labels` configuration option. For example:
+
+```
+SidekiqPrometheus.configure do |config|
+  config.base_labels  = { service: 'image_api' }
+  config.metrics_port = 9090
+end
+
+# always call setup after configure.
+SidekiqPrometheus.setup
+```
+
+The call to `setup` is necessary as that it what registers the metrics and instruments Sidekiq.
+There is also a helper method: `SidekiqPrometheus.configure!` which can be used to configure and setup in one step:
+
+```
+SidekiqPrometheus.configure! do |config|
+  config.base_labels  = { service: 'dogs_api' }
+end
+
+# No need to call SidekiqPrometheus.setup because the bang method did it for us/
+```
+
+Once sidekiq server is running you can see your metrics or scrape them with Prometheus:
+
+```
+curl http://localhost:8675/metrics
+```
+
+#### Configuration options
+
+* `base_labels`: Hash of labels that will be included with every metric when they are registered.
+* `gc_metrics_enabled`: Boolean that determines whether to record object allocation metrics per job. The default is `true`. Setting this to `false` if you don't need this metric.
+* `global_metrics_enabled`: Boolean that determines whether to report global metrics from the PeriodicMetrics reporter. When `true` this will report on a number of stats from the Sidekiq API for the cluster. This requires Sidekiq::Enterprise as the reporter uses the leader election functionality to ensure that only one worker per cluster is reporting metrics.
+* `periodic_metrics_enabled`: Boolean that determines whether to run the periodic metrics reporter. `PeriodicMetrics` runs a separate thread that reports on global metrics (if enabled) as well worker GC stats (if enabled). It reports metrics on the interval defined by `periodic_reporting_interval`. Defatuls to `true`.
+* `periodic_reporting_interval`: interval in seconds for reporting periodic metrics. Default: `30`
+* `metrics_port`: Port on which the rack server will listen. Defaults to `9357`
+
+```
+SidekiqPrometheus.configure do |config|
+  config.base_labels                   = { service: 'myapp' }
+  config.gc_metrics_enabled            = false
+  config.global_metrics_enabled        = true
+  config.periodic_metrics_enabled      = true
+  config.periodic_reporting_interval   = 20
+  config.metrics_port                  = 8675
+end
+```
+
+Custom labels may be added by defining the `prometheus_labels` method in the worker class:
+
+```
+class SomeWorker
+  include Sidekiq::Worker
+
+  def prometheus_labels
+    { some: 'label' }
+  end
+end
+```
+
+## Metrics
+
+### JobMetrics
+
+All Sidekiq job metrics are reported with these labels:
+
+* `class`: Sidekiq worker class name
+* `queue`: Sidekiq queue name
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| sidekiq_job_count | counter | Count of Sidekiq jobs |
+| sidekiq_job_duration | histogram | Sidekiq job processing duration |
+| sidekiq_job_success | counter | Count of successful Sidekiq jobs |
+| sidekiq_job_allocated_objects | histogram | Count of ruby objects allocated by a Sidekiq job |
+| sidekiq_job_failed | counter | Count of failed Sidekiq jobs |
+
+Notes:
+
+* when a job fails only `sidekiq_job_count` and `sidekiq_job_failed` will be reported.
+* `sidekiq_job_allocated_objects` will only be reported if `SidekiqPrometheus.gc_metrics_enabled? == true`
+
+### Periodic GC Metrics
+
+These require `SidekiqPrometheus.gc_metrics_enabled? == true` and `SidekiqPrometheus.periodic_metrics_enabled? == true`
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| sidekiq_allocated_objects | counter | Count of allocated objects by the worker |
+| sidekiq_heap_free_slots | gauge | Number of free heap slots as reported by GC.stat |
+| sidekiq_heap_live_slots | gauge | Number of live heap slots as reported by GC.stat |
+| sidekiq_major_gc_count | counter | Count of major GC runs |
+| sidekiq_minor_gc_count | counter | Count of minor GC runs |
+| sidekiq_rss | gauge | RSS memory usage for worker process |
+
+### Periodic Global Metrics
+
+These require `SidekiqPrometheus.global_metrics_enabled? == true` and `SidekiqPrometheus.periodic_metrics_enabled? == true`
+
+Periodic metric reporting relies onSidekiq Enterprise's leader election functionality ([Ent Leader Election ](https://github.com/mperham/sidekiq/wiki/Ent-Leader-Election))
+which ensures that metrics are only reported once per cluster.
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| sidekiq_workers_size | gauge | Total number of workers processing jobs |
+| sidekiq_dead_size | gauge | Total Dead Size |
+| sidekiq_enqueued | gauge | Total Size of all known queues |
+| sidekiq_queue_latency | summary | Latency (in seconds) of all queues |
+| sidekiq_failed | gauge | Number of job executions which raised an error |
+| sidekiq_processed | gauge | Number of job executions completed (success or failure) |
+| sidekiq_retry_size | gauge | Total Retries Size |
+| sidekiq_scheduled_size | gauge | Total Scheduled Size |
+| sidekiq_redis_connected_clients | gauge | Number of clients connected to Redis instance for Sidekiq |
+| sidekiq_redis_used_memory | gauge | Used memory from Redis.info
+| sidekiq_redis_used_memory_peak | gauge | Used memory peak from Redis.info |
+| sidekiq_redis_keys | gauge | Number of redis keys |
+| sidekiq_redis_expires | gauge | Number of redis keys with expiry set |
+
+The global metrics are reported with the only the `base_labels` with the exception of `sidekiq_enqueued` which will add a `queue` label and record a metric per Sidekiq queue.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/fastly/sidekiq_prometheus. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## Copyright
+
+Copyright 2019 Fastly, Inc.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the SidekiqPrometheus project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/sidekiq_prometheus/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+begin
+  require 'rubocop/rake_task'
+  RuboCop::RakeTask.new(:rubocop) do |task|
+    task.options = ['-c.rubocop.yml']
+  end
+rescue LoadError
+end
+
+task default: %w[
+  rubocop
+  spec
+]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'sidekiq_prometheus'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/sidekiq_prometheus.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+require 'rack'
+require 'prometheus/client'
+require 'prometheus/middleware/exporter'
+require 'sidekiq'
+require 'sidekiq/api'
+
+begin
+  require 'sidekiq/ent'
+rescue LoadError
+end
+
+module SidekiqPrometheus
+  class << self
+    # @return [Hash] Base labels applied to every registered metric
+    attr_accessor :base_labels
+
+    # @return [Hash] Custom labels applied to specific metrics
+    attr_accessor :custom_labels
+
+    # @return [Boolean] Setting to control enabling/disabling GC metrics. Default: true
+    attr_accessor :gc_metrics_enabled
+
+    # @return [Boolean] Setting to control enabling/disabling global metrics. Default: true
+    attr_accessor :global_metrics_enabled
+
+    # @return [Boolean] Setting to control enabling/disabling periodic metrics. Default: true
+    attr_accessor :periodic_metrics_enabled
+
+    # @return [Integer] Interval in seconds to record metrics. Default: 30
+    attr_accessor :periodic_reporting_interval
+
+    # @return [Integer] Port on which the metrics server will listen. Default: 9357
+    attr_accessor :metrics_port
+
+    # Override the default Prometheus::Client
+    # @return [Prometheus::Client]
+    attr_writer :client
+
+    # Orverride the default Prometheus Metric Registry
+    # @return [Prometheus::Client::Registry]
+    attr_writer :registry
+  end
+
+  self.gc_metrics_enabled = true
+  self.periodic_metrics_enabled = true
+  self.global_metrics_enabled = true
+  self.periodic_reporting_interval = 30
+  self.metrics_port = 9359
+  self.custom_labels = {}
+
+  module_function
+
+  ##
+  # @return Prometheus::Client
+  def client
+    @client ||= Prometheus::Client
+  end
+
+  ##
+  # Configure SidekiqPrometheus
+  # @example
+  #   SidekiqPrometheus.configure do |config|
+  #     config.base_labels = { service: 'images_api' }
+  #     config.custom_labels = { sidekiq_job_count: { object_klass: nil } }
+  #     config.gc_metrics_enabled = true
+  #   end
+  def configure
+    yield self
+  end
+
+  ##
+  # Configure and call setup immediately after
+  def configure!
+    yield self
+    setup
+  end
+
+  ##
+  # Helper method for +gc_metrics_enabled+ configuration setting
+  # @return [Boolean] defaults to true
+  def gc_metrics_enabled?
+    gc_metrics_enabled
+  end
+
+  ##
+  # Helper method for +global_metrics_enabled+ configuration setting
+  # Requires +Sidekiq::Enterprise+ as it uses the leader election functionality
+  # @return [Boolean] defaults to true if +Sidekiq::Enterprise+ is available
+  def global_metrics_enabled?
+    Object.const_defined?('Sidekiq::Enterprise') && global_metrics_enabled
+  end
+
+  ##
+  # Helper method for +periodic_metrics_enabled+ configuration setting
+  # Requires +Sidekiq::Enterprise+ as it uses the leader election functionality
+  # @return [Boolean] defaults to true if +Sidekiq::Enterprise+ is available
+  def periodic_metrics_enabled?
+    periodic_metrics_enabled
+  end
+
+  ##
+  # Get a metric from the registry
+  # @param metric [Symbol] name of metric to fetch
+  # @return [Prometheus::Client::Metric]
+  def [](metric)
+    registry.get(metric.to_sym)
+  end
+
+  class << self
+    alias get []
+  end
+
+  ##
+  # Prometheus client metric registry
+  # @return [Prometheus::Client::Registry]
+  def registry
+    @registry ||= client.registry
+  end
+
+  ##
+  # register metrics and instrument sidekiq
+  def setup
+    SidekiqPrometheus::Metrics.register_sidekiq_job_metrics
+    SidekiqPrometheus::Metrics.register_sidekiq_gc_metric if gc_metrics_enabled?
+    SidekiqPrometheus::Metrics.register_sidekiq_worker_gc_metrics if gc_metrics_enabled? && periodic_metrics_enabled?
+    SidekiqPrometheus::Metrics.register_sidekiq_global_metrics if global_metrics_enabled? && periodic_metrics_enabled?
+    sidekiq_setup
+  end
+
+  ##
+  # Add Prometheus instrumentation to sidekiq
+  def sidekiq_setup
+    Sidekiq.configure_server do |config|
+      config.server_middleware do |chain|
+        chain.add SidekiqPrometheus::JobMetrics
+      end
+
+      if periodic_metrics_enabled?
+        config.on(:startup)  { SidekiqPrometheus::PeriodicMetrics.reporter.start }
+        config.on(:shutdown) { SidekiqPrometheus::PeriodicMetrics.reporter.stop }
+      end
+
+      config.on(:startup)  { SidekiqPrometheus.metrics_server }
+      config.on(:shutdown) { SidekiqPrometheus.metrics_server.kill }
+    end
+  end
+
+  ##
+  # Start a new Prometheus exporter in a new thread.
+  # Will listen on SidekiqPrometheus.metrics_port
+  def metrics_server
+    @_metrics_server ||= Thread.new do
+      Rack::Handler::WEBrick.run(
+        Rack::Builder.new {
+          use Prometheus::Middleware::Exporter, registry: SidekiqPrometheus.registry
+          run ->(_) { [301, { 'Location' => '/metrics' }, []] }
+        },
+        Port: SidekiqPrometheus.metrics_port,
+        BindAddress: '127.0.0.1',
+      )
+    end
+  end
+
+  private
+
+  def metrics
+    SidekiqPrometheus::Metrics
+  end
+end
+
+require 'sidekiq_prometheus/job_metrics'
+require 'sidekiq_prometheus/metrics'
+require 'sidekiq_prometheus/periodic_metrics'
+require 'sidekiq_prometheus/version'

data/lib/sidekiq_prometheus/job_metrics.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class SidekiqPrometheus::JobMetrics
+  def call(worker, _job, queue)
+    before = GC.stat(:total_allocated_objects) if SidekiqPrometheus.gc_metrics_enabled?
+
+    labels = { class: worker.class.to_s, queue: queue }
+
+    begin
+      labels.merge!(custom_labels(worker))
+
+      result = nil
+      duration = Benchmark.realtime { result = yield }
+
+      # In case the labels have changed after the worker perform method has been called
+      labels.merge!(custom_labels(worker))
+
+      registry[:sidekiq_job_duration].observe(labels, duration)
+      registry[:sidekiq_job_success].increment(labels)
+
+      if SidekiqPrometheus.gc_metrics_enabled?
+        allocated = GC.stat(:total_allocated_objects) - before
+        registry[:sidekiq_job_allocated_objects].observe(labels, allocated)
+      end
+
+      result
+    rescue StandardError => e
+      registry[:sidekiq_job_failed].increment(labels)
+      raise e
+    ensure
+      registry[:sidekiq_job_count].increment(labels)
+    end
+  end
+
+  private
+
+  def registry
+    SidekiqPrometheus
+  end
+
+  def custom_labels(worker)
+    worker.respond_to?(:prometheus_labels) ? worker.prometheus_labels : {}
+  end
+end

data/lib/sidekiq_prometheus/metrics.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module SidekiqPrometheus::Metrics
+  module_function
+
+  UNKNOWN = 'unknown'
+
+  VALID_TYPES = %i[counter gauge histogram summary].freeze
+  SIDEKIQ_GLOBAL_METRICS = [
+    { name:      :sidekiq_workers_size,
+      type:      :gauge,
+      docstring: 'Total number of workers processing jobs', },
+    { name:      :sidekiq_dead_size,
+      type:      :gauge,
+      docstring: 'Total Dead Size', },
+    { name:      :sidekiq_enqueued,
+      type:      :gauge,
+      docstring: 'Total Size of all known queues', },
+    { name:      :sidekiq_queue_latency,
+      type:      :summary,
+      docstring: 'Latency (in seconds) of all queues', },
+    { name:      :sidekiq_failed,
+      type:      :gauge,
+      docstring: 'Number of job executions which raised an error', },
+    { name:      :sidekiq_processed,
+      type:      :gauge,
+      docstring: 'Number of job executions completed (success or failure)', },
+    { name:      :sidekiq_retry_size,
+      type:      :gauge,
+      docstring: 'Total Retries Size', },
+    { name:      :sidekiq_scheduled_size,
+      type:      :gauge,
+      docstring: 'Total Scheduled Size', },
+    { name:      :sidekiq_redis_connected_clients,
+      type:      :gauge,
+      docstring: 'Number of clients connected to Redis instance for Sidekiq', },
+    { name:      :sidekiq_redis_used_memory,
+      type:      :gauge,
+      docstring: 'Used memory from Redis.info', },
+    { name:      :sidekiq_redis_used_memory_peak,
+      type:      :gauge,
+      docstring: 'Used memory peak from Redis.info', },
+    { name:      :sidekiq_redis_keys,
+      type:      :gauge,
+      docstring: 'Number of redis keys', },
+    { name:      :sidekiq_redis_expires,
+      type:      :gauge,
+      docstring: 'Number of redis keys with expiry set', },
+  ].freeze
+  SIDEKIQ_JOB_METRICS = [
+    { name:      :sidekiq_job_count,
+      type:      :counter,
+      docstring: 'Count of Sidekiq jobs', },
+    { name:      :sidekiq_job_duration,
+      type:      :histogram,
+      docstring: 'Sidekiq job processing duration', },
+    { name:      :sidekiq_job_failed,
+      type:      :counter,
+      docstring: 'Count of failed Sidekiq jobs', },
+    { name:      :sidekiq_job_success,
+      type:      :counter,
+      docstring: 'Count of successful Sidekiq jobs', },
+  ].freeze
+  SIDEKIQ_GC_METRIC = {
+    name:      :sidekiq_job_allocated_objects,
+    type:      :histogram,
+    docstring: 'Count of ruby objects allocated by a Sidekiq job',
+    buckets:   [10, 50, 100, 500, 1_000, 2_500, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 25_000_000],
+  }.freeze
+  SIDEKIQ_WORKER_GC_METRICS = [
+    { name:      :sidekiq_allocated_objects,
+      type:      :counter,
+      docstring: 'Count of ruby objects allocated by a Sidekiq worker', },
+    { name:      :sidekiq_heap_free_slots,
+      type:      :gauge,
+      docstring: 'Sidekiq worker GC.stat[:heap_free_slots]', },
+    { name:      :sidekiq_heap_live_slots,
+      type:      :gauge,
+      docstring: 'Sidekiq worker GC.stat[:heap_live_slots]', },
+    { name:      :sidekiq_major_gc_count,
+      type:      :counter,
+      docstring: 'Sidekiq worker GC.stat[:major_gc_count]', },
+    { name:      :sidekiq_minor_gc_count,
+      type:      :counter,
+      docstring: 'Sidekiq worker GC.stat[:minor_gc_count]', },
+    { name:      :sidekiq_rss,
+      type:      :gauge,
+      docstring: 'Sidekiq process RSS', },
+  ].freeze
+
+  def registry
+    SidekiqPrometheus.registry
+  end
+
+  def register_sidekiq_job_metrics
+    register_metrics SIDEKIQ_JOB_METRICS
+  end
+
+  def register_sidekiq_gc_metric
+    register SIDEKIQ_GC_METRIC
+  end
+
+  def register_sidekiq_worker_gc_metrics
+    register_metrics SIDEKIQ_WORKER_GC_METRICS
+  end
+
+  def register_sidekiq_global_metrics
+    register_metrics SIDEKIQ_GLOBAL_METRICS
+  end
+
+  def register_metrics(metrics)
+    metrics.each do |metric|
+      register(metric)
+    end
+  end
+
+  ##
+  # Fetch a metric from the registry
+  # @param name [Symbol] name of metric to fetch
+  def [](name)
+    registry.get(name.to_sym)
+  end
+
+  class << self
+    alias get []
+  end
+
+  ##
+  # Register a new metric
+  # @param types [Symbol] type of metric to register. Valid types: %w(counter gauge summary histogram)
+  # @param name [Symbol] name of metric
+  # @param docstring [String] help text for metric
+  # @param base_labels [Hash] Optional hash of base labels to use for every instance of this metric
+  # @param buckets [Hash] Optional hash of bucket values. Only used for histogram metrics.
+  def register(type:, name:, docstring:, base_labels: {}, buckets: nil)
+    raise InvalidMetricType, type unless VALID_TYPES.include? type
+
+    custom_labels = SidekiqPrometheus.custom_labels[name]
+
+    base_labels.merge! custom_labels if custom_labels&.is_a?(Hash)
+    base_labels.merge! SidekiqPrometheus.base_labels if SidekiqPrometheus.base_labels
+
+    args = [name.to_sym, docstring]
+    args << base_labels
+    args << buckets if buckets
+
+    registry.send(type, *args)
+  end
+
+  def unregister(name:)
+    registry.unregister(name.to_sym)
+  end
+
+  class InvalidMetricType < StandardError; end
+end

data/lib/sidekiq_prometheus/periodic_metrics.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+##
+# Report Sidekiq::Stats to prometheus on a defined interval
+#
+# Global Metrics reporting requires Sidekiq::Enterprise as it uses the leader
+# election functionality to ensure that the global metrics are only reported by
+# one worker.
+#
+# @see https://github.com/mperham/sidekiq/wiki/Ent-Leader-Election
+# @see https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/api.rb
+
+class SidekiqPrometheus::PeriodicMetrics
+  # @return [Boolean] When +true+ will stop the reporting loop.
+  attr_accessor :done
+
+  # @return [Integer] Interval in seconds to record metrics. Default: [SidekiqPrometheus.periodic_reporting_interval]
+  attr_reader :interval
+  attr_reader :senate, :sidekiq_stats, :sidekiq_queue
+
+  GLOBAL_STATS = %i[failed processed retry_size dead_size scheduled_size workers_size].freeze
+  GC_STATS = {
+    counters: %i[major_gc_count minor_gc_count total_allocated_objects],
+    gauges: %i[heap_live_slots heap_free_slots],
+  }.freeze
+  REDIS_STATS = %w[connected_clients used_memory used_memory_peak].freeze
+
+  ##
+  # Instance of SidekiqPrometheus::PeriodicMetrics
+  # @return [SidekiqPrometheus:PeriodicMetrics]
+  def self.reporter
+    @reporter ||= new
+  end
+
+  ##
+  # @param interval [Integer] Interval in seconds to record metrics.
+  # @param sidekiq_stats [Sidekiq::Stats]
+  # @param senate [#leader?] Sidekiq::Senate
+  def initialize(interval: SidekiqPrometheus.periodic_reporting_interval, sidekiq_stats: Sidekiq::Stats, sidekiq_queue: Sidekiq::Queue, senate: nil)
+    self.done = false
+    @interval = interval
+
+    @sidekiq_stats = sidekiq_stats
+    @sidekiq_queue = sidekiq_queue
+    @senate = if senate.nil?
+                if Object.const_defined?('Sidekiq::Senate')
+                  Sidekiq::Senate
+                else
+                  Senate
+                end
+              else
+                senate
+              end
+  end
+
+  ##
+  # Start the period mettric reporter
+  def start
+    Sidekiq.logger.info('SidekiqPrometheus: Starting periodic metrics reporting')
+    @thread = Thread.new(&method(:run))
+  end
+
+  ##
+  ## Stop the periodic metric reporter
+  def stop
+    self.done = true
+  end
+
+  ##
+  # Record GC and RSS metrics
+  def report_gc_metrics
+    stats = GC.stat
+    GC_STATS[:counters].each do |stat|
+      SidekiqPrometheus["sidekiq_#{stat}"]&.increment({}, stats[stat])
+    end
+    GC_STATS[:gauges].each do |stat|
+      SidekiqPrometheus["sidekiq_#{stat}"]&.set({}, stats[stat])
+    end
+
+    SidekiqPrometheus[:sidekiq_rss]&.set({}, rss)
+  end
+
+  ##
+  # Records Sidekiq global metrics
+  def report_global_metrics
+    current_stats = sidekiq_stats.new
+    GLOBAL_STATS.each do |stat|
+      SidekiqPrometheus["sidekiq_#{stat}"]&.set({}, current_stats.send(stat))
+    end
+
+    sidekiq_queue.all.each do |queue|
+      SidekiqPrometheus[:sidekiq_enqueued]&.set({ queue: queue.name }, queue.size)
+      SidekiqPrometheus[:sidekiq_queue_latency]&.observe({ queue: queue.name }, queue.latency)
+    end
+  end
+
+  ##
+  # Records metrics from Redis
+  def report_redis_metrics
+    redis_info = begin
+      Sidekiq.redis_info
+    rescue Redis::BaseConnectionError
+      nil
+    end
+
+    return if redis_info.nil?
+
+    REDIS_STATS.each do |stat|
+      SidekiqPrometheus["sidekiq_redis_#{stat}"]&.set({}, redis_info[stat].to_i)
+    end
+
+    db_stats = redis_info.select { |k, _v| k.match(/^db/) }
+    db_stats.each do |db, stat|
+      label = { database: db }
+      values = stat.scan(/\d+/)
+      SidekiqPrometheus[:sidekiq_redis_keys]&.set(label, values[0].to_i)
+      SidekiqPrometheus[:sidekiq_redis_expires]&.set(label, values[1].to_i)
+    end
+  end
+
+  ##
+  # Fetch rss from proc filesystem
+  # @see https://github.com/discourse/prometheus_exporter/blob/v0.3.3/lib/prometheus_exporter/instrumentation/process.rb#L39-L42
+  def rss
+    pid = Process.pid
+    @pagesize ||= `getconf PAGESIZE`.to_i rescue 4096
+    File.read("/proc/#{pid}/statm").split(' ')[1].to_i * @pagesize rescue 0
+  end
+
+  ##
+  # Report metrics and sleep for @interval seconds in a loop.
+  # Runs until @done is true
+  def run
+    until done
+      begin
+        report_global_metrics if SidekiqPrometheus.global_metrics_enabled? && senate.leader?
+        report_redis_metrics if SidekiqPrometheus.global_metrics_enabled? && senate.leader?
+        report_gc_metrics if SidekiqPrometheus.gc_metrics_enabled?
+      rescue StandardError => e
+        Sidekiq.logger.error e
+      ensure
+        sleep interval
+      end
+    end
+  end
+
+  ##
+  # Fake Senate class to guard against undefined constant errors.
+  # @private
+  class Senate
+    def leader?
+      false
+    end
+  end
+end

data/lib/sidekiq_prometheus/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SidekiqPrometheus
+  VERSION = '0.8.1'
+end

data/sidekiq_prometheus.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq_prometheus/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'sidekiq_prometheus'
+  spec.version       = SidekiqPrometheus::VERSION
+  spec.authors       = ['Lukas Eklund']
+  spec.email         = ['leklund@fastly.com']
+
+  spec.summary       = 'Prometheus Instrumentation for Sidekiq'
+  spec.homepage      = 'https://github.com/fastly/sidekiq-prometheus'
+  spec.license       = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.58.0'
+
+  spec.add_runtime_dependency 'prometheus-client', '~> 0.8.0'
+  spec.add_runtime_dependency 'rack'
+  spec.add_runtime_dependency 'sidekiq', '~> 5.1'
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq_prometheus
+version: !ruby/object:Gem::Version
+  version: 0.8.1
+platform: ruby
+authors:
+- Lukas Eklund
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-02-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.58.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.58.0
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+description: 
+email:
+- leklund@fastly.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/sidekiq_prometheus.rb
+- lib/sidekiq_prometheus/job_metrics.rb
+- lib/sidekiq_prometheus/metrics.rb
+- lib/sidekiq_prometheus/periodic_metrics.rb
+- lib/sidekiq_prometheus/version.rb
+- sidekiq_prometheus.gemspec
+homepage: https://github.com/fastly/sidekiq-prometheus
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: Prometheus Instrumentation for Sidekiq
+test_files: []