sidekiq_prometheus 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2382deff51bcba928543262df3da7a6c1acd5c90b79a9aa427349d1d190cdf3
-  data.tar.gz: 905e723a9efa06ae33df67872e254af30a9f145fb2ef45dfd4bcf5bbe447c120
+  metadata.gz: 748f399320010ab1e703781c7a2c938b43c7d2a546218fe2a53397f60785be47
+  data.tar.gz: f7122e72637d68d0a5357308a6695994a0a6671569307e4223a1fd38efc624ba
 SHA512:
-  metadata.gz: 35eb0aecd12715f0e2c29d500a89ee4ed33042f1a004fef4e8db62ddf942bcd85b1f41aac228f66e3d7e9bc29360871d7b0034d194a4057846a5afb5827093e4
-  data.tar.gz: a7f66949d6db7dde393c77a19872cb2384be079988f3d598bf7031510d30c766d28311223c56fa54353e72b5b1f12ce65f986317bc97e37ed1a143fd1a65b068
+  metadata.gz: 598de4311c9de184940ead8482a150eb242cf610500404914a7f1ed3ce6eea8395021b1bb007d65567227d97056b8ebe1e392688ad5a7f40b4e760d90f71683b
+  data.tar.gz: b56334f343b2cce15e905a609ed11ac33ce5f2a42042bc6641ffe20444df2c26f5108ee275f76bca18ebcfd588321529a2d86e5edbb9d48acfff99fef1d4adb8

data/.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,39 @@
+CHANGELOG
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }})
+
+{{ range .CommitGroups -}}
+### {{ .Title }}
+
+{{ range .Commits -}}
+* {{ .Subject }}
+{{ end }}
+{{ end -}}
+
+{{- if .RevertCommits -}}
+### Reverts
+
+{{ range .RevertCommits -}}
+* {{ .Revert.Header }}
+{{ end }}
+{{ end -}}
+
+{{- if .MergeCommits -}}
+### Pull Requests
+
+{{ range .MergeCommits -}}
+* {{ .Header }}
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

data/.chglog/config.yml

@@ -0,0 +1,27 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/fastly/sidekiq-prometheus
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+    # title_maps:
+    #   feat: Features
+    #   fix: Bug Fixes
+    #   perf: Performance Improvements
+    #   refactor: Code Refactoring
+  header:
+    pattern: "^(\\w*)\\:\\s(.*)$"
+    pattern_maps:
+      - Type
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+CHANGELOG
+
+<a name="v0.8.1"></a>
+## v0.8.1 (2019-02-08)
+
+Initial Release!

data/README.md

@@ -1,11 +1,15 @@
-# SidekiqPrometheus
+# Sidekiq Prometheus
 
-Prometheus Instrumentation for Sidekiq.
 
-* Sidekiq server middleware for reporting job metrics
-* Global metrics reporter using the Sidekiq API for reporting Sidekiq cluster stats (requires Sidekiq::Enterprise)
-* Sidecar Rack server to provide scrape-able endpoint for Prometheus
+[![Status](https://travis-ci.org/fastly/sidekiq-prometheus.svg?branch=master)](https://travis-ci.org/fastly/sidekiq-prometheus)
+![Gem](https://img.shields.io/gem/v/sidekiq_prometheus.svg?color=blue)
+[![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg)](http://opensource.org/licenses/MIT)
+
+Prometheus Instrumentation for Sidekiq. This gem provides:
 
+* Sidekiq server middleware for reporting job metrics
+* Global metrics reporter that uses the Sidekiq API for reporting Sidekiq cluster stats (*requires Sidekiq::Enterprise*)
+* Sidecar Rack server to provide scrape-able endpoint for Prometheus. This allows for metrics to be reported without having to run a separate prometheus exporter process.
 
 ## Installation
 
@@ -31,35 +35,42 @@ To run with the defaults add this to your Sidekiq initializer
 SidekiqPrometheus.setup
 ```
 
-This will register metrics, start the global reporter (if available), and start the Rack server for scraping. The default port is 9357 but this is easily configurable.
+This will register metrics, start the global reporter (if available), and start the Rack server for scraping. The default port is 9359 but this is easily configurable.
 
-If you are running multiple services that will be reporting Sidekiq metrics you will want to take advantage of the `base_labels` configuration option. For example:
+Once Sidekiq server is running you can see your metrics or scrape them with Prometheus:
 
 ```
-SidekiqPrometheus.configure do |config|
-  config.base_labels  = { service: 'image_api' }
-  config.metrics_port = 9090
-end
+$curl http://localhost:9359/metrics
 
-# always call setup after configure.
-SidekiqPrometheus.setup
+# TYPE sidekiq_job_count counter
+# HELP sidekiq_job_count Count of Sidekiq jobs
+# TYPE sidekiq_job_duration histogram
+# HELP sidekiq_job_duration Sidekiq job processing duration
+
+[etc]
 ```
 
-The call to `setup` is necessary as that it what registers the metrics and instruments Sidekiq.
-There is also a helper method: `SidekiqPrometheus.configure!` which can be used to configure and setup in one step:
+[Full documentation](https://www.rubydoc.info/gems/sidekiq_prometheus)
 
-```
-SidekiqPrometheus.configure! do |config|
-  config.base_labels  = { service: 'dogs_api' }
-end
+## Configuration
 
-# No need to call SidekiqPrometheus.setup because the bang method did it for us/
+You can configure the gem by calling `configure`:
+
+```ruby
+SidekiqPrometheus.configure do |config|
+  config.base_labels = { service: 'kubernandos_api' }
+end
 ```
 
-Once sidekiq server is running you can see your metrics or scrape them with Prometheus:
+`configure` will automatically call setup so
 
-```
-curl http://localhost:8675/metrics
+If you are running multiple services that will be reporting Sidekiq metrics you will want to take advantage of the `base_labels` configuration option. For example:
+
+```ruby
+SidekiqPrometheus.configure do |config|
+  config.base_labels  = { service: 'image_api' }
+  config.metrics_port = 9090
+end
 ```
 
 #### Configuration options
@@ -67,11 +78,12 @@ curl http://localhost:8675/metrics
 * `base_labels`: Hash of labels that will be included with every metric when they are registered.
 * `gc_metrics_enabled`: Boolean that determines whether to record object allocation metrics per job. The default is `true`. Setting this to `false` if you don't need this metric.
 * `global_metrics_enabled`: Boolean that determines whether to report global metrics from the PeriodicMetrics reporter. When `true` this will report on a number of stats from the Sidekiq API for the cluster. This requires Sidekiq::Enterprise as the reporter uses the leader election functionality to ensure that only one worker per cluster is reporting metrics.
-* `periodic_metrics_enabled`: Boolean that determines whether to run the periodic metrics reporter. `PeriodicMetrics` runs a separate thread that reports on global metrics (if enabled) as well worker GC stats (if enabled). It reports metrics on the interval defined by `periodic_reporting_interval`. Defatuls to `true`.
+* `periodic_metrics_enabled`: Boolean that determines whether to run the periodic metrics reporter. `PeriodicMetrics` runs a separate thread that reports on global metrics (if enabled) as well worker GC stats (if enabled). It reports metrics on the interval defined by `periodic_reporting_interval`. Defaults to `true`.
 * `periodic_reporting_interval`: interval in seconds for reporting periodic metrics. Default: `30`
-* `metrics_port`: Port on which the rack server will listen. Defaults to `9357`
+* `metrics_port`: Port on which the rack server will listen. Defaults to `9359`
+* `registry`: An instance of `Prometheus::Client::Registry`. If you have a registry with defined metrics you can use this option to pass in your registry.
 
-```
+```ruby
 SidekiqPrometheus.configure do |config|
   config.base_labels                   = { service: 'myapp' }
   config.gc_metrics_enabled            = false
@@ -84,7 +96,7 @@ end
 
 Custom labels may be added by defining the `prometheus_labels` method in the worker class:
 
-```
+```ruby
 class SomeWorker
   include Sidekiq::Worker
 
@@ -133,7 +145,7 @@ These require `SidekiqPrometheus.gc_metrics_enabled? == true` and `SidekiqPromet
 
 These require `SidekiqPrometheus.global_metrics_enabled? == true` and `SidekiqPrometheus.periodic_metrics_enabled? == true`
 
-Periodic metric reporting relies onSidekiq Enterprise's leader election functionality ([Ent Leader Election ](https://github.com/mperham/sidekiq/wiki/Ent-Leader-Election))
+Periodic metric reporting relies on Sidekiq Enterprise's leader election functionality ([Ent Leader Election ](https://github.com/mperham/sidekiq/wiki/Ent-Leader-Election))
 which ensures that metrics are only reported once per cluster.
 
 | Metric | Type | Description |
@@ -154,6 +166,78 @@ which ensures that metrics are only reported once per cluster.
 
 The global metrics are reported with the only the `base_labels` with the exception of `sidekiq_enqueued` which will add a `queue` label and record a metric per Sidekiq queue.
 
+## Custom Worker Metrics
+
+There are a few different ways to register custom metrics with SidekiqPrometheus. Each custom metric should be defined as a Hash with the following form:
+
+```ruby
+{
+  name:        :metric_name,
+  type:        :gauge,
+  docstring:   'description',
+  base_labels: { label_name: 'label_text' },
+}
+```
+
+* `:name` (required) - Unique name of the metric and should be a symbol.
+* `:type` (required) - Prometheus metric type. Supported values are: `:counter`, `:gauge`, `:histogram`, and `:summary`.
+* `:docstring` (required) - Human readable description of the metric.
+* `:base_labels` (optional) - Hash of labels that will be applied to every instance of this metric.
+
+#### Registering custom metrics:
+
+Registering a set of custom metrics is done by defining `custom_metrics` in the `configure` block:
+
+```ruby
+SidekiqPrometheus.configure do |config|
+  config.custom_metrics = [
+    { name: :imported_records, type: :counter, :docstring: 'Count of successfully imported records' },
+    { name: :failed_records,   type: counter:, :docstring: 'Count of failed records' },
+  ]
+end
+```
+
+Metrics can also be registered directly. This must done *after* `SidekiqPrometheus.configure` or `setup` has been run.
+
+```ruby
+SidekiqPrometheus::Metrics.register(name: :logged_in_users, type: :gauge, docstring: 'Logged in users')
+```
+
+There is also a method to register more than one metric at a time:
+
+```ruby
+customer_worker_metrics = [
+  {
+    name: :file_count, type: :counter, docstring: 'Number of active files',
+    name: :file_size,  type: :gauge,   docstring: 'Size of files in bytes',    
+  }
+]
+
+SidekiqPrometheus::Metrics.register_metrics(customer_worker_metrics)
+```
+
+#### Using custom metrics:
+
+Once metrics are registered they can be used in your Sidekiq workers.
+
+```ruby
+class ImportWorker
+  include Sidekiq::Worker
+
+  LABELS = {}
+
+  def perform(*args)
+    # worker code
+
+    SidekiqPrometheus[:file_count].increment(LABELS, new_file_count)
+    SidekiqPrometheus[:file_size].set(LABELS, total_file_size)
+  end
+
+end
+```
+
+See the [documentation of the Prometheus::Client library](https://github.com/prometheus/client_ruby) for all of the available options for setting metric values.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -162,7 +246,7 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/fastly/sidekiq_prometheus. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/fastly/sidekiq-prometheus. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## Copyright
 
@@ -174,4 +258,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the SidekiqPrometheus project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/sidekiq_prometheus/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the SidekiqPrometheus project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/fastly/sidekiq-prometheus/blob/master/CODE_OF_CONDUCT.md).

data/lib/sidekiq_prometheus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqPrometheus
-  VERSION = '0.8.1'
+  VERSION = '0.9.0'
 end

data/lib/sidekiq_prometheus.rb

@@ -20,6 +20,21 @@ module SidekiqPrometheus
     # @return [Hash] Custom labels applied to specific metrics
     attr_accessor :custom_labels
 
+    # @return [Array] Custom metrics that will be registered on setup.
+    # @example
+    #   [
+    #     {
+    #       name: :metric_name,
+    #       type: :prometheus_metric_type,
+    #       docstring: 'Description of the metric',
+    #       base_labels : { label: 'value' },
+    #     }
+    #   ]
+    # @note Each element of the array is a hash and must have the required keys: `:name`, `:type`, and `:docstring`.
+    #   The values for `:name` and `:type` should be symbols and `:docstring` should be a string.
+    #   `base_labels` is optional and, if used, must be a hash of labels that will be included on every instance of this metric.
+    attr_accessor :custom_metrics
+
     # @return [Boolean] Setting to control enabling/disabling GC metrics. Default: true
     attr_accessor :gc_metrics_enabled
 
@@ -42,6 +57,9 @@ module SidekiqPrometheus
     # Orverride the default Prometheus Metric Registry
     # @return [Prometheus::Client::Registry]
     attr_writer :registry
+
+    # @private
+    attr_writer :setup_complete
   end
 
   self.gc_metrics_enabled = true
@@ -50,6 +68,7 @@ module SidekiqPrometheus
   self.periodic_reporting_interval = 30
   self.metrics_port = 9359
   self.custom_labels = {}
+  self.custom_metrics = []
 
   module_function
 
@@ -60,7 +79,7 @@ module SidekiqPrometheus
   end
 
   ##
-  # Configure SidekiqPrometheus
+  # Configure SidekiqPrometheus and setup for reporting
   # @example
   #   SidekiqPrometheus.configure do |config|
   #     config.base_labels = { service: 'images_api' }
@@ -69,16 +88,11 @@ module SidekiqPrometheus
   #   end
   def configure
     yield self
-  end
-
-  ##
-  # Configure and call setup immediately after
-  def configure!
-    yield self
     setup
   end
 
-  ##
+  alias configure! configure
+
   # Helper method for +gc_metrics_enabled+ configuration setting
   # @return [Boolean] defaults to true
   def gc_metrics_enabled?
@@ -117,17 +131,33 @@ module SidekiqPrometheus
   # Prometheus client metric registry
   # @return [Prometheus::Client::Registry]
   def registry
-    @registry ||= client.registry
+    @registry ||= client::Registry.new
+  end
+
+  ##
+  # Register custom metrics
+  # Internal method called by +setup+. This method should not be called from application code in most cases.
+  def register_custom_metrics
+    return if custom_metrics.empty?
+
+    raise SidekiqPrometheus::Error, 'custom_metrics is not an array.' unless custom_metrics.is_a?(Array)
+
+    SidekiqPrometheus::Metrics.register_metrics(custom_metrics)
   end
 
   ##
   # register metrics and instrument sidekiq
   def setup
+    return false if @setup_complete
     SidekiqPrometheus::Metrics.register_sidekiq_job_metrics
     SidekiqPrometheus::Metrics.register_sidekiq_gc_metric if gc_metrics_enabled?
     SidekiqPrometheus::Metrics.register_sidekiq_worker_gc_metrics if gc_metrics_enabled? && periodic_metrics_enabled?
     SidekiqPrometheus::Metrics.register_sidekiq_global_metrics if global_metrics_enabled? && periodic_metrics_enabled?
+    register_custom_metrics
+
     sidekiq_setup
+
+    self.setup_complete = true
   end
 
   ##
@@ -163,14 +193,10 @@ module SidekiqPrometheus
       )
     end
   end
-
-  private
-
-  def metrics
-    SidekiqPrometheus::Metrics
-  end
 end
 
+class SidekiqPrometheus::Error < StandardError; end
+
 require 'sidekiq_prometheus/job_metrics'
 require 'sidekiq_prometheus/metrics'
 require 'sidekiq_prometheus/periodic_metrics'

data/sidekiq_prometheus.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq_prometheus
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Lukas Eklund
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-08 00:00:00.000000000 Z
+date: 2019-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -129,9 +129,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".chglog/CHANGELOG.tpl.md"
+- ".chglog/config.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -165,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: Prometheus Instrumentation for Sidekiq