sidekiq-monitor-stats 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 00425866b37d303ebbc7a42819cc4826f1b0080e
+  data.tar.gz: 1a1b4d7432c44b4d34830e9675a5c611167fefdc
+SHA512:
+  metadata.gz: abd30bb60dac07941aff830d9ce0988ac7804b554e3ecd2784e178f3de7baa890eae86a3928219dbc46bdc8fe8e37223532eae649bc37524297a3fb7e4b596a7
+  data.tar.gz: b87592c27829b55f208fdd0651e6ba46ed368aa2bc9f3331d35625afe300ecb89b01e05ae95759600630469060d93944ebd66e39b65fa5f22b184e5f0d05ca85

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+cache: bundler
+services:
+  - redis-server
+rvm:
+  - 2.0.0
+  - 2.1
+  - 2.2

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-stats.gemspec
+gemspec
+
+gem 'byebug'

data/LICENSE.txt

@@ -0,0 +1,23 @@
+by Albert Llop for Harvest
+Copyright (c) 2015 by Harvest
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,87 @@
+# Sidekiq Monitor Stats
+
+[![Build Status](https://travis-ci.org/harvesthq/sidekiq-monitor-stats.svg)](https://travis-ci.org/harvesthq/sidekiq-monitor-stats)
+
+Add an endpoint to your running application that is running Sidekiq that
+returns useful data in JSON format.
+
+The intention is to provide useful data for something like a sensu check
+to monitor if Sidekiq is working correctly.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-monitor-stats'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-monitor-stats
+
+## Usage
+
+This will automatically add another endpoint wherever you have mounted
+your sidekiq web. Just visit `<mounted-path>/monitor-stats` to see some stats.
+
+This is an example of the output you'll get:
+
+```json
+{
+  "queues":{
+    "default":{
+      "backlog":15,
+      "latency":2
+    },
+    "high":{
+      "backlog":0,
+      "latency":0
+    },
+    "low":{
+      "backlog":533,
+      "latency":54
+    }
+  },
+  "processes":[
+    {
+      "hostname":"kip1.example.com",
+      "pid":23324,
+      "queues":[
+        "high"
+      ],
+      "concurrency":5,
+      "busy":2
+    },
+    {
+      "hostname":"kip2.example.com",
+      "pid":23390,
+      "queues":[
+        "default",
+        "low"
+      ],
+      "concurrency":5,
+      "busy":5
+    }
+  ]
+}
+```
+
+It will return a `queues` hash and a `processes` array.
+
+The `queues` hash contains the existing queues with their names, the backlog
+and the latency. The backlog is the amount of jobs waiting to be processed,
+and the latency is the maximum time (in seconds) a job has been waiting in
+the queue.
+
+The `processes` array returns useful information about each sidekiq process.
+The `concurrency`, in Sidekiq terms, is the maximum jobs a process can go
+through in parallel. The `busy` is how many of those are in use.
+
+If you plan on using this for your monitoring, the `latency` is a much
+better metric than the `backlog`, and you can use the total `concurrency` and
+total `busy` to know how much of your available workforce is in use.

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new("test") do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+
+desc "Run the tests"
+task default: :test

data/lib/sidekiq/monitor/stats.rb

@@ -0,0 +1,35 @@
+require 'sidekiq/api'
+require 'sidekiq/web'
+require 'sidekiq/monitor/web'
+
+module Sidekiq
+  module Monitor
+    class Stats
+      def queue_metrics
+        Sidekiq::Queue.all.each_with_object({}) do |queue, hash|
+          hash[queue.name] = {
+            backlog: queue.size,
+            latency: queue.latency.to_i
+          }
+        end
+      end
+
+      def process_metrics
+        Sidekiq::ProcessSet.new.map do |process|
+          {
+            hostname:    process['hostname'],
+            pid:         process['pid'],
+            tag:         process['tag'],
+            started_at:  Time.at(process['started_at']),
+            queues:      process['queues'],
+            labels:      process['labels'],
+            concurrency: process['concurrency'],
+            busy:        process['busy']
+          }
+        end
+      end
+    end
+  end
+end
+
+Sidekiq::Web.register(Sidekiq::Monitor::Web)

data/lib/sidekiq/monitor/web.rb

@@ -0,0 +1,19 @@
+require 'sidekiq/monitor/stats'
+
+module Sidekiq
+  module Monitor
+    module Web
+      def self.registered(app)
+        app.get "/monitor-stats" do
+          monitor_stats = Monitor::Stats.new
+
+          content_type :json
+          Sidekiq.dump_json(
+            queues:    monitor_stats.queue_metrics,
+            processes: monitor_stats.process_metrics
+          )
+        end
+      end
+    end
+  end
+end

data/sidekiq-monitor-stats.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-monitor-stats"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Albert Llop"]
+  spec.email         = ["albert@getharvest.com"]
+  spec.summary       = %q{Add an endpoint to your running application that is running Sidekiq that returns useful data in JSON format.}
+  spec.homepage      = "https://github.com/harvesthq/sidekiq-monitor-stats"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", ">= 5.0.0"
+  spec.add_development_dependency "sinatra"
+  spec.add_development_dependency "rack-test"
+  spec.add_development_dependency "mocha"
+  spec.add_dependency "sidekiq"
+end

data/test/sidekiq-monitor-stats_test.rb

@@ -0,0 +1,53 @@
+require 'test_helper'
+
+class Sidekiq::Monitor::StatsTest < Minitest::Test
+  def setup
+    Sidekiq.redis { |conn| conn.flushdb }
+  end
+
+  def test_returns_empty_data
+    assert_equal({}, stats.queue_metrics)
+    assert_equal([], stats.process_metrics)
+  end
+
+  def test_with_some_data
+    Sidekiq.redis do |conn|
+      @started_at = Time.now
+
+      conn.sadd("queues", "default")
+      conn.lpush("queue:default", Sidekiq.dump_json('enqueued_at' => @started_at.to_f))
+
+      conn.sadd('processes', 'foo:1234')
+
+      process_stats = {
+        hostname:   'foo',
+        pid:        1234,
+        tag:        'default',
+        started_at: Time.now.to_f,
+        queues:     ['default'],
+        labels:     ['reliable'],
+        concurrency: 25
+      }
+
+      conn.hmset('foo:1234', 'info', Sidekiq.dump_json(process_stats), 'at', @started_at.to_f, 'busy', 4)
+    end
+
+    assert_equal({ backlog: 1, latency: 0 }, stats.queue_metrics['default'])
+
+    process = stats.process_metrics.first
+    assert_equal "foo",        process[:hostname]
+    assert_equal 1234,         process[:pid]
+    assert_equal 'default',    process[:tag]
+    assert_equal ['default'],  process[:queues]
+    assert_equal ['reliable'], process[:labels]
+    assert_equal 25,           process[:concurrency]
+    assert_equal 4,            process[:busy]
+    assert_in_delta @started_at, process[:started_at], 0.01
+  end
+
+  private
+
+  def stats
+    Sidekiq::Monitor::Stats.new
+  end
+end

data/test/sidekiq-monitor-web_test.rb

@@ -0,0 +1,54 @@
+require 'test_helper'
+require 'rack/test'
+
+class Sidekiq::Monitor::WebTest < Minitest::Test
+  include Rack::Test::Methods
+
+  def setup
+    Sidekiq.redis { |conn| conn.flushdb }
+  end
+
+  def test_monitor_stats_empty
+    get '/monitor-stats'
+
+    assert_equal 200, last_response.status
+
+    body = Sidekiq.load_json(last_response.body)
+    assert_equal({}, body['queues'])
+    assert_equal([], body['processes'])
+  end
+
+  def test_monitor_stats_with_some_data
+    Sidekiq::Monitor::Stats.any_instance.expects(
+      queue_metrics: { 'default' => { 'backlog' => 5, 'latency' => 10 } },
+      process_metrics: [
+        'hostname'   => 'kip.local',
+        'pid'        => '12345',
+        'queues'     => ['default', 'high'],
+        'concurrency'=> 25,
+        'busy'       => 4
+      ]
+    )
+    get '/monitor-stats'
+
+    assert_equal 200, last_response.status
+
+    body = Sidekiq.load_json(last_response.body)
+    assert_equal 5,  body['queues']['default']['backlog']
+    assert_equal 10, body['queues']['default']['latency']
+
+    assert_equal 1, body['processes'].size
+    process = body['processes'].first
+    assert_equal 'kip.local',        process['hostname']
+    assert_equal '12345',            process['pid']
+    assert_equal ['default','high'], process['queues']
+    assert_equal 25,                 process['concurrency']
+    assert_equal 4,                  process['busy']
+  end
+
+  private
+
+  def app
+    Sidekiq::Web
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require 'minitest/autorun'
+require 'mocha/setup'
+require 'sidekiq/monitor/stats'
+require 'byebug'

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-monitor-stats
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Albert Llop
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- albert@getharvest.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq/monitor/stats.rb
+- lib/sidekiq/monitor/web.rb
+- sidekiq-monitor-stats.gemspec
+- test/sidekiq-monitor-stats_test.rb
+- test/sidekiq-monitor-web_test.rb
+- test/test_helper.rb
+homepage: https://github.com/harvesthq/sidekiq-monitor-stats
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Add an endpoint to your running application that is running Sidekiq that
+  returns useful data in JSON format.
+test_files:
+- test/sidekiq-monitor-stats_test.rb
+- test/sidekiq-monitor-web_test.rb
+- test/test_helper.rb