sidekiq-monitor 0.0.2

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Dimko
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+## Sidekiq::Monitor
+
+Jobs and queue stats for your Sidekiq.
+
+### Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sidekiq-monitor'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-monitor
+
+After you install Sidekiq monitor and add it to your Gemfile, you need to run the generator:
+
+    $ rails g sidekiq:monitor:migration
+
+### TODO
+
+1. Test it.
+
+### Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/lib/generators/sidekiq/monitor/migration_generator.rb

@@ -0,0 +1,27 @@
+require 'rails/generators'
+
+module Sidekiq
+  module Monitor
+    class MigrationGenerator < ::Rails::Generators::Base
+      include ::Rails::Generators::Migration
+
+      desc 'Generates migration for Sidekiq::Monitor::Event model'
+
+      def self.source_root
+        File.join(File.dirname(__FILE__), 'templates')
+      end
+
+      def self.next_migration_number(dirname)
+        if ActiveRecord::Base.timestamped_migrations
+          Time.now.utc.strftime('%Y%m%d%H%M%S').to_i.succ.to_s
+        else
+          '%.3d' % current_migration_number(dirname).succ
+        end
+      end
+
+      def create_migration_file
+        migration_template 'migration.rb', 'db/migrate/create_sidekiq_monitor_events.rb'
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/monitor/templates/migration.rb

@@ -0,0 +1,16 @@
+class CreateSidekiqMonitorEvents < ActiveRecord::Migration
+  def change
+    create_table :sidekiq_monitor_events do |t|
+      t.string   :revision,     null: false
+      t.string   :worker_class, null: false
+      t.string   :queue,        null: false
+      t.string   :jid,          null: false
+      t.integer  :retry_count
+      t.text     :args
+      t.text     :exception
+      t.datetime :started_at
+      t.float    :runtime
+      t.timestamps
+    end
+  end
+end

data/lib/sidekiq-monitor.rb

@@ -0,0 +1 @@
+require 'sidekiq/monitor'

data/lib/sidekiq/monitor.rb

@@ -0,0 +1,40 @@
+require 'sidekiq'
+require 'sidekiq/web'
+require 'will_paginate'
+require 'will_paginate/view_helpers/sinatra'
+
+require 'sidekiq/monitor/version'
+require 'sidekiq/monitor/models/event'
+require 'sidekiq/monitor/counters/base'
+require 'sidekiq/monitor/counters/queue'
+require 'sidekiq/monitor/counters/worker'
+require 'sidekiq/monitor/middleware'
+require 'sidekiq/monitor/railtie' if defined?(Rails)
+require 'sidekiq/monitor/web/paginate_renderer'
+require 'sidekiq/monitor/web'
+
+module Sidekiq
+  module Monitor
+    extend self
+
+    include ActiveSupport::Configurable
+
+    config_accessor :events_ttl, :github_repo
+
+    def current_revision
+      @current_revision ||= begin
+        `git rev-parse HEAD`.strip
+      end
+    end
+  end
+end
+
+Sidekiq::Web.helpers  WillPaginate::Sinatra::Helpers
+Sidekiq::Web.register Sidekiq::Monitor::Web
+Sidekiq::Web.tabs['Monitor'] = 'monitor'
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Monitor::Middleware
+  end
+end

data/lib/sidekiq/monitor/counters/base.rb

@@ -0,0 +1,70 @@
+module Sidekiq
+  module Monitor
+    module Counters
+      class Base
+        attr_reader :name, :enqueued, :runtime
+
+        def initialize(name, enqueued = nil, runtime = nil)
+          @name, @enqueued, @runtime = name, enqueued, runtime
+        end
+
+        def self.namespace
+          raise NotImplementedError
+        end
+
+        def self.events_column
+          raise NotImplementedError
+        end
+
+        def self.all(force = false)
+          update if force || update_needed?
+
+          Sidekiq.redis do |conn|
+            names.inject({}) do |memo, name|
+              memo[name] = conn.hgetall(new(name).key)
+              memo
+            end
+          end
+        end
+
+        def self.names
+          Sidekiq.redis do |conn|
+            conn.smembers(namespace).sort
+          end
+        end
+
+        def self.update
+          events = Sidekiq::Monitor::Event
+          Sidekiq.redis { |conn| conn.set("#{namespace}:updated_at", Time.now) }
+          events.connection.execute("SELECT #{events_column}, COUNT(id), AVG(runtime) FROM #{events.table_name} \
+            GROUP BY #{events_column}").values.each do |name, enqueued, runtime|
+
+            new(name, enqueued, runtime).save
+          end
+        end
+
+        def self.update_needed?
+          updated_at.nil? || updated_at.advance(seconds: 10).past?
+        end
+
+        def self.updated_at
+          Sidekiq.redis { |conn| Time.parse conn.get("#{namespace}:updated_at") }
+        rescue
+        end
+
+        def key
+          "#{self.class.namespace}:#{name}"
+        end
+
+        def save
+          Sidekiq.redis do |conn|
+            conn.pipelined do
+              conn.hmset(key, :enqueued, enqueued, :runtime, runtime)
+              conn.sadd(self.class.namespace, name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor/counters/queue.rb

@@ -0,0 +1,15 @@
+module Sidekiq
+  module Monitor
+    module Counters
+      class Queue < Base
+        def self.namespace
+          'monitor:queues'
+        end
+
+        def self.events_column
+          'queue'
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor/counters/worker.rb

@@ -0,0 +1,15 @@
+module Sidekiq
+  module Monitor
+    module Counters
+      class Worker < Base
+        def self.namespace
+          'monitor:workers'
+        end
+
+        def self.events_column
+          'worker_class'
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor/middleware.rb

@@ -0,0 +1,17 @@
+module Sidekiq
+  module Monitor
+    class Middleware
+      def call(worker, message, queue)
+        event = Sidekiq::Monitor::Event.create_or_update_with(worker, message, queue)
+
+        begin
+          yield
+          event.finish
+        rescue Exception => e
+          event.error(e)
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor/models/event.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+module Sidekiq
+  module Monitor
+    class Event < ActiveRecord::Base
+      self.table_name = 'sidekiq_monitor_events'
+
+      serialize :exception, Hash
+      serialize :args,      Array
+
+      scope :recent,     lambda { order(:id).reverse_order }
+      scope :older_than, lambda { |time| where(arel_table[:created_at].lt(time)) }
+
+      before_save :assign_revision
+
+      class << self
+
+        def create_or_update_with(worker, message, queue)
+          transaction do
+            find_or_initialize_by_id(message['monitor_id']).tap do |e|
+              e.started_at   = Time.now
+              e.retry_count  = message['retry_count'].to_i + 1 if message.has_key?('retry_count')
+              e.worker_class = worker.class.name
+              e.args  = message['args']
+              e.jid   = message['jid']
+              e.queue = queue
+              e.save
+
+              message['monitor_id'] ||= e.id
+            end
+          end
+        end
+
+        def find_or_initialize_by_id(id, options = {}, &block)
+          id.nil? ? new(options, &block) : super
+        end
+
+      end
+
+      def finished?
+        !!finished_at
+      end
+
+      def finished_at
+        if runtime?
+          started_at + runtime.seconds
+        end
+      end
+
+      def finish
+        update_attribute(:runtime, Time.now - started_at)
+      end
+
+      def error(exception)
+        update_attribute(:exception, {
+          class:     exception.class.name,
+          backtrace: exception.backtrace,
+          message:   exception.message
+        })
+      end
+
+      protected
+
+      def assign_revision
+        self.revision = Sidekiq::Monitor.current_revision
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor/railtie.rb

@@ -0,0 +1,9 @@
+module Sidekiq
+  module Monitor
+    class Railtie < ::Rails::Railtie
+      rake_tasks do
+        require 'sidekiq/monitor/tasks'
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor/tasks.rb

@@ -0,0 +1,9 @@
+namespace :sidekiq do
+  namespace :monitor do
+    desc 'Sidekiq-monitor events cleanup'
+    task :cleanup => :environment do
+      ttl = Sidekiq::Monitor.events_ttl || 3.weeks
+      Sidekiq::Monitor::Event.older_than(ttl.ago).delete_all
+    end
+  end
+end

data/lib/sidekiq/monitor/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Monitor
+    VERSION = '0.0.2'
+  end
+end

data/lib/sidekiq/monitor/views/monitor/events/index.slim

@@ -0,0 +1,47 @@
+h3
+  ' Processed messages
+  - if @name
+    ' in
+    span.title= @name
+
+- if @events.size > 0
+  table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;"
+    thead
+      th style="width: 25%" Worker, args
+      th style="width: 50%" Runtime or Exception
+      th style="width: 15%" Started at
+      th style="width: 10%" Revision
+    tbody
+      - @events.each do |event|
+        tr
+          td style="overflow: hidden; text-overflow: ellipsis;"
+            = event.worker_class
+
+            div style="color: #999; font-size: 0.9em;"
+              = display_args event.args
+
+          td style="overflow: auto;"
+            - if event.finished?
+              = time_with_sec event.runtime
+
+              - if event.retry_count?
+                = " after #{event.retry_count} retries"
+
+            - elsif event.exception?
+              a.backtrace href="#" onclick="$(this).next().toggle(); return false"
+                = "#{event.exception[:class]}: #{event.exception[:message]}"
+              pre style="display: none; background: none; border: 0; width: 100%; max-height: 30em; font-size: 0.8em; white-space: nowrap;"
+                == event.exception[:backtrace].to_a.join('<br />')
+
+            - else
+              span style="color: #AAA;" performing ...
+          td
+            == relative_time(event.started_at)
+          td
+            == link_to_tree event.revision
+
+  == will_paginate @events, renderer: Sidekiq::Monitor::Web::PaginateRenderer, inner_window: 2
+  br
+
+- else
+  .alert.alert-success No jobs found.

data/lib/sidekiq/monitor/views/monitor/index.slim

@@ -0,0 +1,47 @@
+h3 Monitor
+
+h5 Queues
+
+- if @queues.size > 0
+  table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;"
+    thead
+      th style="width: 55%" Name
+      / th style="width: 15%" Current
+      th style="width: 15%" Enqueued
+      th style="width: 15%" Average runtime
+    tbody
+      - @queues.each do |queue, counters|
+        tr
+          td
+            a href="#{root_path}monitor/queues/#{queue}" = queue
+          / td -
+          td= counters['enqueued']
+          td= time_with_sec counters['runtime']
+
+- else
+  .alert.alert-success No queues found.
+
+br
+
+h5 Workers
+
+- if @workers.size > 0
+  table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;"
+    thead
+      th style="width: 55%" Name
+      / th style="width: 15%" Current
+      th style="width: 15%" Enqueued
+      th style="width: 15%" Average runtime
+    tbody
+      - @workers.each do |worker, counters|
+        tr
+          td
+            a href="#{root_path}monitor/workers/#{worker}" = worker
+          / td -
+          td= counters['enqueued']
+          td= time_with_sec counters['runtime']
+
+- else
+  .alert.alert-success No workers found.
+
+br

data/lib/sidekiq/monitor/web.rb

@@ -0,0 +1,63 @@
+module Sidekiq
+  module Monitor
+    module Web
+      def self.registered(app)
+        app.instance_eval do
+          set :views, [ *views, File.expand_path('views', File.dirname(__FILE__)) ].flatten
+
+          helpers do
+            def time_with_sec(time)
+              "#{time.to_f.round(3)} sec"
+            end
+
+            def link_to_tree(revision)
+              if repo = Sidekiq::Monitor.github_repo
+                %(<a href="https://github.com/#{repo}/tree/#{revision}">#{revision[0..9]}</a>)
+              else
+                revision[0..9]
+              end
+            end
+
+            def find_template(views, name, engine, &block)
+              Array(views).each { |v| super(v, name, engine, &block) }
+            end
+          end
+
+          get '/monitor' do
+            @queues  = Sidekiq::Monitor::Counters::Queue.all
+            @workers = Sidekiq::Monitor::Counters::Worker.all
+
+            slim :'monitor/index'
+          end
+
+          get '/monitor/events' do
+            @events = Sidekiq::Monitor::Event.recent.
+              paginate(page: params[:page])
+
+            slim :'monitor/events/index'
+          end
+
+          get '/monitor/queues/:name' do
+            halt 404 unless params[:name]
+
+            @name   = params[:name]
+            @events = Sidekiq::Monitor::Event.where(queue: @name).recent.
+              paginate(page: params[:page])
+
+            slim :'monitor/events/index'
+          end
+
+          get '/monitor/workers/:name' do
+            halt 404 unless params[:name]
+
+            @name   = params[:name]
+            @events = Sidekiq::Monitor::Event.where(worker_class: @name).recent.
+              paginate(page: params[:page])
+
+            slim :'monitor/events/index'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor/web/paginate_renderer.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+module Sidekiq
+  module Monitor
+    module Web
+      class PaginateRenderer < WillPaginate::Sinatra::LinkRenderer
+        protected
+
+        def html_container(html)
+          tag :div, tag(:ul, html), :class => 'pagination pagination-right'
+        end
+
+        def page_number(page)
+          tag :li, link(page, page, :rel => rel_value(page)), :class => ('active' if page == current_page)
+        end
+
+        def gap
+          tag :li, tag(:span, '&hellip;'), :class => 'gap'
+        end
+
+        def previous_page
+          num = @collection.current_page > 1 && @collection.current_page - 1
+          previous_or_next_page num, '«', 'previous_page'
+        end
+
+        def next_page
+          num = @collection.current_page < total_pages && @collection.current_page + 1
+          previous_or_next_page num, '»', 'next_page'
+        end
+
+        def previous_or_next_page(page, text, classname)
+          tag :li, link(text, page || '#'), :class => [classname[0..3], classname, ('disabled' unless page)].join(' ')
+        end
+      end
+    end
+  end
+end

data/sidekiq-monitor.gemspec

@@ -0,0 +1,27 @@
+# encoding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/monitor/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'sidekiq-monitor'
+  spec.version       = Sidekiq::Monitor::VERSION
+  spec.authors       = ['Dimko']
+  spec.email         = ['deemox@gmail.com']
+  spec.description   = %q{Tracks jobs enqueued, average duration and etc.}
+  spec.summary       = %q{Jobs and queue stats for Sidekiq}
+  spec.homepage      = 'https://github.com/dimko/sidekiq-monitor'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'sidekiq',        '~> 2.6'
+  spec.add_dependency 'rails',          '~> 3.2'
+  spec.add_dependency 'will_paginate',  '~> 3.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-monitor
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Dimko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: will_paginate
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tracks jobs enqueued, average duration and etc.
+email:
+- deemox@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/generators/sidekiq/monitor/migration_generator.rb
+- lib/generators/sidekiq/monitor/templates/migration.rb
+- lib/sidekiq-monitor.rb
+- lib/sidekiq/monitor.rb
+- lib/sidekiq/monitor/counters/base.rb
+- lib/sidekiq/monitor/counters/queue.rb
+- lib/sidekiq/monitor/counters/worker.rb
+- lib/sidekiq/monitor/middleware.rb
+- lib/sidekiq/monitor/models/event.rb
+- lib/sidekiq/monitor/railtie.rb
+- lib/sidekiq/monitor/tasks.rb
+- lib/sidekiq/monitor/version.rb
+- lib/sidekiq/monitor/views/monitor/events/index.slim
+- lib/sidekiq/monitor/views/monitor/index.slim
+- lib/sidekiq/monitor/web.rb
+- lib/sidekiq/monitor/web/paginate_renderer.rb
+- sidekiq-monitor.gemspec
+homepage: https://github.com/dimko/sidekiq-monitor
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -45752073300806058
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -45752073300806058
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Jobs and queue stats for Sidekiq
+test_files: []