sidekiq_status 1.0.0

data/lib/sidekiq_status/worker.rb

@@ -0,0 +1,64 @@
+# -*- encoding : utf-8 -*-
+module SidekiqStatus
+  module Worker
+    def self.included(base)
+      base.class_eval do
+        include Sidekiq::Worker
+
+        include(InstanceMethods)
+
+        base.define_singleton_method(:new) do |*args, &block|
+          super(*args, &block).extend(Prepending)
+        end
+      end
+    end
+
+    module Prepending
+      def perform(jid)
+        @status_container = SidekiqStatus::Container.load(jid)
+
+        begin
+          catch(:killed) do
+            set_status('working')
+            super(*@status_container.args)
+            set_status('complete')
+          end
+        rescue Exception => exc
+          set_status('failed', exc.class.name + ': ' + exc.message + "   \n\n " + exc.backtrace.join("\n    "))
+          raise exc
+        end
+      end
+    end
+
+    module InstanceMethods
+      def status_container
+        kill if @status_container.kill_requested?
+        @status_container
+      end
+      alias_method :sc, :status_container
+
+      def kill
+        # NOTE: status_container below should be accessed by instance var instead of an accessor method
+        # because the second option will lead to infinite recursing
+        @status_container.kill
+        throw(:killed)
+      end
+
+      def set_status(status, message = nil)
+        self.sc.update_attributes('status' => status, 'message' => message)
+      end
+
+      def at(at, message = nil)
+        self.sc.update_attributes('at' => at, 'message' => message)
+      end
+
+      def total=(total)
+        self.sc.update_attributes('total' => total)
+      end
+
+      def payload=(payload)
+        self.sc.update_attributes('payload' => payload)
+      end
+    end
+  end
+end

data/lib/sidekiq_status.rb

@@ -0,0 +1,13 @@
+# -*- encoding : utf-8 -*-
+require 'sidekiq'
+
+require 'securerandom'
+require "sidekiq_status/version"
+require "sidekiq_status/client_middleware"
+require "sidekiq_status/container"
+require "sidekiq_status/worker"
+Sidekiq.client_middleware do |chain|
+  chain.add SidekiqStatus::ClientMiddleware
+end
+
+require 'sidekiq_status/web' if defined?(Sidekiq::Web)

data/sidekiq_status.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/sidekiq_status/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Artem Ignatyev"]
+  gem.email         = ["cryo28@gmail.com"]
+  gem.description   = "Job status tracking extension for Sidekiq"
+  gem.summary       = "A Sidekiq extension to track job execution statuses and return job results back to the client in a convenient manner"
+  gem.homepage      = "https://github.com/cryo28/sidekiq_status"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "sidekiq_status"
+  gem.require_paths = ["lib"]
+  gem.version       = SidekiqStatus::VERSION
+
+  gem.add_runtime_dependency("sidekiq", "~> 2.3.3")
+
+  gem.add_development_dependency("rspec")
+  gem.add_development_dependency("simplecov")
+  gem.add_development_dependency("rake")
+  gem.add_development_dependency("timecop")
+
+  gem.add_development_dependency("yard")
+  gem.add_development_dependency("maruku")
+end

data/spec/container_spec.rb

@@ -0,0 +1,313 @@
+# -*- encoding : utf-8 -*-
+require 'spec_helper'
+
+def test_container(container, hash, jid = nil)
+  hash.reject { |k, v| k == :last_updated_at }.find do |k, v|
+    container.send(k).should == v
+  end
+
+  container.last_updated_at.should == Time.at(hash['last_updated_at']) if hash['last_updated_at']
+  container.jid.should == jid if jid
+end
+
+
+describe SidekiqStatus::Container do
+  let(:jid) { "c2db8b1b460608fb32d76b7a" }
+  let(:status_key) { described_class.status_key(jid) }
+  let(:sample_json_hash) do
+    {
+        'args'            => ['arg1', 'arg2'],
+        'worker'          => 'SidekiqStatus::Worker',
+        'queue'           => '',
+
+        'status'          => "completed",
+        'at'              => 50,
+        'total'           => 200,
+        'message'         => "Some message",
+
+        'payload'         => {},
+        'last_updated_at' => 1344855831
+    }
+  end
+
+  specify ".status_key" do
+    jid = SecureRandom.base64
+    described_class.status_key(jid).should == "sidekiq_status:#{jid}"
+  end
+
+  specify ".kill_key" do
+    described_class.kill_key.should == described_class::KILL_KEY
+  end
+
+
+  context "finders" do
+    let!(:containers) do
+      described_class::STATUS_NAMES.inject({}) do |accum, status_name|
+        container = described_class.create()
+        container.update_attributes(:status => status_name)
+
+        accum[status_name] = container
+        accum
+      end
+    end
+
+    specify ".size" do
+      described_class.size.should == containers.size
+    end
+
+    specify ".status_jids" do
+      expected = containers.values.map(&:jid).map{ |jid| [jid, anything()] }
+      described_class.status_jids.should =~ expected
+      described_class.status_jids(0, 0).size.should == 1
+    end
+
+    specify ".statuses" do
+      described_class.statuses.should be_all{|st| st.is_a?(described_class) }
+      described_class.statuses.size.should == containers.size
+      described_class.statuses(0, 0).size.should == 1
+    end
+
+    describe ".delete" do
+      before do
+        described_class.status_jids.map(&:first).should =~ containers.values.map(&:jid)
+      end
+
+      specify "deletes jobs in specific status" do
+        statuses_to_delete = ['waiting', 'complete']
+        described_class.delete(statuses_to_delete)
+
+        described_class.status_jids.map(&:first).should =~ containers.
+            reject{ |status_name, container|  statuses_to_delete.include?(status_name) }.
+            values.
+            map(&:jid)
+      end
+
+      specify "deletes jobs in all statuses" do
+        described_class.delete()
+
+        described_class.status_jids.should be_empty
+      end
+    end
+  end
+
+  specify ".create" do
+    SecureRandom.should_receive(:hex).with(12).and_return(jid)
+    args = ['arg1', 'arg2', {arg3: 'val3'}]
+
+    container = described_class.create('args' => args)
+    container.should be_a(described_class)
+    container.args.should == args
+
+    # Check default values are set
+    test_container(container, described_class::DEFAULTS.reject{|k, v| k == 'args' }, jid)
+
+    Sidekiq.redis do |conn|
+      conn.exists(status_key).should be_true
+    end
+  end
+
+  describe ".load" do
+    it "raises StatusNotFound exception if status is missing in Redis" do
+      expect { described_class.load(jid) }.to raise_exception(described_class::StatusNotFound, jid)
+    end
+
+    it "loads a container from the redis key" do
+      json = MultiJson.dump(sample_json_hash)
+      Sidekiq.redis { |conn| conn.set(status_key, json) }
+
+      container = described_class.load(jid)
+      test_container(container, sample_json_hash, jid)
+    end
+
+    it "cleans up unprocessed expired kill requests as well" do
+      Sidekiq.redis do |conn|
+        conn.zadd(described_class.kill_key, [
+            [(Time.now - described_class.ttl - 1).to_i, 'a'],
+            [(Time.now - described_class.ttl + 1).to_i, 'b'],
+        ]
+        )
+      end
+
+      json = MultiJson.dump(sample_json_hash)
+      Sidekiq.redis { |conn| conn.set(status_key, json) }
+      described_class.load(jid)
+
+      Sidekiq.redis do |conn|
+        conn.zscore(described_class.kill_key, 'a').should be_nil
+        conn.zscore(described_class.kill_key, 'b').should_not be_nil
+      end
+    end
+  end
+
+  specify "#dump" do
+    hash = sample_json_hash.reject{ |k, v| k == 'last_updated_at' }
+    container = described_class.new(jid, hash)
+    dump = container.send(:dump)
+    dump.should == hash.merge('last_updated_at' => Time.now.to_i)
+  end
+
+  specify "#save saves container to Redis" do
+    hash = sample_json_hash.reject{ |k, v| k == 'last_updated_at' }
+    described_class.new(jid, hash).save
+
+    result = Sidekiq.redis{ |conn| conn.get(status_key) }
+    result = MultiJson.load(result)
+
+    result.should == hash.merge('last_updated_at' => Time.now.to_i)
+
+    Sidekiq.redis{ |conn| conn.ttl(status_key).should >= 0 }
+  end
+
+  specify "#delete" do
+    Sidekiq.redis do |conn|
+      conn.set(status_key, "something")
+      conn.zadd(described_class.kill_key, 0, jid)
+    end
+
+    container = described_class.new(jid)
+    container.delete
+
+    Sidekiq.redis do |conn|
+      conn.exists(status_key).should be_false
+      conn.zscore(described_class.kill_key, jid).should be_nil
+    end
+  end
+
+  specify "#request_kill, #should_kill?, #killable?" do
+    container = described_class.new(jid)
+    container.kill_requested?.should be_false
+    container.should be_killable
+
+    Sidekiq.redis do |conn|
+      conn.zscore(described_class.kill_key, jid).should be_nil
+    end
+
+
+    container.request_kill
+
+    Sidekiq.redis do |conn|
+      conn.zscore(described_class.kill_key, jid).should == Time.now.to_i
+    end
+    container.should be_kill_requested
+    container.should_not be_killable
+  end
+
+  specify "#kill" do
+    container = described_class.new(jid)
+    container.request_kill
+    Sidekiq.redis do |conn|
+      conn.zscore(described_class.kill_key, jid).should == Time.now.to_i
+    end
+    container.status.should_not == 'killed'
+
+
+    container.kill
+
+    Sidekiq.redis do |conn|
+      conn.zscore(described_class.kill_key, jid).should be_nil
+    end
+
+    container.status.should == 'killed'
+    described_class.load(jid).status.should == 'killed'
+  end
+
+  specify "#pct_complete" do
+    container = described_class.new(jid)
+    container.at = 1
+    container.total = 100
+    container.pct_complete.should == 1
+
+    container.at = 5
+    container.total = 200
+    container.pct_complete.should == 3 # 2.5.round(0) => 3
+  end
+
+  context "setters" do
+    let(:container) { described_class.new(jid) }
+
+    describe "#at=" do
+      it "sets numeric value" do
+        container.total = 100
+        container.at = 3
+        container.at.should == 3
+        container.total.should == 100
+      end
+
+      it "raises ArgumentError otherwise" do
+        expect{ container.at = "Wrong" }.to raise_exception(ArgumentError)
+      end
+
+      it "adjusts total if its less than new at" do
+        container.total = 200
+        container.at = 250
+        container.total.should == 250
+      end
+    end
+
+    describe "#total=" do
+      it "sets numeric value" do
+        container.total = 50
+        container.total.should == 50
+      end
+
+      it "raises ArgumentError otherwise" do
+        expect{ container.total = "Wrong" }.to raise_exception(ArgumentError)
+      end
+    end
+
+    describe "#status=" do
+      described_class::STATUS_NAMES.each do |status|
+        it "sets status #{status.inspect}" do
+          container.status = status
+          container.status.should == status
+        end
+      end
+
+      it "raises ArgumentError otherwise" do
+        expect{ container.status = 'Wrong' }.to raise_exception(ArgumentError)
+      end
+    end
+
+    specify "#message=" do
+      container.message = 'abcd'
+      container.message.should == 'abcd'
+
+      container.message = nil
+      container.message.should be_nil
+
+      message = double('Message', :to_s => 'to_s')
+      container.message = message
+      container.message.should == 'to_s'
+    end
+
+    specify "#payload=" do
+      container.should respond_to(:payload=)
+    end
+
+    specify "update_attributes" do
+      container.update_attributes(:at => 1, 'total' => 3, :message => 'msg', 'status' => 'working')
+      reloaded_container = described_class.load(container.jid)
+
+      reloaded_container.at.should == 1
+      reloaded_container.total.should == 3
+      reloaded_container.message.should == 'msg'
+      reloaded_container.status.should == 'working'
+
+      expect{ container.update_attributes(:at => 'Invalid') }.to raise_exception(ArgumentError)
+    end
+  end
+
+  context "predicates" do
+    described_class::STATUS_NAMES.each do |status_name1|
+      context "status is #{status_name1}" do
+        subject{ described_class.create().tap{|c| c.status = status_name1} }
+
+        its("#{status_name1}?") { should be_true }
+
+        (described_class::STATUS_NAMES - [status_name1]).each do |status_name2|
+          its("#{status_name2}?") { should be_false }
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,33 @@
+# -*- encoding : utf-8 -*-
+require 'bundler'
+Bundler.setup
+
+ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
+
+require 'simplecov'
+SimpleCov.start
+
+#require 'sidekiq'
+
+require 'sidekiq_status'
+require 'sidekiq/util'
+
+require 'timecop'
+
+Sidekiq.logger.level = Logger::ERROR
+
+require 'sidekiq/redis_connection'
+REDIS = Sidekiq::RedisConnection.create(:url => "redis://localhost/15", :namespace => 'test', :size => 1)
+
+RSpec.configure do |c|
+  c.before do
+    Sidekiq.redis = REDIS
+    Sidekiq.redis{ |conn| conn.flushdb }
+  end
+
+  c.around do |example|
+    Timecop.freeze(Time.utc(2012)) do
+      example.call
+    end
+  end
+end

data/spec/worker_spec.rb

@@ -0,0 +1,159 @@
+require 'spec_helper'
+
+describe Sidekiq::Worker do
+  class SomeWorker
+    include SidekiqStatus::Worker
+
+    def perform(*args)
+      some_method(*args)
+    end
+
+    def some_method(*args); end
+  end
+
+  let(:args) { ['arg1', 'arg2', {'arg3' => 'val3'}]}
+
+  describe ".perform_async" do
+    it "invokes middleware which creates sidekiq_status container with the same jid" do
+      jid = SomeWorker.perform_async(*args)
+      jid.should be_a(String)
+
+      container = SidekiqStatus::Container.load(jid)
+      container.args.should == args
+    end
+  end
+
+  describe "#perform (Worker context)" do
+    let(:worker) { SomeWorker.new }
+
+    it "receives jid as parameters, loads container and runs original perform with enqueued args" do
+      worker.should_receive(:some_method).with(*args)
+      jid = SomeWorker.perform_async(*args)
+      worker.perform(jid)
+    end
+
+    it "changes status to working" do
+      has_been_run = false
+      worker.extend(Module.new do
+        define_method(:some_method) do |*args|
+          status_container.status.should == 'working'
+          has_been_run = true
+        end
+      end)
+
+      jid = SomeWorker.perform_async(*args)
+      worker.perform(jid)
+
+      has_been_run.should be_true
+      worker.status_container.reload.status.should == 'complete'
+    end
+
+    it "intercepts failures and set status to 'failed' then re-raises the exception" do
+      exc = RuntimeError.new('Some error')
+      worker.stub(:some_method).and_raise(exc)
+
+      jid = SomeWorker.perform_async(*args)
+
+      expect{ worker.perform(jid) }.to raise_exception(exc)
+
+      container = SidekiqStatus::Container.load(jid)
+      container.status.should == 'failed'
+    end
+
+    it "sets status to 'complete' if finishes without errors" do
+      jid = SomeWorker.perform_async(*args)
+      worker.perform(jid)
+
+      container = SidekiqStatus::Container.load(jid)
+      container.status.should == 'complete'
+    end
+
+    it "handles kill requests if kill requested before job execution" do
+      jid = SomeWorker.perform_async(*args)
+      container = SidekiqStatus::Container.load(jid)
+      container.request_kill
+
+      worker.perform(jid)
+
+      container.reload
+      container.status.should == 'killed'
+    end
+
+    it "handles kill requests if kill requested amid job execution" do
+      jid = SomeWorker.perform_async(*args)
+      container = SidekiqStatus::Container.load(jid)
+      container.status.should == 'waiting'
+
+      i = 0
+      i_mut = Mutex.new
+
+      worker.extend(Module.new do
+        define_method(:some_method) do |*args|
+          loop do
+            i_mut.synchronize do
+              i += 1
+            end
+
+            status_container.at = i
+          end
+        end
+      end)
+
+      worker_thread = Thread.new{ worker.perform(jid) }
+
+
+      killer_thread = Thread.new do
+        sleep(0.01) while i < 100
+        container.reload.status.should == 'working'
+        container.request_kill
+      end
+
+      worker_thread.join(2)
+      killer_thread.join(1)
+
+      container.reload
+      container.status.should == 'killed'
+      container.at.should >= 100
+    end
+
+    it "allows to set at, total and customer payload from the worker" do
+      jid = SomeWorker.perform_async(*args)
+      container = SidekiqStatus::Container.load(jid)
+
+      ready = false
+      lets_stop = false
+
+      worker.extend(Module.new do
+        define_method(:some_method) do |*args|
+          self.total=(200)
+          self.at(50, "25% done")
+          self.payload = 'some payload'
+          ready = true
+          sleep(0.01) unless lets_stop
+        end
+      end)
+
+      worker_thread = Thread.new{ worker.perform(jid) }
+      checker_thread = Thread.new do
+        sleep(0.01) unless ready
+
+        container.reload
+        container.status.should  == 'working'
+        container.at.should      == 50
+        container.total.should   == 200
+        container.message.should == '25% done'
+        container.payload        == 'some payload'
+
+        lets_stop = true
+      end
+
+      worker_thread.join(10)
+      checker_thread.join(10)
+
+      container.reload
+      container.status.should  == 'complete'
+      container.payload.should == 'some payload'
+      container.message.should be_nil
+    end
+  end
+end

data/web/views/status.slim

@@ -0,0 +1,44 @@
+h1.wi
+  | Job #{@status.jid} is #{@status.status} (#{@status.pct_complete}% done)
+p.intro
+
+div id="status_#{@status.jid}" rel="#{@status.status}"
+  h2 Details
+  div
+    table class="table table-striped table-bordered"
+      thead
+        tr
+          th Attribute
+          th Value
+      tbody
+        tr
+          td jid
+          td= @status.jid
+        tr
+          td status
+          td= @status.status
+        tr
+          td last updated at
+          td= @status.last_updated_at
+        tr
+          td at
+          td= @status.at
+        tr
+          td total
+          td= @status.total
+        tr
+          td message
+          td= @status.message
+        tr
+          td payload
+          td
+            code class="prettyprint language-javascript"
+              = @status.payload.to_json
+        tr
+          td job args
+          td
+            code class="prettyprint language-javascript"
+              = @status.args.to_json
+
+    a href=(to(:statuses)) Back
+

data/web/views/statuses.slim

@@ -0,0 +1,37 @@
+h1.wi Recent job statuses
+
+div.delete_jobs
+  | Delete jobs in 
+  a href="#{to(:statuses)}/delete/complete" onclick="return confirm('Are you sure? Delete is irreversible')" = "complete"
+  |, 
+  a href="#{to(:statuses)}/delete/finished" onclick="return confirm('Are you sure? Delete is irreversible')" title="#{SidekiqStatus::Container::FINISHED_STATUS_NAMES.join(', ')}" = "finished"
+  |, 
+  a href="#{to(:statuses)}/delete/all"      onclick="return confirm('Are you sure? Delete is irreversible')" = "all"
+  | statuses
+
+table class="table table-striped table-bordered"
+  tr
+    th jid
+    th Status
+    th Last Updated ↆ
+    th Progress
+    th Message
+  - @statuses.each do |container|
+    tr
+      td
+        a href="#{to(:statuses)}/#{container.jid}" = container.jid
+      td= container.status
+      td= container.last_updated_at
+      td
+        - if container.killable?
+          a.kill href="#{to(:statuses)}/#{container.jid}/kill" onclick="return confirm('Are you sure?');" Kill
+        - elsif container.kill_requested?
+          |Kill requested
+      td= container.message
+  - if @statuses.empty?
+    tr
+      td colspan="5"
+
+== slim :_paging, :locals => { :url => "#{root_path}statuses" }
+
+