sidekiq-status 0.1.0

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+.idea/

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Evgeniy Tsvigun
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+# Sidekiq::Status
+
+An extension to [Sidekiq](http://github.com/mperham/sidekiq) message processing to track your jobs. Inspired
+by [resque-status](http://github.com/quirkey/resque-status) and mostly copying its features, using Sidekiq's middleware.
+
+## Installation
+
+gem install sidekiq-status
+
+## Usage
+
+Configure your middleware chains, lookup [Middleware usage](https://github.com/mperham/sidekiq/wiki/Middleware)
+on Sidekiq wiki for more info.
+
+``` ruby
+require 'sidekiq'
+require 'sidekiq-status'
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Status::ClientMiddleware
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Status::ServerMiddleware
+  end
+end
+```
+
+When defining those jobs you want to track later, include one more module. Jobs defined without Sidekiq::Status::Worker
+will be processed as usual.
+
+``` ruby
+class MyJob
+  include Sidekiq::Worker
+  include Sidekiq::Status::Worker
+
+  def perform(*args)
+  # your code goes here
+  end
+end
+```
+
+Query for job status any time later:
+
+``` ruby
+job_id = MyJob.perform_async(*args)
+# "queued", "working", "complete" or "failed" , nil after expiry (30 minutes)
+status = Sidekiq::Status::get(job_id)
+```
+
+### Features coming
+* Progress tracking, messages from running jobs
+* Stopping jobs by id
+* Minimal web UI
+
+## License
+MIT License , see LICENSE for more details.
+© 2012 Evgeniy Tsvigun

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task :test => :spec
+
+task :default => :spec

data/lib/sidekiq-status/client_middleware.rb

@@ -0,0 +1,16 @@
+module Sidekiq::Status
+# Should be in the client middleware chain
+  class ClientMiddleware
+    include Storage
+    # Uses the first argument as id and puts :queued status in the job's Redis hash
+    # @param [Class] worker_class if includes Sidekiq::Status::Worker, the job gets processed with the plugin
+    # @param [Array] msg job arguments, the firs one becomes the id
+    # @param [String] queue the queue's name
+    def call(worker_class, msg, queue)
+      if worker_class.include? Worker
+        store_for_id(msg['args'][0], :status => :queued)
+      end
+      yield
+    end
+  end
+end

data/lib/sidekiq-status/server_middleware.rb

@@ -0,0 +1,46 @@
+module Sidekiq::Status
+# Should be in the server middleware chain
+  class ServerMiddleware
+    # Parameterized initialization, use it when adding middleware to server chain
+    # chain.add Sidekiq::Status::ServerMiddleware, :expiration => 60 * 5
+    # @param [Hash] opts middleware initialization options
+    # @option opts [Fixnum] :expiration ttl for complete jobs
+    def initialize(opts = {:expiration => 30 * 60})
+      @expiration = opts[:expiration]
+    end
+
+    # Takes out the first job argument to use as id
+    # puts :working status into Redis hash
+    # initializes worker instance with id
+    #
+    # Exception handler sets :failed status, re-inserts worker it to job args and re-throws the exception
+    # Worker::Stopped exception type are processed separately - :stopped status is set, no re-throwing
+    #
+    # @param [Worker] worker worker instance, processed here if its class includes Status::Worker
+    # @param [Array] msg job args, first of them used as job id, should have uuid format
+    # @param [String] queue queue name
+    def call(worker, msg, queue)
+      if worker.is_a? Worker
+        worker.id = msg['args'].shift
+        unless worker.id.is_a?(String) && UUID_REGEXP.match(worker.id)
+          raise ArgumentError, "First job argument for a #{worker.class.name} should have uuid format"
+        end
+        worker.store 'status' => 'working'
+        yield
+        worker.store 'status' => 'complete'
+      else
+        yield
+      end
+    rescue Worker::Stopped
+      worker.store 'status' => 'stopped'
+    rescue
+      if worker.is_a? Worker
+        worker.store 'status' => 'failed'
+        msg['args'].unshift worker.id
+      end
+      raise
+    ensure
+      Sidekiq.redis { |conn| conn.expire worker.id, @expiration } if worker.is_a? Worker
+    end
+  end
+end

data/lib/sidekiq-status/storage.rb

@@ -0,0 +1,31 @@
+module Sidekiq::Status::Storage
+  RESERVED_FIELDS=%w(status stop update_time).freeze
+
+  protected
+
+  # Stores multiple values into a job's status hash,
+  # sets last update time
+  # @param [String] id job id
+  # @param [Hash] status_updates updated values
+  # @return [String] Redis operation status code
+  def store_for_id(id, status_updates)
+    Sidekiq.redis do |conn|
+      answers = conn.multi do
+        conn.hmset id, 'update_time', Time.now.to_i, *(status_updates.to_a.flatten)
+        conn.expire id, Sidekiq::Status::DEFAULT_EXPIRY
+        conn.publish "status_updates", id
+      end
+      answers[0]
+    end
+  end
+
+  # Gets a single valued from job status hash
+  # @param [String] id job id
+  # @param [String] Symbol field fetched field name
+  # @return [String] Redis operation status code
+  def read_field_for_id(uuid, field)
+    Sidekiq.redis do |conn|
+      conn.hmget(uuid, field)[0]
+    end
+  end
+end

data/lib/sidekiq-status/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Status
+    VERSION = "0.1.0"
+  end
+end

data/lib/sidekiq-status/worker.rb

@@ -0,0 +1,47 @@
+require "active_support"
+
+module Sidekiq::Status::Worker
+  include Sidekiq::Status::Storage
+  extend ActiveSupport::Concern
+
+  # Adding ID generation to .perform_async
+  module ClassMethods
+    # :nodoc:
+    def self.extended(base)
+      class << base
+        alias_method_chain :perform_async, :uuid
+      end
+    end
+
+    # Add an id to job arguments
+    def perform_async_with_uuid(*args)
+      id = SecureRandom.uuid
+      args.unshift id
+      perform_async_without_uuid(*args)
+      id
+    end
+  end
+
+  class Stopped < StandardError
+  end
+
+  attr_reader :id
+
+  # Worker id initialization
+  # @param [String] id id generated on client-side
+  # @raise [RuntimeError] raised in case of second id initialization attempt
+  # @return [String] id
+  def id=(id)
+    raise RuntimeError("Worker ID is already set : #@id") if @id
+    @id=id
+  end
+
+  # Stores multiple values into a job's status hash,
+  # sets last update time
+  # @param [Hash] status_updates updated values
+  # @return [String] Redis operation status code
+  def store(hash)
+    store_for_id(@id, hash)
+  end
+
+end

data/lib/sidekiq-status.rb

@@ -0,0 +1,20 @@
+require "sidekiq-status/version"
+require 'sidekiq-status/storage'
+require 'sidekiq-status/worker'
+require 'sidekiq-status/client_middleware'
+require 'sidekiq-status/server_middleware'
+
+module Sidekiq
+  module Status
+    extend Storage
+    DEFAULT_EXPIRY = 60 * 30
+    UUID_REGEXP = /[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/
+
+    # Job status by id
+    # @param [String] id job id returned by async_perform
+    # @return [String] job status, possible values: "queued" , "working" , "complete"
+    def self.get(id)
+      read_field_for_id(id, :status)
+    end
+  end
+end

data/sidekiq-status.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/sidekiq-status/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ['Evgeniy Tsvigun']
+  gem.email         = ['utgarda@gmail.com']
+  gem.summary       = 'An extension to the sidekiq message processing to track your jobs'
+  gem.homepage      = 'http://github.com/utgarda/sidekiq-status'
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split($\)
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = 'sidekiq-status'
+  gem.require_paths = ['lib']
+  gem.version       = Sidekiq::Status::VERSION
+
+  gem.add_dependency                  'sidekiq', '~> 2.2'
+  gem.add_development_dependency      'rspec'
+end

data/spec/lib/sidekiq-status/client_middleware_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Sidekiq::Status::ClientMiddleware do
+
+  let!(:redis) { Sidekiq.redis { |conn| conn } }
+  let!(:job_id) { SecureRandom.uuid }
+
+  # Clean Redis before each test
+  before { redis.flushall }
+
+  describe "#call" do
+    it "sets queued status" do
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+      StubJob.perform_async(:arg1 => 'val1').should == job_id
+      redis.hget(job_id, :status).should == 'queued'
+    end
+
+    it "sets status hash ttl" do
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+      StubJob.perform_async(:arg1 => 'val1').should == job_id
+      (1..Sidekiq::Status::DEFAULT_EXPIRY).should cover redis.ttl(job_id)
+    end
+
+  end
+end

data/spec/lib/sidekiq-status/server_middleware_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe Sidekiq::Status::ServerMiddleware do
+
+  let!(:redis) { Sidekiq.redis { |conn| conn } }
+  let!(:job_id) { SecureRandom.uuid }
+
+  # Clean Redis before each test
+  # Seems like flushall has no effect on recently published messages,
+  # so we should wait till they expire
+  before { redis.flushall; sleep 0.1 }
+
+  describe "#call" do
+    it "sets working/complete status" do
+      thread = confirmations_thread 4, "status_updates", "job_messages_#{job_id}"
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+      start_server do
+        ConfirmationJob.perform_async(:arg1 => 'val1').should == job_id
+        thread.value.should == [job_id, job_id,
+                                "while in #perform, status = working",
+                                job_id]
+      end
+      redis.hget(job_id, :status).should == 'complete'
+    end
+
+    it "sets failed status" do
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+      start_server do
+        capture_status_updates(3) {
+          FailingJob.perform_async.should == job_id
+        }.should == [job_id]*3
+      end
+      redis.hget(job_id, :status).should == 'failed'
+    end
+
+    it "sets status hash ttl" do
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+      StubJob.perform_async(:arg1 => 'val1').should == job_id
+      (1..Sidekiq::Status::DEFAULT_EXPIRY).should cover redis.ttl(job_id)
+    end
+
+  end
+end

data/spec/lib/sidekiq-status/worker_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Sidekiq::Status::Worker do
+
+  let!(:job_id) { SecureRandom.uuid }
+
+  describe ".perform_async" do
+    it "generates and returns job id" do
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+      StubJob.perform_async().should == job_id
+    end
+  end
+end

data/spec/lib/sidekiq-status_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe Sidekiq::Status do
+
+  let!(:redis) { Sidekiq.redis { |conn| conn } }
+  let!(:job_id) { SecureRandom.uuid }
+  let!(:job_id_1) { SecureRandom.uuid }
+
+  # Clean Redis before each test
+  # Seems like flushall has no effect on recently published messages,
+  # so we should wait till they expire
+  before { redis.flushall; sleep 0.1 }
+
+  describe ".get" do
+    it "gets job status by id" do
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+
+      start_server do
+        capture_status_updates(2) {
+          LongJob.perform_async(1).should == job_id
+        }.should == [job_id]*2
+        Sidekiq::Status.get(job_id).should == "working"
+      end
+      Sidekiq::Status.get(job_id).should == 'complete'
+    end
+  end
+
+  context "keeps normal Sidekiq functionality" do
+    it "does jobs with and without status processing" do
+      SecureRandom.should_receive(:uuid).twice.and_return(job_id, job_id_1)
+      start_server do
+        capture_status_updates(6) {
+          StubJob.perform_async.should == job_id
+          NoStatusConfirmationJob.perform_async(1)
+          StubJob.perform_async.should == job_id_1
+          NoStatusConfirmationJob.perform_async(2)
+        }.should =~ [job_id, job_id_1] * 3
+      end
+      redis.mget('NoStatusConfirmationJob_1', 'NoStatusConfirmationJob_2').should == %w(done)*2
+      Sidekiq::Status.get(job_id).should == 'complete'
+      Sidekiq::Status.get(job_id_1).should == 'complete'
+    end
+
+    it "retries failed jobs" do
+      SecureRandom.should_receive(:uuid).once.and_return(job_id)
+      start_server do
+        capture_status_updates(5) {
+          RetriedJob.perform_async().should == job_id
+        }.should == [job_id] * 5
+      end
+      Sidekiq::Status.get(job_id).should == 'complete'
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,66 @@
+require "rspec"
+
+require 'sidekiq'
+require 'sidekiq-status'
+
+
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Status::ClientMiddleware
+  end
+end
+
+def confirmations_thread(messages_limit, *channels)
+  parent = Thread.current
+  thread = Thread.new {
+    confirmations = []
+    Sidekiq.redis do |conn|
+      conn.subscribe *channels do |on|
+        on.subscribe do |ch, subscriptions|
+          if subscriptions == channels.size
+            sleep 0.1 while parent.status != "sleep"
+            parent.run
+          end
+        end
+        on.message do |ch, msg|
+          confirmations << msg
+          conn.unsubscribe if confirmations.length >= messages_limit
+        end
+      end
+    end
+    confirmations
+  }
+  Thread.stop
+  yield if block_given?
+  thread
+end
+
+def capture_status_updates(n, &block)
+  confirmations_thread(n, "status_updates", &block).value
+end
+
+def start_server()
+  pid = Process.fork do
+    $stdout.reopen File::NULL, 'w'
+    $stderr.reopen File::NULL, 'w'
+    require 'sidekiq/cli'
+    Sidekiq.options[:queues] << 'default'
+    Sidekiq.configure_server do |config|
+      config.redis = {:url => 'redis://localhost:6379'}
+      config.server_middleware do |chain|
+        chain.add Sidekiq::Status::ServerMiddleware
+      end
+    end
+    Sidekiq::CLI.instance.run
+  end
+
+  yield
+
+  sleep 0.1
+  Process.kill 'TERM', pid
+  Timeout::timeout(10) { Process.wait pid } rescue Timeout::Error
+ensure
+  Process.kill 'KILL', pid rescue "OK" # it's OK if the process is gone already
+end

data/spec/support/test_jobs.rb

@@ -0,0 +1,52 @@
+class StubJob
+  include Sidekiq::Worker
+  include Sidekiq::Status::Worker
+
+  sidekiq_options 'retry' => 'false'
+
+  def perform(*args)
+  end
+end
+
+class LongJob < StubJob
+  def perform(*args)
+    sleep args[0] || 1
+  end
+end
+
+class ConfirmationJob < StubJob
+  def perform(*args)
+    Sidekiq.redis do |conn|
+      conn.publish "job_messages_#{id}", "while in #perform, status = #{conn.hget id, :status}"
+    end
+  end
+end
+
+class NoStatusConfirmationJob
+  include Sidekiq::Worker
+  def perform(id)
+    Sidekiq.redis do |conn|
+      conn.set "NoStatusConfirmationJob_#{id}", "done"
+    end
+
+  end
+end
+
+class FailingJob < StubJob
+  def perform
+    raise StandardError
+  end
+end
+
+class RetriedJob < StubJob
+  sidekiq_options 'retry' => 'true'
+  def perform()
+    Sidekiq.redis do |conn|
+      key = "RetriedJob_#{id}"
+      unless conn.exists key
+        conn.set key, 'tried'
+        raise StandardError
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-status
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Evgeniy Tsvigun
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- utgarda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- LICENSE
+- README.md
+- Rakefile
+- lib/sidekiq-status.rb
+- lib/sidekiq-status/client_middleware.rb
+- lib/sidekiq-status/server_middleware.rb
+- lib/sidekiq-status/storage.rb
+- lib/sidekiq-status/version.rb
+- lib/sidekiq-status/worker.rb
+- sidekiq-status.gemspec
+- spec/lib/sidekiq-status/client_middleware_spec.rb
+- spec/lib/sidekiq-status/server_middleware_spec.rb
+- spec/lib/sidekiq-status/worker_spec.rb
+- spec/lib/sidekiq-status_spec.rb
+- spec/spec_helper.rb
+- spec/support/test_jobs.rb
+homepage: http://github.com/utgarda/sidekiq-status
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: An extension to the sidekiq message processing to track your jobs
+test_files:
+- spec/lib/sidekiq-status/client_middleware_spec.rb
+- spec/lib/sidekiq-status/server_middleware_spec.rb
+- spec/lib/sidekiq-status/worker_spec.rb
+- spec/lib/sidekiq-status_spec.rb
+- spec/spec_helper.rb
+- spec/support/test_jobs.rb
+has_rdoc: