RubyGems - kulesa-sidekiq - Versions diffs - 1.2.2 - Mend

kulesa-sidekiq 1.2.2

Files changed (135) hide show

data/.gitignore +6 -0
data/.rvmrc +4 -0
data/COMM-LICENSE +83 -0
data/Changes.md +207 -0
data/Gemfile +12 -0
data/LICENSE +22 -0
data/README.md +61 -0
data/Rakefile +9 -0
data/bin/client +7 -0
data/bin/sidekiq +14 -0
data/bin/sidekiqctl +74 -0
data/config.ru +18 -0
data/examples/chef/cookbooks/sidekiq/README.rdoc +11 -0
data/examples/chef/cookbooks/sidekiq/recipes/default.rb +55 -0
data/examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb +8 -0
data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb +219 -0
data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb +22 -0
data/examples/clockwork.rb +44 -0
data/examples/config.yml +10 -0
data/examples/monitrc.conf +6 -0
data/examples/por.rb +27 -0
data/examples/scheduling.rb +37 -0
data/examples/sinkiq.rb +59 -0
data/examples/web-ui.png +0 -0
data/lib/sidekiq.rb +98 -0
data/lib/sidekiq/capistrano.rb +35 -0
data/lib/sidekiq/cli.rb +214 -0
data/lib/sidekiq/client.rb +72 -0
data/lib/sidekiq/extensions/action_mailer.rb +26 -0
data/lib/sidekiq/extensions/active_record.rb +27 -0
data/lib/sidekiq/extensions/generic_proxy.rb +21 -0
data/lib/sidekiq/fetch.rb +76 -0
data/lib/sidekiq/logging.rb +46 -0
data/lib/sidekiq/manager.rb +163 -0
data/lib/sidekiq/middleware/chain.rb +96 -0
data/lib/sidekiq/middleware/client/unique_jobs.rb +36 -0
data/lib/sidekiq/middleware/server/active_record.rb +13 -0
data/lib/sidekiq/middleware/server/exception_handler.rb +38 -0
data/lib/sidekiq/middleware/server/failure_jobs.rb +25 -0
data/lib/sidekiq/middleware/server/logging.rb +31 -0
data/lib/sidekiq/middleware/server/retry_jobs.rb +69 -0
data/lib/sidekiq/middleware/server/timeout.rb +21 -0
data/lib/sidekiq/middleware/server/unique_jobs.rb +17 -0
data/lib/sidekiq/processor.rb +92 -0
data/lib/sidekiq/rails.rb +21 -0
data/lib/sidekiq/redis_connection.rb +27 -0
data/lib/sidekiq/retry.rb +59 -0
data/lib/sidekiq/testing.rb +44 -0
data/lib/sidekiq/testing/inline.rb +37 -0
data/lib/sidekiq/util.rb +40 -0
data/lib/sidekiq/version.rb +3 -0
data/lib/sidekiq/web.rb +185 -0
data/lib/sidekiq/worker.rb +62 -0
data/lib/sidekiq/yaml_patch.rb +21 -0
data/myapp/.gitignore +15 -0
data/myapp/Capfile +5 -0
data/myapp/Gemfile +19 -0
data/myapp/Rakefile +7 -0
data/myapp/app/controllers/application_controller.rb +3 -0
data/myapp/app/controllers/work_controller.rb +38 -0
data/myapp/app/helpers/application_helper.rb +2 -0
data/myapp/app/mailers/.gitkeep +0 -0
data/myapp/app/mailers/user_mailer.rb +9 -0
data/myapp/app/models/.gitkeep +0 -0
data/myapp/app/models/post.rb +5 -0
data/myapp/app/views/layouts/application.html.erb +14 -0
data/myapp/app/views/user_mailer/greetings.html.erb +3 -0
data/myapp/app/views/work/index.html.erb +1 -0
data/myapp/app/workers/hard_worker.rb +10 -0
data/myapp/config.ru +4 -0
data/myapp/config/application.rb +59 -0
data/myapp/config/boot.rb +6 -0
data/myapp/config/database.yml +25 -0
data/myapp/config/deploy.rb +15 -0
data/myapp/config/environment.rb +5 -0
data/myapp/config/environments/development.rb +38 -0
data/myapp/config/environments/production.rb +67 -0
data/myapp/config/environments/test.rb +37 -0
data/myapp/config/initializers/backtrace_silencers.rb +7 -0
data/myapp/config/initializers/inflections.rb +15 -0
data/myapp/config/initializers/mime_types.rb +5 -0
data/myapp/config/initializers/secret_token.rb +7 -0
data/myapp/config/initializers/session_store.rb +8 -0
data/myapp/config/initializers/sidekiq.rb +6 -0
data/myapp/config/initializers/wrap_parameters.rb +14 -0
data/myapp/config/locales/en.yml +5 -0
data/myapp/config/routes.rb +10 -0
data/myapp/db/migrate/20120123214055_create_posts.rb +10 -0
data/myapp/db/seeds.rb +7 -0
data/myapp/lib/assets/.gitkeep +0 -0
data/myapp/lib/tasks/.gitkeep +0 -0
data/myapp/log/.gitkeep +0 -0
data/myapp/script/rails +6 -0
data/sidekiq.gemspec +27 -0
data/test/config.yml +9 -0
data/test/fake_env.rb +0 -0
data/test/helper.rb +16 -0
data/test/test_cli.rb +168 -0
data/test/test_client.rb +119 -0
data/test/test_extensions.rb +69 -0
data/test/test_manager.rb +51 -0
data/test/test_middleware.rb +92 -0
data/test/test_processor.rb +32 -0
data/test/test_retry.rb +125 -0
data/test/test_stats.rb +68 -0
data/test/test_testing.rb +97 -0
data/test/test_testing_inline.rb +75 -0
data/test/test_web.rb +122 -0
data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
data/web/assets/javascripts/application.js +20 -0
data/web/assets/javascripts/vendor/bootstrap.js +12 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-alert.js +91 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-button.js +98 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-carousel.js +154 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-collapse.js +136 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-dropdown.js +92 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-modal.js +210 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-popover.js +95 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-scrollspy.js +125 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-tab.js +130 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-tooltip.js +270 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-transition.js +51 -0
data/web/assets/javascripts/vendor/bootstrap/bootstrap-typeahead.js +271 -0
data/web/assets/javascripts/vendor/jquery.js +9266 -0
data/web/assets/javascripts/vendor/jquery.timeago.js +148 -0
data/web/assets/stylesheets/application.css +27 -0
data/web/assets/stylesheets/vendor/bootstrap-responsive.css +567 -0
data/web/assets/stylesheets/vendor/bootstrap.css +3365 -0
data/web/views/index.slim +48 -0
data/web/views/layout.slim +26 -0
data/web/views/queue.slim +11 -0
data/web/views/retries.slim +29 -0
data/web/views/retry.slim +52 -0
metadata +371 -0

@@ -0,0 +1,59 @@
+# Make sure you have Sinatra installed, then start sidekiq with
+# ./bin/sidekiq -r ./examples/sinkiq.rb
+# Simply run Sinatra with
+# ruby examples/sinkiq.rb
+# and then browse to http://localhost:4567
+#
+require 'sinatra'
+require 'sidekiq'
+require 'redis'
+$redis = Redis.connect
+class SinatraWorker
+  include Sidekiq::Worker
+  def perform(msg="lulz you forgot a msg!")
+    $redis.lpush("sinkiq-example-messages", msg)
+  end
+end
+get '/' do
+  @failed = $redis.get('stat:failed')
+  @processed = $redis.get('stat:processed')
+  @messages = $redis.lrange('sinkiq-example-messages', 0, -1)
+  erb :index
+end
+post '/msg' do
+  SinatraWorker.perform_async params[:msg]
+  redirect to('/')
+end
+__END__
+@@ layout
+<html>
+  <head>
+    <title>Sinatra + Sidekiq</title>
+    <body>
+      <%= yield %>
+    </body>
+</html>
+@@ index
+  <h1>Sinatra + Sidekiq Example</h1>
+  <h2>Failed: <%= @failed %></h2>
+  <h2>Processed: <%= @processed %></h2>
+  <form method="post" action="/msg">
+    <input type="text" name="msg">
+    <input type="submit" value="Add Message">
+  </form>
+  <a href="/">Refresh page</a>
+  <h3>Messages</h3>
+  <% @messages.each do |msg| %>
+    <p><%= msg %></p>
+  <% end %>

data/examples/web-ui.png ADDED

Binary file

data/lib/sidekiq.rb ADDED

@@ -0,0 +1,98 @@
+require 'sidekiq/version'
+require 'sidekiq/logging'
+require 'sidekiq/client'
+require 'sidekiq/worker'
+require 'sidekiq/redis_connection'
+require 'sidekiq/util'
+require 'sidekiq/extensions/action_mailer'
+require 'sidekiq/extensions/active_record'
+require 'sidekiq/rails' if defined?(::Rails)
+module Sidekiq
+  DEFAULTS = {
+    :queues => [],
+    :concurrency => 25,
+    :require => '.',
+    :environment => nil,
+    :timeout => 8,
+    :enable_rails_extensions => true,
+  }
+  def self.options
+    @options ||= DEFAULTS.dup
+  end
+  def self.options=(opts)
+    @options = opts
+  end
+  ##
+  # Configuration for Sidekiq server, use like:
+  #
+  #   Sidekiq.configure_server do |config|
+  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/mydb' }
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #     end
+  #   end
+  def self.configure_server
+    yield self if server?
+  end
+  ##
+  # Configuration for Sidekiq client, use like:
+  #
+  #   Sidekiq.configure_client do |config|
+  #     config.redis = { :namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/mydb' }
+  #   end
+  def self.configure_client
+    yield self unless server?
+  end
+  def self.server?
+    defined?(Sidekiq::CLI)
+  end
+  def self.redis(&block)
+    @redis ||= Sidekiq::RedisConnection.create
+    raise ArgumentError, "requires a block" if !block
+    @redis.with(&block)
+  end
+  def self.redis=(hash)
+    if hash.is_a?(Hash)
+      @redis = RedisConnection.create(hash)
+    elsif hash.is_a?(ConnectionPool)
+      @redis = hash
+    else
+      raise ArgumentError, "redis= requires a Hash or ConnectionPool"
+    end
+  end
+  def self.client_middleware
+    @client_chain ||= Client.default_middleware
+    yield @client_chain if block_given?
+    @client_chain
+  end
+  def self.server_middleware
+    @server_chain ||= Processor.default_middleware
+    yield @server_chain if block_given?
+    @server_chain
+  end
+  def self.load_json(string)
+    MultiJson.decode(string)
+  end
+  def self.dump_json(object)
+    MultiJson.encode(object)
+  end
+  def self.logger
+    Sidekiq::Logging.logger
+  end
+end

data/lib/sidekiq/capistrano.rb ADDED

@@ -0,0 +1,35 @@
+Capistrano::Configuration.instance.load do
+  before "deploy",        "sidekiq:quiet"
+  after "deploy:stop",    "sidekiq:stop"
+  after "deploy:start",   "sidekiq:start"
+  after "deploy:restart", "sidekiq:restart"
+  _cset(:sidekiq_timeout) { 10 }
+  _cset(:sidekiq_role) { :app }
+  namespace :sidekiq do
+    desc "Quiet sidekiq (stop accepting new work)"
+    task :quiet, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl quiet #{current_path}/tmp/pids/sidekiq.pid ; fi"
+    end
+    desc "Stop sidekiq"
+    task :stop, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl stop #{current_path}/tmp/pids/sidekiq.pid #{fetch :sidekiq_timeout} ; fi"
+    end
+    desc "Start sidekiq"
+    task :start, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path} ; nohup #{fetch(:bundle_cmd, "bundle")} exec sidekiq -e #{rails_env} -C #{current_path}/config/sidekiq.yml -P #{current_path}/tmp/pids/sidekiq.pid >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
+    end
+    desc "Restart sidekiq"
+    task :restart, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      stop
+      start
+    end
+  end
+end

data/lib/sidekiq/cli.rb ADDED

@@ -0,0 +1,214 @@
+trap 'INT' do
+  # Handle Ctrl-C in JRuby like MRI
+  # http://jira.codehaus.org/browse/JRUBY-4637
+  Sidekiq::CLI.instance.interrupt
+end
+trap 'TERM' do
+  # Heroku sends TERM and then waits 10 seconds for process to exit.
+  Sidekiq::CLI.instance.interrupt
+end
+trap 'USR1' do
+  Sidekiq.logger.info "Received USR1, no longer accepting new work"
+  mgr = Sidekiq::CLI.instance.manager
+  mgr.stop! if mgr
+end
+trap 'TTIN' do
+  Thread.list.each do |thread|
+    puts "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+    puts thread.backtrace.join("\n")
+  end
+end
+require 'yaml'
+require 'singleton'
+require 'optparse'
+require 'celluloid'
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/manager'
+require 'sidekiq/retry'
+module Sidekiq
+  class CLI
+    include Util
+    include Singleton
+    # Used for CLI testing
+    attr_accessor :code
+    attr_accessor :manager
+    def initialize
+      @code = nil
+      @interrupt_mutex = Mutex.new
+      @interrupted = false
+    end
+    def parse(args=ARGV)
+      @code = nil
+      Sidekiq.logger
+      cli = parse_options(args)
+      config = parse_config(cli)
+      options.merge!(config.merge(cli))
+      Sidekiq.logger.level = Logger::DEBUG if options[:verbose]
+      Celluloid.logger = nil
+      validate!
+      write_pid
+      boot_system
+    end
+    def run
+      @manager = Sidekiq::Manager.new(options)
+      poller = Sidekiq::Retry::Poller.new
+      begin
+        logger.info 'Starting processing, hit Ctrl-C to stop'
+        @manager.start!
+        poller.poll!
+        sleep
+      rescue Interrupt
+        logger.info 'Shutting down'
+        poller.terminate! if poller.alive?
+        @manager.stop!(:shutdown => true, :timeout => options[:timeout])
+        @manager.wait(:shutdown)
+        # Explicitly exit so busy Processor threads can't block
+        # process shutdown.
+        exit(0)
+      end
+    end
+    def interrupt
+      @interrupt_mutex.synchronize do
+        unless @interrupted
+          @interrupted = true
+          Thread.main.raise Interrupt
+        end
+      end
+    end
+    private
+    def die(code)
+      exit(code)
+    end
+    def options
+      Sidekiq.options
+    end
+    def detected_environment
+      options[:environment] ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+    def boot_system
+      ENV['RACK_ENV'] = ENV['RAILS_ENV'] = detected_environment
+      raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
+      if File.directory?(options[:require])
+        require 'rails'
+        require 'sidekiq/rails'
+        require File.expand_path("#{options[:require]}/config/environment.rb")
+        ::Rails.application.eager_load!
+      else
+        require options[:require]
+      end
+    end
+    def validate!
+      options[:queues] << 'default' if options[:queues].empty?
+      options[:queues].shuffle!
+      if !File.exist?(options[:require]) ||
+         (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
+        logger.info "=================================================================="
+        logger.info "  Please point sidekiq to a Rails 3 application or a Ruby file    "
+        logger.info "  to load your worker classes with -r [DIR|FILE]."
+        logger.info "=================================================================="
+        logger.info @parser
+        die(1)
+      end
+    end
+    def parse_options(argv)
+      opts = {}
+      @parser = OptionParser.new do |o|
+         o.on "-q", "--queue QUEUE,WEIGHT", "Queue to process, with optional weight" do |arg|
+          q, weight = arg.split(",")
+          parse_queues(opts, q, weight)
+        end
+        o.on "-v", "--verbose", "Print more verbose output" do
+          Sidekiq.logger.level = ::Logger::DEBUG
+        end
+        o.on '-e', '--environment ENV', "Application environment" do |arg|
+          opts[:environment] = arg
+        end
+        o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
+          opts[:timeout] = arg.to_i
+        end
+        o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
+          opts[:require] = arg
+        end
+        o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
+          opts[:concurrency] = arg.to_i
+        end
+        o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+          opts[:pidfile] = arg
+        end
+        o.on '-C', '--config PATH', "path to YAML config file" do |arg|
+          opts[:config_file] = arg
+        end
+        o.on '-V', '--version', "Print version and exit" do |arg|
+          puts "Sidekiq #{Sidekiq::VERSION}"
+          die(0)
+        end
+      end
+      @parser.banner = "sidekiq [options]"
+      @parser.on_tail "-h", "--help", "Show help" do
+        logger.info @parser
+        die 1
+      end
+      @parser.parse!(argv)
+      opts
+    end
+    def write_pid
+      if path = options[:pidfile]
+        File.open(path, 'w') do |f|
+          f.puts Process.pid
+        end
+      end
+    end
+    def parse_config(cli)
+      opts = {}
+      if cli[:config_file] && File.exist?(cli[:config_file])
+        opts = YAML.load_file cli[:config_file]
+        queues = opts.delete(:queues) || []
+        queues.each { |name, weight| parse_queues(opts, name, weight) }
+      end
+      opts
+    end
+    def parse_queues(opts, q, weight)
+      (weight || 1).to_i.times do
+       (opts[:queues] ||= []) << q
+      end
+    end
+  end
+end

data/lib/sidekiq/client.rb ADDED

@@ -0,0 +1,72 @@
+require 'multi_json'
+require 'sidekiq/middleware/chain'
+require 'sidekiq/middleware/client/unique_jobs'
+module Sidekiq
+  class Client
+    def self.default_middleware
+      Middleware::Chain.new do |m|
+      end
+    end
+    def self.registered_workers
+      Sidekiq.redis { |x| x.smembers('workers') }
+    end
+    def self.registered_queues
+      Sidekiq.redis { |x| x.smembers('queues') }
+    end
+    ##
+    # The main method used to push a job to Redis.  Accepts a number of options:
+    #
+    #   queue - the named queue to use, default 'default'
+    #   class - the worker class to call, required
+    #   args - an array of simple arguments to the perform method, must be JSON-serializable
+    #   retry - whether to retry this job if it fails, true or false, default true
+    #   backtrace - whether to save any error backtrace, default false
+    #
+    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # symbols in 'args' will be converted to strings.
+    #
+    # Example:
+    #   Sidekiq::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #
+    def self.push(item)
+      raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
+      raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
+      raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].is_a?(Class) || !item['class'].respond_to?('get_sidekiq_options')
+      worker_class = item['class']
+      item['class'] = item['class'].to_s
+      item = worker_class.get_sidekiq_options.merge(item)
+      item['retry'] = !!item['retry']
+      queue = item['queue']
+      pushed = false
+      Sidekiq.client_middleware.invoke(worker_class, item, queue) do
+        payload = Sidekiq.dump_json(item)
+        Sidekiq.redis do |conn|
+          _, pushed = conn.multi do
+            conn.sadd('queues', queue)
+            conn.rpush("queue:#{queue}", payload)
+          end
+        end
+      end
+      !! pushed
+    end
+    # Redis compatibility helper.  Example usage:
+    #
+    #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+    #
+    # Messages are enqueued to the 'default' queue.
+    #
+    def self.enqueue(klass, *args)
+      klass.perform_async(*args)
+    end
+  end
+end