multi_worker 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eff1b67f7d6f0861193b1164fd5976aa053eaa37
+  data.tar.gz: 7d54dffeb1ce6da386919473cb194b0046658e34
+SHA512:
+  metadata.gz: bbaca27df3e2b4c4aaff3faa79658934fd846ffab3da1f76dac7d24586955983099f34e858109bb723b85b91adb31a85061e2d6165aa100e1f79d0d590369b19
+  data.tar.gz: 688cc31e9c28ba87847fccf894dc2bf41cab53005fbe61f7b905fe44d2845df213adee4b574b4f97a6bf82c010daa452d1b69e3e332ee3cc4b63fa536628fcb5

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,4 @@
+--color
+--format progress
+--require spec_helper
+--pattern "spec/*_spec.rb"

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - jruby-19mode
+  - rbx
+matrix:
+  allow_failures:
+    - rvm: rbx

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in multi_worker.gemspec
+gemspec
+
+#gem 'pry-debugger'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 David Butler
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,110 @@
+# MultiWorker [![Build Status](https://travis-ci.org/dwbutler/multi_worker.png?branch=master)](https://travis-ci.org/dwbutler/multi_worker)
+
+MultiWorker provides a common interface to the (many) Ruby queueing/worker libraries.
+They are all very similar, but provide slightly different interfaces. This makes it difficult
+to switch from one library to another, or to use multiple libraries at the same time.
+
+Similar to MultiJSON or ExecJS, MultiWorker automatically detects installed queuing libaries
+and the correct adapter is loaded up by default. Changing queueing libraries does not require
+any change to worker code, as long as the standard interface is used.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'multi_worker'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install multi_worker
+
+## Basic Usage
+
+### Define worker
+
+```ruby
+require 'sidekiq'
+require 'multi_worker'
+
+class ExampleWorker
+  worker
+
+  def perform(foo, bar)
+    # long running code
+  end
+end
+```
+
+### Queue jobs
+
+```ruby
+ExampleWorker.perform_async(1, 2)
+# Equivalent:
+MultiWorker.enqueue(ExampleWorker, 1, 2)
+```
+
+### Work jobs
+
+Add to Rakefile:
+```ruby
+require 'resque'
+require 'multi_worker/tasks'
+
+# If not using Rails, define your own :environment task that will require dependencies
+task :environment do
+  require 'resque'
+end
+```
+
+Run:
+
+```
+QUEUE=default rake multi_worker:work
+```
+
+## Advanced Configuration
+
+```ruby
+MultiWorker.configure do
+  default_queue :processing
+  default_adapter :resque
+  default_options :retry => true
+end
+
+class AdvancedWorker
+  worker :queue => :background, :unique => true, :adapter => :sidekiq
+
+  def perform(foo)
+    ...
+  end
+end
+```
+
+## Feature Comparison
+
+| Library                  | Backends             | Status | Retry | Lock | Unique | Scheduling | Priority | Async Method Proxy | Rake Task | Inline |
+|--------------------------|----------------------|--------|-------|------|--------|------------|----------|--------------------|-----------|--------|
+| Resque                   | Redis                | Gem    | Gem   | Gem  | Gem    | Gem        |          | Gem                | ✓         | ✓      |
+| Sidekiq                  | Redis                | Gem    | ✓     | Gem  | Gem    | ✓          |          | ✓                  | ✗         | ✓      |
+| Delayed Job              | Active Record, Mongo |        |       |      |        | ✓          |          | ✓                  | ✓         | ✓      |
+| Qu                       | Redis, Mongo, SQS    |        |       |      |        |            |          |                    | ✓         | ✓      |
+| Queue Classic            | PostgreSQL           |        |       |      |        |            |          |                    | ✓         | ✗      |
+| Que                      | PostgreSQL           |        |       |      |        | ✓          | ✓        |                    | ✓         | ✓      |
+| Sneakers                 | RabbitMQ             |        |       |      |        |            |          |                    | ✗         | ✗      |
+| TorqueBox Backgroundable | HornetQ              | ✓      |       |      |        |            |          | ✓                  | ✗         | ✗      |
+| Backburner               | Beanstalkd           |        | ✓     |      |        | ✓          | ✓        | ✓                  | ✓         | ✓      |
+| Threaded in Memory Queue | In-Memory            |        |       |      |        |            |          |                    | N/A       | ✓      |
+| Sucker Punch             | In-Memory            |        |       |      |        |            |          |                    | N/A       | ✓      |
+| Inline                   | N/A                  |        |       |      |        |            |          |                    | N/A       | ✓      |
+
+## Contributing
+
+1. Fork it ( http://github.com/dwbutler/multi_worker/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,22 @@
+require "bundler/gem_tasks"
+require "multi_worker"
+
+require 'rspec/core/rake_task'
+
+desc "Run RSpec"
+RSpec::Core::RakeTask.new do |t|
+  t.verbose = true
+  t.pattern = "spec/*_spec.rb"
+end
+
+MultiWorker::AdapterNames.each do |adapter_name|
+  namespace :spec do
+    desc "Run #{adapter_name} adapter spec"
+    RSpec::Core::RakeTask.new(adapter_name) do |t|
+      t.verbose = true
+      t.pattern = "spec/adapters/#{adapter_name}_spec.rb"
+    end
+  end
+end
+
+task :default => [:spec].concat(MultiWorker::AdapterNames.map {|adapter_name| "spec:#{adapter_name}"})

data/lib/multi_worker/adapters/backburner.rb

@@ -0,0 +1,32 @@
+module MultiWorker
+  module Adapters
+    class Backburner
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = opts[:queue]
+          include ::Backburner::Queue
+          queue @queue
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            ::Backburner.enqueue(self, *args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+        end
+      end
+
+      def self.rake_task
+        require 'rake'
+        require 'backburner/tasks'
+
+        ::Rake::Task['backburner:work']
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/delayed_job.rb

@@ -0,0 +1,30 @@
+module MultiWorker
+  module Adapters
+    class DelayedJob
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = opts[:queue]
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            delay(:queue => @queue).perform(*args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+        end
+      end
+
+      def self.rake_task
+        require 'rake'
+        require 'delayed/tasks'
+
+        ::Rake::Task['jobs:work']
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/inline.rb

@@ -0,0 +1,21 @@
+module MultiWorker
+  module Adapters
+    class Inline
+      def self.configure(base, opts={})
+        base.instance_eval do
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            perform(*args)
+          end
+
+          def perform_async(*args)
+            perform(*args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/qu.rb

@@ -0,0 +1,30 @@
+module MultiWorker
+  module Adapters
+    class Qu
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = opts[:queue]
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            ::Qu.enqueue(self, *args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+        end
+      end
+
+      def self.rake_task
+        require 'rake'
+        require 'qu/tasks'
+
+        ::Rake::Task['qu:work']
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/que.rb

@@ -0,0 +1,47 @@
+module MultiWorker
+  module Adapters
+    class Que
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = opts[:queue]
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            @job_klass.queue(*args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+
+          worker_klass = self
+
+          @job_klass = Class.new(::Que::Job)
+          @job_klass.class_eval do
+            @worker_klass = worker_klass
+
+            class << self
+              attr_reader :worker_klass
+            end
+
+            def run(*args)
+              self.class.worker_klass.new.perform(*args)
+            end
+          end
+
+          const_set :Job, @job_klass
+        end
+      end
+
+      def self.rake_task
+        require 'rake'
+        require 'que/rake_tasks'
+
+        ::Rake::Task['que:work']
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/queue_classic.rb

@@ -0,0 +1,29 @@
+module MultiWorker
+  module Adapters
+    class QueueClassic
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = ::QC::Queue.new(opts[:queue])
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            @queue.enqueue("#{self}.perform", *args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+        end
+      end
+
+      def self.rake_task
+        require 'rake'
+        require 'queue_classic/tasks'
+        ::Rake::Task["qc:work"]
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/resque.rb

@@ -0,0 +1,59 @@
+module MultiWorker
+  module Adapters
+    class Resque
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = opts[:queue]
+
+          if @retry = opts[:retry]
+            require "resque-retry"
+            extend ::Resque::Plugins::Retry
+
+            @retry_limit = @retry[:limit]
+            @retry_delay = @retry[:delay]
+          end
+
+          if @lock = opts[:lock]
+            require "resque-lock-timeout"
+            extend ::Resque::Plugins::LockTimeout
+
+            if @lock.is_a?(Hash) && @lock[:timeout]
+              @lock_timeout = @lock[:timeout].to_i
+            end
+
+            if opts[:unique]
+              @loner = true
+            end
+          elsif opts[:unique]
+            require "resque-loner"
+            include ::Resque::Plugins::UniqueJob
+          end
+
+          if opts[:status]
+            require "resque-status"
+            include ::Resque::Plugins::Status
+          end
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            ::Resque.enqueue(self, *args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+        end
+      end
+
+      def self.rake_task
+        require 'rake'
+        require 'resque/tasks'
+
+        ::Rake::Task['resque:work']
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/sidekiq.rb

@@ -0,0 +1,56 @@
+module MultiWorker
+  module Adapters
+    class Sidekiq
+      def self.configure(base, opts={})
+        base.class_eval do
+          if opts[:status]
+            require 'sidekiq_status'
+            include ::SidekiqStatus::Worker
+          else
+            include ::Sidekiq::Worker
+          end
+          
+          @sidekiq_options = ::Sidekiq.default_worker_options.symbolize_keys
+          @sidekiq_options.merge! :queue => opts[:queue]
+
+          if opts.has_key?(:backtrace)
+            @sidekiq_options.merge! :backtrace => opts[:backtrace]
+          end
+
+          if opts.has_key?(:retry)
+            @retry = opts[:retry]
+            case @retry
+            when Hash
+              @sidekiq_options.merge!(:retry => @retry[:limit])
+              sidekiq_retry_in(&@retry[:delay])
+            when true, false, Fixnum
+              @sidekiq_options.merge!(:retry => @retry)
+            end
+          end
+
+          if opts[:lock]
+            require 'sidekiq-lock'
+            include ::Sidekiq::Lock::Worker
+
+            @sidekiq_options.merge!(:lock => opts[:lock])
+          end
+
+          if opts[:unique]
+            require "sidekiq-unique-jobs"
+            @sidekiq_options.merge!(:unique => true)
+          end
+
+          sidekiq_options @sidekiq_options
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/sneakers.rb

@@ -0,0 +1,32 @@
+module MultiWorker
+  module Adapters
+    class Sneakers
+      def self.configure(base, opts={})
+        require 'json'
+
+        base.class_eval do
+          include ::Sneakers::Worker
+          from_queue opts[:queue], opts.fetch(:adapter_opts, {})
+
+          def work(msg)
+            args = JSON.parse(msg)
+            perform(*args)
+          end
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            ::Sneakers.publish(args.to_json, :to_queue => @queue)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+            #@queue.exchange.publish(args.to_json, :to_queue => @queue)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/sucker_punch.rb

@@ -0,0 +1,25 @@
+module MultiWorker
+  module Adapters
+    class SuckerPunch
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = opts[:queue]
+
+          include ::SuckerPunch::Job
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            self.new.async.perform(*args)
+          end
+
+          def perform_async(*args)
+            async.perform(*args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/threaded_in_memory_queue.rb

@@ -0,0 +1,27 @@
+module MultiWorker
+  module Adapters
+    class ThreadedInMemoryQueue
+      def self.configure(base, opts={})
+        base.class_eval do
+          @queue = opts[:queue]
+
+          def self.call(*args)
+            perform(*args)
+          end
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            ::ThreadedInMemoryQueue.enqueue(self, *args)
+          end
+
+          def perform_async(*args)
+            self.class.perform_async(*args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multi_worker/adapters/torquebox_backgroundable.rb

@@ -0,0 +1,25 @@
+module MultiWorker
+  module Adapters
+    class TorqueboxBackgroundable
+      def self.configure(base, opts={})
+        base.class_eval do
+          include ::TorqueBox::Messaging::Backgroundable
+
+          def self.perform(*args)
+            self.new.perform(*args)
+          end
+
+          def self.perform_async(*args)
+            perform(*args)
+          end
+
+          def perform_async(*args)
+            perform(*args)
+          end
+
+          always_background :perform_async
+        end
+      end
+    end
+  end
+end

data/lib/multi_worker/interface.rb

@@ -0,0 +1,24 @@
+module MultiWorker
+	module Interface
+    # Configures queueing for a class or module.
+    # Options:
+    # :queue => name of the queue to use (defaults to :default)
+    # :mailer => Queue mail messages (only for ActionMailer) (defaults to false)
+    # :lockable => Use locking on the job (defaults to true)
+    # :lock_timeout => Optional lock timeout
+    # :loner => Make this job unique in the queue (defaults to false)
+    # :status => Turn on status tracking (defaults to false)
+    #
+    # Example:
+    #
+    # class WorkerClass
+    # worker :queue => :processing, :loner => true
+    def worker(opts={})
+      opts = MultiWorker.default_options.merge(opts)
+      adapter_klass = MultiWorker.adapter(opts[:adapter])
+      adapter_klass.configure(self, opts)
+    end
+  end
+end
+
+Class.send :include, MultiWorker::Interface

data/lib/multi_worker/tasks.rb

@@ -0,0 +1,13 @@
+require 'rake'
+require 'multi_worker'
+
+# No-op task in case it doesn't already exist
+task :environment
+
+namespace :multi_worker do
+  desc "Start a new worker for #{MultiWorker.default_adapter}"
+  task :work => :environment do
+    rake_task = MultiWorker.adapter.rake_task rescue fail("No rake task available for #{MultiWorker.default_adapter}")
+    rake_task.execute
+  end
+end

data/lib/multi_worker/version.rb

@@ -0,0 +1,3 @@
+module MultiWorker
+  VERSION = "0.1.0"
+end