drb_queue 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.ruby-gemset

@@ -0,0 +1 @@
+drb_queue

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.0.0-p195

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - "1.9.2"
+  - "1.9.3"
+  - "2.0.0"
+services:
+  - redis-server

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Andrew Warner
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# DRbQueue
+
+![](https://travis-ci.org/a-warner/drb_queue.png)
+
+Simple separate-process queue system using DRb
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'drb_queue'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install drb_queue
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/drb_queue.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'drb_queue/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "drb_queue"
+  spec.version       = DRbQueue::VERSION
+  spec.authors       = ["Andrew Warner"]
+  spec.email         = ["wwarner.andrew@gmail.com"]
+  spec.description   = %q{Simple drb-based queue/worker system}
+  spec.summary       = %q{Simple drb-based queue/worker system}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency     "uuid", "~> 2.3.7"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-doc"
+  spec.add_development_dependency "redis"
+  spec.add_development_dependency "redis-namespace"
+end

data/lib/drb_queue/configuration.rb

@@ -0,0 +1,52 @@
+module DRbQueue
+  class Configuration
+    attr_accessor :socket_location, :num_workers, :logger, :error_handler, :immediate, :persistence_store
+
+    def initialize
+      self.socket_location = '/tmp/drb_queue'
+      self.num_workers = 1
+      self.logger = Logger.new(STDOUT)
+      self.error_handler = lambda { |e| logger.error(([e.message] + e.backtrace).join("\n")) }
+    end
+
+    def store(klass, options = {})
+      raise "Whoops, that's not a #{DRbQueue::Store}" unless klass <= DRbQueue::Store
+
+      self.persistence_store = [klass, options]
+    end
+
+    def construct_persistence_store
+      return unless persistence_store
+
+      persistence_store[0].new(persistence_store[1])
+    end
+
+    def immediate!
+      self.immediate = true
+    end
+
+    def on_error(&block)
+      self.error_handler = block
+    end
+
+    def after_fork(&block)
+      after_fork_callbacks << block
+    end
+
+    def before_fork(&block)
+      before_fork_callbacks << block
+    end
+
+    def after_fork_callbacks
+      @after_fork_callbacks ||= []
+    end
+
+    def before_fork_callbacks
+      @before_fork_callbacks ||= []
+    end
+
+    def server_uri
+      "drbunix:#{socket_location}"
+    end
+  end
+end

data/lib/drb_queue/server.rb

@@ -0,0 +1,130 @@
+require "uuid"
+require "thread"
+
+module DRbQueue
+  class Server
+    NotStarted = Class.new(StandardError)
+    AlreadyStarted = Class.new(StandardError)
+
+    class UnableToStart < StandardError
+      attr_reader :cause
+      def initialize(message, cause)
+        super("#{message}: Cause is #{cause.inspect}")
+        @cause = cause
+      end
+    end
+
+    class Work < Struct.new(:worker, :args)
+      def perform
+        worker.perform(*args)
+      end
+
+      def self.unserialize(serialized)
+        hash = JSON.parse(serialized)
+        worker = hash['worker'].split('::').inject(Object) { |o, k| o.const_get(k) }
+
+        new(worker, hash['args'])
+      end
+
+      def serialize
+        {'worker' => worker.to_s, 'args' => args}.to_json
+      end
+    end
+
+    def initialize(configuration)
+      [:logger, :error_handler, :immediate].each do |p|
+        __send__("#{p}=", configuration.__send__(p))
+      end
+
+      self.queue = Queue.new
+      self.running = true
+
+      self.store = configuration.construct_persistence_store
+
+      start_workers(configuration.num_workers)
+
+      if store
+        store.each_persisted_work do |serialized_work|
+          enqueue_work(Work.unserialize(serialized_work))
+        end
+      end
+    end
+
+    def enqueue(worker, *args)
+      uuid.generate.tap do |id|
+        enqueue_work(Work.new(worker, args))
+      end
+    end
+
+    def uuid
+      @uuid ||= UUID.new
+    end
+
+    def ping
+      'pong'
+    end
+
+    def shutdown!
+      self.running = false
+
+      if store
+        begin
+          while work = queue.pop(:dont_block)
+            store.persist(work)
+          end
+        rescue ThreadError => e
+        rescue => e
+          error_handler.call(e)
+        end
+      elsif queue.size > 0
+        logger.error("Queue is non-empty and we're shutting down...probably better to configure a persistence store\n")
+      end
+
+      workers.each(&:join)
+    end
+
+    private
+    attr_accessor :queue, :logger, :error_handler, :immediate, :store, :running
+    alias_method :immediate?, :immediate
+    alias_method :running?, :running
+
+    def enqueue_work(work)
+      if immediate?
+        work.perform
+      else
+        queue << work
+      end
+    end
+
+    def start_workers(num)
+      num.times.map { start_worker }
+    end
+
+    def workers
+      @workers ||= []
+    end
+
+    def start_worker
+      thread = Thread.new do
+        loop do
+          begin
+            break unless running?
+
+            work = queue.pop(:non_blocking)
+            work.perform
+          rescue ThreadError => e
+            sleep 0.05
+          rescue => e
+            error_handler.call(e)
+            start_worker
+            break
+          end
+        end
+
+        workers.delete(thread)
+      end
+
+      workers << thread
+    end
+  end
+end

data/lib/drb_queue/store/redis.rb

@@ -0,0 +1,31 @@
+require 'redis'
+require 'redis-namespace'
+
+module DRbQueue
+  class Store
+    class Redis < Store
+      def initialize(options = {})
+        @redis = ::Redis::Namespace.new(:DRbQueue, :redis => options.fetch(:redis, lambda { ::Redis.new }).call)
+      end
+
+      def persist(work)
+        redis.rpush(persistence_key, work.serialize)
+      end
+
+      def each_persisted_work
+        return enum_for(:each_persisted_work) unless block_given?
+
+        while serialized_work = redis.lpop(persistence_key)
+          yield(serialized_work)
+        end
+      end
+
+      private
+      attr_reader :redis
+
+      def persistence_key
+        'persisted:work'
+      end
+    end
+  end
+end

data/lib/drb_queue/store.rb

@@ -0,0 +1,13 @@
+module DRbQueue
+  class Store
+    NotImplementedError = Class.new(StandardError)
+
+    def persist(work)
+      raise NotImplementedError, "Must implement #persist"
+    end
+
+    def each_persisted_work
+      raise NotImplementedError, "Must implement #each_persisted_work"
+    end
+  end
+end

data/lib/drb_queue/version.rb

@@ -0,0 +1,3 @@
+module DRbQueue
+  VERSION = "0.0.1"
+end

data/lib/drb_queue.rb

@@ -0,0 +1,135 @@
+require "drb_queue/version"
+require "drb_queue/store"
+require 'drb/drb'
+require 'drb/unix'
+require "fileutils"
+require 'timeout'
+require 'json'
+require 'monitor'
+
+module DRbQueue
+  extend self
+  extend Forwardable
+
+  autoload :Server, 'drb_queue/server'
+  autoload :Configuration, 'drb_queue/configuration'
+
+  ConfiguredAfterStarted = Class.new(StandardError)
+
+  attr_reader :started
+  alias_method :started?, :started
+
+  def enqueue(worker, *args)
+    raise Server::NotStarted, "You must start the server first" unless started?
+    raise ArgumentError, "#{worker} is not a module" unless worker.is_a?(Module)
+    raise ArgumentError, "#{worker} does not respond to perform" unless worker.respond_to?(:perform)
+
+    server.enqueue(worker, *args)
+  end
+
+  def start!
+    raise Server::AlreadyStarted, "The server is already started" if started?
+
+    synchronize do
+      return if started?
+
+      @pid = fork_server
+      connect_client!
+
+      at_exit { shutdown! }
+      @started = true
+    end
+  end
+
+  def configure
+    raise ConfiguredAfterStarted, "You must configure #{self.name} BEFORE starting the server" if started?
+
+    synchronize { yield configuration }
+  end
+
+  def shutdown!(immediately = false)
+    return unless started?
+
+    synchronize do
+      return unless started?
+
+      Process.kill(immediately ? 'KILL' : 'TERM', pid)
+
+      begin
+        ::Timeout.timeout(20) { Process.wait }
+      rescue Timeout::Error
+        Process.kill('KILL', pid)
+        Process.wait
+        logger.error("#{self}: forced shutdown")
+      ensure
+        cleanup_socket
+        @started = false
+        @pid = nil
+      end
+    end
+  end
+
+  def connect_client!
+    synchronize do
+      tries = 0
+
+      begin
+        @server = DRbObject.new_with_uri(server_uri)
+        @server.ping
+      rescue DRb::DRbConnError => e
+        raise Server::UnableToStart.new("Couldn't start up the queue server", e) if tries > 3
+        tries += 1
+        sleep 0.2
+        retry
+      end
+    end
+  end
+
+  private
+  attr_reader :pid, :server
+
+  def fork_server
+    cleanup_socket
+
+    execute_before_fork_callbacks
+
+    fork do
+      execute_after_fork_callbacks
+
+      server = Server.new(configuration)
+      DRb.start_service(server_uri, server)
+
+      shutting_down = false
+      trap('TERM') { shutting_down = true }
+      sleep 0.1 until shutting_down
+
+      server.shutdown!
+      DRb.stop_service
+    end
+  end
+
+  def execute_before_fork_callbacks
+    before_fork_callbacks.each(&:call)
+  end
+
+  def execute_after_fork_callbacks
+    after_fork_callbacks.each(&:call)
+  end
+
+  def configuration
+    @configuration ||= Configuration.new
+  end
+  def_delegators :configuration, :server_uri, :socket_location, :before_fork_callbacks, :after_fork_callbacks, :num_workers, :logger
+
+  def synchronize(&block)
+    synchronization_monitor.synchronize(&block)
+  end
+
+  def synchronization_monitor
+    @synchronization_monitor ||= Monitor.new
+  end
+
+  def cleanup_socket
+    FileUtils.rm(socket_location) if File.exist?(socket_location)
+  end
+end

data/spec/lib/drb_queue_spec.rb

@@ -0,0 +1,161 @@
+require 'spec_helper'
+
+describe DRbQueue do
+  it { should_not be_nil }
+
+  def connect_to_redis!
+    Redis.current = Redis::Namespace.new(described_class.to_s, :redis => Redis.new(:db => 8))
+  end
+
+  before(:all) do
+    DRbQueue.configure do |c|
+      c.after_fork do
+        connect_to_redis!
+      end
+    end
+  end
+
+  def per_test_configuration(config); end
+
+  before do
+    connect_to_redis!
+
+    DRbQueue.configure do |c|
+      @old_config = c.dup
+      per_test_configuration(c)
+    end
+
+    DRbQueue.start!
+  end
+
+  let(:shutdown_immediately) { false }
+  after do
+    DRbQueue.shutdown!(shutdown_immediately)
+    DRbQueue.instance_variable_set('@configuration', @old_config)
+    Redis.current.flushall
+  end
+
+  class SetKeyToValueWorker
+    def self.perform(key, value, options = {})
+      sleep options[:sleep_time_before_working] if options[:sleep_time_before_working]
+      Redis.current.set(key, value)
+    end
+  end
+
+  context SetKeyToValueWorker do
+    let(:key) { 'foo' }
+    let(:value) { 'bar' }
+
+    it 'should do work asynchronously' do
+      DRbQueue.enqueue(SetKeyToValueWorker, key, value, :sleep_time_before_working => 0.1)
+      expect(Redis.current.get(key)).to be_nil
+      sleep 0.2
+      expect(Redis.current.get(key)).to eq(value)
+    end
+
+    it 'should do work in order' do
+      DRbQueue.enqueue(SetKeyToValueWorker, key, 'nottherightanswer')
+      DRbQueue.enqueue(SetKeyToValueWorker, key, value)
+      sleep 0.2
+      expect(Redis.current.get(key)).to eq(value)
+    end
+
+    context 'in parallel' do
+      def per_test_configuration(config)
+        config.num_workers = 5
+      end
+
+      it 'should do work in parallel' do
+        time = Benchmark.realtime do
+          5.times { |i| DRbQueue.enqueue(SetKeyToValueWorker, i, i.to_s, :sleep_time_before_working => 0.1) }
+          sleep 0.2
+          5.times { |i| expect(Redis.current.get(i)).to eq(i.to_s) }
+        end
+
+        expect(time).to be < 0.4
+      end
+    end
+  end
+
+  class SleepyWorker
+    def self.perform
+      sleep 100
+    end
+  end
+
+  context SleepyWorker do
+    let(:shutdown_immediately) { true }
+
+    it 'should not block regular execution' do
+      expect(Benchmark.realtime { DRbQueue.enqueue(SleepyWorker) }).to be < 1
+    end
+  end
+
+  class ExceptionWorker
+    def self.perform
+      raise "Get me outta here!"
+    end
+  end
+
+  context ExceptionWorker do
+    it 'should continue operating normally' do
+      DRbQueue.enqueue(ExceptionWorker)
+      sleep 0.1
+      DRbQueue.enqueue(SetKeyToValueWorker, 'a', 'b')
+      sleep 0.1
+      expect(Redis.current.get('a')).to eq('b')
+    end
+
+    def per_test_configuration(config)
+      config.on_error do |e|
+        Redis.current.set('error', e.message)
+      end
+    end
+
+    it 'allows custom exception handlers' do
+      DRbQueue.enqueue(ExceptionWorker)
+      sleep 0.1
+      expect(Redis.current.get('error')).not_to be_nil
+    end
+  end
+
+  context 'immediate mode' do
+    def per_test_configuration(config)
+      config.immediate!
+    end
+
+    it 'should do work synchronously' do
+      DRbQueue.enqueue(SetKeyToValueWorker, 'immediate', 'results', :sleep_time_before_working => 0.1)
+      expect(Redis.current.get('immediate')).to eq('results')
+    end
+  end
+
+  context 'error cases' do
+    it 'should blow up if something besides a module is passed in' do
+      expect { DRbQueue.enqueue('hahaha', 1, 2) }.to raise_error ArgumentError
+    end
+
+    it 'should blow up unless the module responds to perform' do
+      expect { DRbQueue.enqueue(Class.new) }.to raise_error ArgumentError
+    end
+  end
+
+  context 'job persistence' do
+    def per_test_configuration(config)
+      require 'drb_queue/store/redis'
+      config.store DRbQueue::Store::Redis
+    end
+
+    it 'should persist jobs on clean shutdown' do
+      5.times { |i| DRbQueue.enqueue(SetKeyToValueWorker, i, i.to_s, :sleep_time_before_working => 0.1) }
+      DRbQueue.shutdown!
+
+      expect(0.upto(4).all? { |i| Redis.current.get(i) == i.to_s }).to be_false
+
+      DRbQueue.start!
+      sleep 0.5
+
+      expect(0.upto(4).all? { |i| Redis.current.get(i) == i.to_s }).to be_true
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'rubygems'
+require 'bundler/setup'
+Bundler.require(:development)
+require 'drb_queue'
+require 'benchmark'

metadata

@@ -0,0 +1,202 @@
+--- !ruby/object:Gem::Specification
+name: drb_queue
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Andrew Warner
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.7
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple drb-based queue/worker system
+email:
+- wwarner.andrew@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .ruby-gemset
+- .ruby-version
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- drb_queue.gemspec
+- lib/drb_queue.rb
+- lib/drb_queue/.configuration.swp
+- lib/drb_queue/configuration.rb
+- lib/drb_queue/server.rb
+- lib/drb_queue/store.rb
+- lib/drb_queue/store/redis.rb
+- lib/drb_queue/version.rb
+- spec/lib/drb_queue_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -713522776135624504
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -713522776135624504
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Simple drb-based queue/worker system
+test_files:
+- spec/lib/drb_queue_spec.rb
+- spec/spec_helper.rb
+has_rdoc: