shoryuken-later 0.0.1

data/lib/shoryuken/later/launcher.rb

@@ -0,0 +1,42 @@
+# All of this has been "borrowed" from Shoryuken.
+
+# @see Shoryuken::Launcher
+module Shoryuken
+  module Later
+    class Launcher
+      include Celluloid
+      include Shoryuken::Util
+
+      trap_exit :actor_died
+
+      attr_accessor :manager
+      
+      def initialize
+        @manager = Shoryuken::Later::Manager.new_link
+
+        @done = false
+      end
+
+      def stop(options = {})
+        watchdog('Later::Launcher#stop') do
+          @done = true
+
+          manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken::Later.options[:timeout])
+          manager.wait(:shutdown)
+        end
+      end
+
+      def run
+        watchdog('Later::Launcher#run') do
+          manager.async.start
+        end
+      end
+
+      def actor_died(actor, reason)
+        return if @done
+        logger.warn 'Shoryuken::Later died due to the following error, cannot recover, process exiting'
+        exit 1
+      end
+    end
+  end
+end

data/lib/shoryuken/later/manager.rb

@@ -0,0 +1,138 @@
+# Most of this has been "borrowed" from Shoryuken, but then repurposed for periodic polling.
+
+# @see Shoryuken::Manager
+require 'set'
+require 'shoryuken/later/poller'
+
+module Shoryuken
+  module Later
+    class Manager
+      include Celluloid
+      include Shoryuken::Util
+      
+      def initialize
+        @tables = Shoryuken::Later.tables.dup.uniq
+        
+        @done = false
+        
+        @idle = Set.new([])
+        @busy = Set.new([])
+        @timers = {}
+        
+        @tables.each{|table| Poller.supervise_as :"poller-#{table}", current_actor, table }
+      end
+
+      def start
+        logger.info 'Starting'
+
+        # Start a poller for every table being polled.
+        @tables.each do |table|
+          dispatch table
+          
+          # Save the timer so it can be cancelled at shutdown.
+          @timers[table] = every(Shoryuken::Later.poll_delay) { dispatch table }
+        end
+      end
+
+      def stop(options = {})
+        watchdog('Later::Manager#stop died') do
+          @done = true
+          
+          @timers.each_value{|timer| timer.cancel if timer }
+          @timers.clear
+
+          logger.info { "Shutting down #{@idle.size} idle poller(s)" }
+
+          @idle.each do |name|
+            poller = Actor[name] and poller.alive? and poller.terminate
+          end
+          @idle.clear
+
+          if @busy.empty?
+            return after(0) { signal(:shutdown) }
+          end
+
+          if options[:shutdown]
+            hard_shutdown_in(options[:timeout])
+          else
+            soft_shutdown(options[:timeout])
+          end
+        end
+      end
+
+      def poller_done(table, poller)
+        watchdog('Later::Manager#poller_done died') do
+          logger.debug { "Poller done for '#{table}'" }
+
+          name = :"poller-#{table}"
+          @busy.delete name
+
+          if stopped?
+            poller.terminate if poller.alive?
+          else
+            @idle << name
+          end
+        end
+      end
+
+      def poller_ready(table, poller)
+        watchdog('Later::Manager#poller_ready died') do
+          logger.debug { "Poller for '#{table}' ready" }
+
+          name = :"poller-#{table}"
+          @busy.delete name
+          @idle << name
+        end
+      end
+
+      def stopped?
+        @done
+      end
+
+      private
+      
+      def dispatch(table)
+        name = :"poller-#{table}"
+        
+        # Only start polling if the poller is idle.
+        if ! stopped? && @idle.include?(name)
+          @idle.delete(name)
+          @busy << name
+              
+          Actor[name].async.poll
+        end
+      end
+      
+      def soft_shutdown(delay)
+        logger.info { "Waiting for #{@busy.size} busy pollers" }
+
+        if @busy.size > 0
+          after(delay) { soft_shutdown(delay) }
+        else
+          after(0) { signal(:shutdown) }
+        end
+      end
+
+      def hard_shutdown_in(delay)
+        logger.info { "Waiting for #{@busy.size} busy pollers" }
+        logger.info { "Pausing up to #{delay} seconds to allow pollers to finish..." }
+
+        after(delay) do
+          watchdog("Later::Manager#hard_shutdown_in died") do
+            if @busy.size > 0
+              logger.info { "Hard shutting down #{@busy.size} busy pollers" }
+
+              @busy.each do |busy|
+                if poller = Actor[busy]
+                  t = poller.bare_object.actual_work_thread
+                  t.raise Shutdown if poller.alive?
+                end
+              end
+            end
+            after(0) { signal(:shutdown) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/later/poller.rb

@@ -0,0 +1,91 @@
+require 'json'
+
+module Shoryuken
+  module Later
+    class Poller
+      include Celluloid
+      include Shoryuken::Util
+      
+      attr_reader :table_name
+      
+      def initialize(manager, table_name)
+        @manager = manager
+        @table_name = table_name
+        
+        @manager.async.poller_ready(@table_name, self)
+      end
+      
+      def poll
+        watchdog('Later::Poller#poll died') do
+          started_at = Time.now
+          
+          logger.debug { "Polling for scheduled messages in '#{@table_name}'" }
+          
+          begin
+            while item = next_item
+              id = item.attributes['id']
+              logger.info "Found message #{id} from '#{@table_name}'"
+              defer do
+                if sent_msg = process_item(item)
+                  logger.debug { "Enqueued message #{id} from '#{@table_name}' as #{sent_msg.id}" }
+                else
+                  logger.debug { "Skipping already queued message #{id} from '#{@table_name}'" }
+                end
+              end
+            end
+  
+            logger.debug { "Poller for '#{@table_name}' completed in #{elapsed(started_at)} ms" }
+          rescue => ex
+            logger.error "Error fetching message: #{ex}"
+            logger.error ex.backtrace.first
+          end
+            
+          @manager.async.poller_done(@table_name, self)
+        end
+      end
+
+    private
+    
+      def table
+        Shoryuken::Later::Client.tables(@table_name)
+      end
+
+      # Fetches the next available item from the schedule table.    
+      def next_item
+        table.items.where(:perform_at).less_than((Time.now + Shoryuken::Later::MAX_QUEUE_DELAY).to_i).first
+      end
+    
+      # Processes an item and enqueues it (unless another actor has already enqueued it).
+      def process_item(item)
+        time, worker_class, args, id = item.attributes.values_at('perform_at','shoryuken_class','shoryuken_args','id')
+        
+        worker_class = worker_class.constantize
+        args = JSON.parse(args)
+        time = Time.at(time)
+        queue_name = item.attributes['shoryuken_queue']
+        
+        # Conditionally delete an item prior to enqueuing it, ensuring only one actor may enqueue it.
+        begin item.delete(:if => {id: id})
+        rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
+          # Item was already deleted, so it does not need to be queued.
+          return
+        end
+
+        # Now the item is safe to be enqueued, since the conditional delete succeeded.
+        body, options = args.values_at('body','options')
+        if queue_name.nil?
+          worker_class.perform_in(time, body, options)
+          
+        # For compatibility with Shoryuken's ActiveJob adapter, support an explicit queue name.
+        else
+          delay = (time - Time.now).to_i
+          options[:delay_seconds] = delay if delay > 0
+          options[:message_attributes] ||= {}
+          options[:message_attributes]['shoryuken_class'] = { string_value: worker_class.to_s, data_type: 'String' }
+          Shoryuken::Client.send_message(queue_name, body, options)
+        end
+      end
+
+    end
+  end
+end

data/lib/shoryuken/later/version.rb

@@ -0,0 +1,5 @@
+module Shoryuken
+  module Later
+    VERSION = '0.0.1'
+  end
+end

data/lib/shoryuken/later/worker.rb

@@ -0,0 +1,31 @@
+require 'time'
+require 'json'
+
+module Shoryuken
+  module Later
+    module Worker
+      module ClassMethods
+        
+        def perform_later(time, body, options = {})
+          time = Time.now + time.to_i if Numeric===time
+          time = time.to_time if time.respond_to?(:to_time)
+          raise ArgumentError, 'expected Numeric, Time but got '+time.class.name unless Time===time
+          
+          # Times that are less than 15 minutes in the future can be queued immediately.
+          if time < Time.now + Shoryuken::Later::MAX_QUEUE_DELAY
+            perform_in(time, body, options)
+            
+          # Otherwise, the message is inserted into a DynamoDB table with the same name as the queue.
+          else
+            table = get_shoryuken_options['schedule_table'] || Shoryuken::Later.default_table
+            args = JSON.dump(body: body, options: options)
+            Shoryuken::Later::Client.put_item(table, perform_at: time.to_i, shoryuken_args: args,
+                                                     shoryuken_class: self.to_s)
+          end
+        end
+      end
+    end
+  end
+  
+  Worker::ClassMethods.send :include, Later::Worker::ClassMethods
+end

data/lib/shoryuken/later.rb

@@ -0,0 +1,52 @@
+require 'shoryuken'
+require 'shoryuken/later/version'
+require 'shoryuken/later/client'
+require 'shoryuken/later/worker'
+
+module Shoryuken
+  module Later
+    MAX_QUEUE_DELAY = 15 * 60
+    
+    DEFAULT_POLL_DELAY = 5 * 60
+    
+    DEFAULTS = {
+      aws: {},
+      later: {
+        tables: [],
+        delay: DEFAULT_POLL_DELAY,
+      },
+      timeout: 8
+    }
+    
+    @@tables = []
+    @@default_table = 'shoryuken_later'
+    
+    class << self
+      def options
+        @options ||= DEFAULTS.dup
+      end
+      
+      def poll_delay
+        options[:later][:delay] || DEFAULT_POLL_DELAY
+      end
+      
+      def default_table
+        @@default_table
+      end
+      
+      def default_table=(table)
+        @@default_table = table
+      end
+      
+      def tables
+        @@tables
+      end
+    
+      def logger
+        Shoryuken::Logging.logger
+      end
+    end
+  end
+end
+
+require 'shoryuken/later/active_job_adapter' if defined?(::ActiveJob)

data/lib/shoryuken-later.rb

@@ -0,0 +1 @@
+require 'shoryuken/later'

data/shoryuken-later.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "shoryuken/later/version"
+
+Gem::Specification.new do |spec|
+  spec.name        = "shoryuken-later"
+  spec.version     = Shoryuken::Later::VERSION
+  spec.authors     = ["Joe Khoobyar"]
+  spec.email       = ["joe@khoobyar.name"]
+  spec.homepage    = "http://github.com/joekhoobyar/shoryuken-later"
+  spec.summary     = 'A scheduling plugin (using Dynamo DB) for Shoryuken'
+  spec.description = %Q{
+    This gem provides a scheduling plugin (using Dynamo DB) for Shoryuken, as well as an ActiveJob adapter
+  }
+  
+  spec.license = "LGPL-3.0"
+  spec.files = `git ls-files -z`.split("\x0")
+  spec.executables = %w[shoryuken-later]
+  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  
+  spec.required_ruby_version = '>= 1.9.3'
+  
+  spec.add_development_dependency "bundler", '>= 1.3.5'
+  spec.add_development_dependency "rake",    '~> 10.0'
+  spec.add_development_dependency "rspec",   '~> 3.0', '< 3.1'
+
+  spec.add_dependency "aws-sdk-v1"
+  spec.add_dependency "celluloid", "~> 0.15.2"
+  spec.add_dependency "shoryuken", "~> 0.0.4"
+end

data/spec/shoryuken/later/client_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe Shoryuken::Later::Client do
+  let(:ddb)              { double 'DynamoDB' }
+  let(:table_collection) { double 'Table Collection' }
+  let(:ddb_table)        { double 'DynamoDb Table' }
+  let(:ddb_items)        { double 'Table Items' }
+  let(:table)            { 'shoryuken_later' }
+
+  before do
+    allow(described_class).to receive(:ddb).and_return(ddb)
+    allow(ddb).to receive(:tables).and_return(table_collection)
+    allow(table_collection).to receive(:[]).and_return(ddb_table)
+    allow(ddb_table).to receive(:items).and_return(ddb_items)
+    allow(ddb_table).to receive(:hash_key=)
+  end
+
+  describe '.tables' do
+    it 'memoizes tables and sets the hash_key' do
+      expect(table_collection).to receive(:[]).once.with(table).and_return(ddb_table)
+      expect(ddb_table).to receive(:hash_key=).once.with([:id, :string])
+
+      expect(described_class.tables(table)).to eq(ddb_table)
+      expect(described_class.tables(table)).to eq(ddb_table)
+    end
+  end
+  
+  describe '.put_item' do
+    it 'creates an item with a supplied ID' do
+      expect(ddb_items).to receive(:create).with({'id' => 'fubar'}, {unless_exists: 'id'})
+        
+      described_class.put_item(table,'id' => 'fubar')
+    end
+    
+    it 'creates an item with a auto-generated ID' do
+      expect(SecureRandom).to receive(:uuid).once.and_return('fubar')
+      expect(ddb_items).to receive(:create).with({'id' => 'fubar', 'perform_at' => 1234}, {unless_exists: 'id'})
+        
+      described_class.put_item(table,'perform_at' => 1234)
+    end
+  end
+end

data/spec/shoryuken/later/poller_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+require 'shoryuken/later/poller'
+require 'shoryuken/later/manager'
+
+describe Shoryuken::Later::Poller do
+  let(:manager)   { double Shoryuken::Later::Manager, poller_ready: nil, poller_done: nil }
+  let(:ddb_table) { double 'DynamoDb Table' }
+  let(:ddb_items) { double 'Table Items' }
+  let(:table)     { 'shoryuken_later' }
+  
+  let(:body)      { {'foo' => 'bar'} }  
+  let(:json)      { JSON.dump(body: body, options: {}) }
+    
+  let(:ddb_item)  do
+    double AWS::DynamoDB::Item, delete: nil,
+      attributes: {'id' => 'fubar', 'perform_at' => Time.now + 60, 'shoryuken_args' => json, 'shoryuken_class' => 'TestWorker'}
+  end
+
+  before do
+    allow(manager).to receive(:async).and_return(manager)
+    allow(Shoryuken::Later::Client).to receive(:tables).with(table).and_return(ddb_table)
+  end
+
+  subject do
+    described_class.new(manager, table)
+  end
+  
+  describe '#initialize' do
+    it 'informs the manager that the poller is ready' do
+      expect(manager).to receive(:poller_ready).once
+      
+      subject.inspect
+      subject.inspect
+    end
+  end
+
+  describe '#poll' do
+    it 'pulls items from #next_item, and processes with #process_item' do
+      items = [ddb_item]
+      expect_any_instance_of(described_class).to receive(:next_item).twice { items.pop }
+      expect_any_instance_of(described_class).to receive(:process_item).once.with(ddb_item)
+      expect(manager).to receive(:poller_done).once
+
+      subject.poll
+    end
+    
+    it 'informs the manager after polling is done' do
+      items = []
+      expect_any_instance_of(described_class).to receive(:next_item).once { items.pop }
+      expect_any_instance_of(described_class).not_to receive(:process_item)
+      expect(manager).to receive(:poller_done).once
+
+      subject.poll
+    end
+  end
+  
+  describe '#process_item' do
+    it 'enqueues a message if the item could be deleted' do
+      expect(TestWorker).to receive(:perform_in).once do |time,body,options|
+        expect(time   ).to be > Time.now
+        expect(body   ).to eq(body)
+        expect(options).to be_empty
+      end
+      
+      subject.send(:process_item, ddb_item)
+    end
+    
+    it 'does not enqueue a message if the item could not be deleted' do
+      expect(TestWorker).not_to receive(:perform_in)
+      allow(ddb_item).to receive(:delete) { raise AWS::DynamoDB::Errors::ConditionalCheckFailedException }
+      
+      subject.send(:process_item, ddb_item)
+    end
+  end
+end

data/spec/shoryuken/worker_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe 'Shoryuken::Worker' do
+  let(:sqs_queue) { double 'SQS Queue' }
+  let(:queue)     { 'shoryuken_later' }
+  let(:table)     { 'shoryuken_later' }
+  let(:msg_attrs) { { 'shoryuken_class' => { string_value: TestWorker.to_s, data_type: 'String' } } }
+
+  before do
+    allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
+  end
+
+  describe '.perform_later' do
+    it 'delays a message for up to 15 minutes in the future' do
+      expect(sqs_queue).to receive(:send_message).with('message', { message_attributes: msg_attrs, delay_seconds: 15 * 60 })
+        
+      TestWorker.perform_later(15 * 60, 'message')
+    end
+
+    it 'schedules a message for over 15 minutes in the future' do
+      json = JSON.dump(body: 'message', options: {})
+      future = Time.now + 15 * 60 + 1
+      
+      expect(Shoryuken::Later::Client).to receive(:put_item) do |table,attrs|
+        expect(attrs[:perform_at]).to be >= future.to_i
+        expect(attrs[:shoryuken_args]).to eq(json)
+        expect(attrs[:shoryuken_class]).to eq('TestWorker')
+      end
+      
+      TestWorker.perform_later(15 * 60 + 1, 'message')
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,74 @@
+require 'bundler/setup'
+Bundler.setup
+
+require 'celluloid'
+require 'shoryuken-later'
+require 'json'
+
+options_file = File.join(File.expand_path('../..', __FILE__), 'shoryuken.yml')
+
+$options = {}
+
+if File.exists? options_file
+  $options = YAML.load(File.read(options_file)).deep_symbolize_keys
+
+  AWS.config $options[:aws]
+end
+
+Shoryuken.logger.level = Logger::UNKNOWN
+Celluloid.logger.level = Logger::UNKNOWN
+
+# For Ruby 1.9
+module Kernel
+  def Hash(arg)
+    case arg
+    when NilClass
+      {}
+    when Hash
+      arg
+    when Array
+      Hash[*arg]
+    else
+      raise TypeError
+    end
+  end unless method_defined? :Hash
+end
+
+# For Ruby 1.9
+class Hash
+  def to_h
+    self
+  end unless method_defined? :to_h
+end
+
+class TestWorker
+  include Shoryuken::Worker
+
+  shoryuken_options queue: 'shoryuken_later', schedule_table: 'shoryuken_later'
+
+  def perform(sqs_msg, body); end
+end
+
+RSpec.configure do |config|
+  config.before do
+    Shoryuken::Later::Client.class_variable_set :@@tables, {}
+
+    Shoryuken::Later.options.clear
+    Shoryuken::Later.options.merge!($options)
+
+    Shoryuken::Later.tables.replace(['shoryuken_later'])
+
+    Shoryuken::Later.options[:later][:delay] = 60
+    Shoryuken::Later.options[:later][:tables] = ['shoryuken_later']
+    Shoryuken::Later.options[:timeout]       = 1
+
+    Shoryuken::Later.options[:aws] = {}
+
+    TestWorker.get_shoryuken_options.clear
+    TestWorker.get_shoryuken_options['queue'] = 'shoryuken_later'
+    TestWorker.get_shoryuken_options['schedule_table'] = 'shoryuken_later'
+
+    Shoryuken.workers.clear
+    Shoryuken.register_worker('shoryuken_later', TestWorker)
+  end
+end