shoryuken-later 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4ed2b69dae97667b38ef15a0be1ee0f6989e9c95
-  data.tar.gz: 82fa4cb2cb2f763be48094df3f4224a025d56815
+  metadata.gz: c0ffbca529e8f0d07572a00983be2e0c708044f1
+  data.tar.gz: 65529758322a4bcdb4c38d18fe66f8f7c3f9e59c
 SHA512:
-  metadata.gz: 9f1d6efd2b3521a763e685e65d3c695fa8282ab6f3b0b4b0dc6450d26a0b656ffffaf40057db9bf35cee63ac1f9708715649581ff0094ddf2d0bd707a195cf02
-  data.tar.gz: f5f07fd722f05fc37206b8e4a59e1acf2960282a7656d60275df746458b60bbf0b3be0e2e82d649fd5ae4fefc0875025cfd84d4e839bb45927c20f38453d706a
+  metadata.gz: c64832d614bda4caab3b05b13293acab6a36f7c2352971a0ac64bcf358034547630425ceb5e534eb2176edde0e18b933e060687ca8a20cf2ca5f4306fe2e6955
+  data.tar.gz: a1a8e0ad1961ebc4ce371b26598c1c2ac624faac4ef2a2ae705fa7ccbf5aca5173b6d7e9a8f2955aa34dca937872f617c15b608e5df586da941d282556250cc7

data/lib/shoryuken/later/active_job_adapter.rb

@@ -30,12 +30,13 @@ module ActiveJob
 
           delay = (timestamp - Time.current.to_f).round
           if delay > 15.minutes
-            Shoryuken::Later::Client.put_item(Shoryuken::Later.default_table, perform_at: Time.current.to_i + delay.to_i,
-                                                                              shoryuken_queue: job.queue_name, shoryuken_class: JobWrapper.to_s,
-                                                                              shoryuken_args: JSON.dump(body: job.serialize, options: {}))
+            Shoryuken::Later::Client.create_item(Shoryuken::Later.default_table, perform_at: Time.current.to_i + delay.to_i,
+                                                                                 shoryuken_queue: job.queue_name, shoryuken_class: JobWrapper.to_s,
+                                                                                 shoryuken_args: JSON.dump(body: job.serialize, options: {}))
           else
-            Shoryuken::Client.send_message(job.queue_name, job.serialize, delay_seconds: delay,
-                                                                          message_attributes: message_attributes)
+            Shoryuken::Client.queues(job.queue_name).send_message(message_body: job.serialize,
+                                                                  message_attributes: message_attributes,
+                                                                  delay_seconds: delay)
           end
         end
       end

data/lib/shoryuken/later/cli.rb

@@ -18,7 +18,7 @@ module Shoryuken
       def run(args)
         self_read, self_write = IO.pipe
 
-        %w[INT TERM USR1 USR2 TTIN].each do |sig|
+        %w[INT TERM USR1 USR2].each do |sig|
           trap sig do
             self_write.puts(sig)
           end
@@ -34,31 +34,33 @@ module Shoryuken
         daemonize
         write_pid
         
-        logger.info 'Starting'
-        
-        # Initialize the timers and poller.
-        @timers = Timers::Group.new
-        require 'shoryuken/later/poller'
-        @pollers = Shoryuken::Later.tables.map{|tbl| Poller.new(tbl) }
-          
-        begin
-          # Poll for items on startup, and every :poll_delay
-          poll_tables
-          @timers.every(Shoryuken::Later.poll_delay){ poll_tables }
+        Shoryuken::Logging.with_context '[later]' do
+          logger.info 'Starting'
           
-          # Loop watching for signals and firing off of timers
-          loop do
-            interval = @timers.wait_interval
-            readable, writable = IO.select([self_read], nil, nil, interval)
-            if readable
-              handle_signal readable.first.gets.strip
-            else
-              @timers.fire
+          # Initialize the timers and poller.
+          @timers = Timers::Group.new
+          require 'shoryuken/later/poller'
+          @pollers = Shoryuken::Later.tables.map{|tbl| Poller.new(tbl) }
+            
+          begin
+            # Poll for items on startup, and every :poll_delay
+            poll_tables
+            @timers.every(Shoryuken::Later.poll_delay){ poll_tables }
+            
+            # Loop watching for signals and firing off of timers
+            while @timers
+              interval = @timers.wait_interval
+              readable, writable = IO.select([self_read], nil, nil, interval)
+              if readable
+                handle_signal readable.first.gets.strip
+              else
+                @timers.fire
+              end
             end
+          rescue Interrupt
+            @timers.cancel
+            exit 0
           end
-        rescue Interrupt
-          @timers.cancel
-          exit 0
         end
       end
       
@@ -191,8 +193,7 @@ module Shoryuken
         when 'USR1'
           logger.info "Received USR1, will soft shutdown down"
           @timers.cancel
-          sleep 1 while @busy
-          exit 0
+          @timers = nil
         else
           logger.info "Received #{sig}, will shutdown down"
           raise Interrupt
@@ -252,11 +253,9 @@ module Shoryuken
         Shoryuken::Later.tables.uniq.each do |table|
           # validate all tables and AWS credentials consequently
           begin
-            unless Shoryuken::Later::Client.tables(table).exists?
-              raise ArgumentError, "Table '#{table}' does not exist"
-            end
-          rescue => e
-            raise
+            Shoryuken::Later::Client.tables table
+          rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
+            raise ArgumentError, "Table '#{table}' does not exist"
           end
         end
       end
@@ -264,7 +263,23 @@ module Shoryuken
       def initialize_aws
         # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
         # when not explicit supplied
-        AWS.config Shoryuken::Later.options[:aws] if Shoryuken::Later.options[:aws]
+        return if Shoryuken::Later.options[:aws].empty?
+  
+        shoryuken_keys = %i(
+          account_id
+          sns_endpoint
+          sqs_endpoint
+          receive_message)
+  
+        aws_options = Shoryuken::Later.options[:aws].reject do |k, v|
+          shoryuken_keys.include?(k)
+        end
+  
+        credentials = Aws::Credentials.new(
+          aws_options.delete(:access_key_id),
+          aws_options.delete(:secret_access_key))
+  
+        Aws.config = aws_options.merge(credentials: credentials)
       end
 
       def require_workers

data/lib/shoryuken/later/client.rb

@@ -4,21 +4,31 @@ module Shoryuken
   module Later
     class Client
       @@tables = {}
-        
+      
       class << self
         def tables(table)
-          @@tables[table.to_s] ||= ddb.tables[table].tap do |tbl|
-            tbl.hash_key = [:id, :string]
-          end
+          @@tables[table] ||= ddb.describe_table(table_name: table)
+        end
+        
+        def first_item(table, filter=nil)
+          result = ddb.scan(table_name: table, limit: 1, scan_filter: filter)
+          result.items.first if result
+        end
+        
+        def create_item(table, item)
+          item['id'] ||= SecureRandom.uuid
+          
+          ddb.put_item(table_name: table, item: item,
+                       expected: {id: {exists: false}})
         end
         
-        def put_item(table, attributes, options={unless_exists: 'id'})
-          attributes['id'] ||= SecureRandom.uuid
-          tables(table).items.create(attributes, options)
+        def delete_item(table, item)
+          ddb.delete_item(table_name: table, key: {id: item['id']},
+                          expected: {id: {value: item['id'], exists: true}})
         end
         
         def ddb
-          @ddb ||= AWS::DynamoDB.new
+          @ddb ||= Aws::DynamoDB::Client.new
         end
       end
     end

data/lib/shoryuken/later/poller.rb

@@ -15,20 +15,20 @@ module Shoryuken
         watchdog('Later::Poller#poll died') do
           started_at = Time.now
           
-          logger.debug { "Polling for scheduled messages in '#{@table_name}'" }
+          logger.debug { "Polling for scheduled messages in '#{table_name}'" }
           
           begin
             while item = next_item
-              id = item.attributes['id']
-              logger.info "Found message #{id} from '#{@table_name}'"
+              id = item['id']
+              logger.info "Found message #{id} from '#{table_name}'"
               if sent_msg = process_item(item)
-                logger.debug { "Enqueued message #{id} from '#{@table_name}' as #{sent_msg.id}" }
+                logger.debug { "Enqueued message #{id} from '#{table_name}'" }
               else
-                logger.debug { "Skipping already queued message #{id} from '#{@table_name}'" }
+                logger.debug { "Skipping already queued message #{id} from '#{table_name}'" }
               end
             end
   
-            logger.debug { "Poller for '#{@table_name}' completed in #{elapsed(started_at)} ms" }
+            logger.debug { "Poller for '#{table_name}' completed in #{elapsed(started_at)} ms" }
           rescue => ex
             logger.error "Error fetching message: #{ex}"
             logger.error ex.backtrace.first
@@ -38,26 +38,29 @@ module Shoryuken
 
     private
     
-      def table
-        Shoryuken::Later::Client.tables(@table_name)
+      def client
+        Shoryuken::Later::Client
       end
-
+    
       # Fetches the next available item from the schedule table.    
       def next_item
-        table.items.where(:perform_at).less_than((Time.now + Shoryuken::Later::MAX_QUEUE_DELAY).to_i).first
+        client.first_item table_name, 'perform_at' => {
+                attribute_value_list: [ (Time.now + Shoryuken::Later::MAX_QUEUE_DELAY).to_i ],
+                comparison_operator:  'LT'
+              }
       end
     
       # Processes an item and enqueues it (unless another actor has already enqueued it).
       def process_item(item)
-        time, worker_class, args, id = item.attributes.values_at('perform_at','shoryuken_class','shoryuken_args','id')
+        time, worker_class, args, id = item.values_at('perform_at','shoryuken_class','shoryuken_args','id')
         
         worker_class = worker_class.constantize
         args = JSON.parse(args)
         time = Time.at(time)
-        queue_name = item.attributes['shoryuken_queue']
+        queue_name = item['shoryuken_queue']
         
         # Conditionally delete an item prior to enqueuing it, ensuring only one actor may enqueue it.
-        begin item.delete(:if => {id: id})
+        begin client.delete_item table_name, item
         rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
           # Item was already deleted, so it does not need to be queued.
           return
@@ -71,10 +74,12 @@ module Shoryuken
         # For compatibility with Shoryuken's ActiveJob adapter, support an explicit queue name.
         else
           delay = (time - Time.now).to_i
+          body = JSON.dump(body) if body.is_a? Hash
           options[:delay_seconds] = delay if delay > 0
+          options[:message_body] = body
           options[:message_attributes] ||= {}
           options[:message_attributes]['shoryuken_class'] = { string_value: worker_class.to_s, data_type: 'String' }
-          Shoryuken::Client.send_message(queue_name, body, options)
+          Shoryuken::Client.queues(queue_name).send_message(options)
         end
       end
 

data/lib/shoryuken/later/version.rb

@@ -1,5 +1,5 @@
 module Shoryuken
   module Later
-    VERSION = '0.0.6'
+    VERSION = '0.1.0'
   end
 end

data/lib/shoryuken/later/worker.rb

@@ -19,8 +19,8 @@ module Shoryuken
           else
             table = get_shoryuken_options['schedule_table'] || Shoryuken::Later.default_table
             args = JSON.dump(body: body, options: options)
-            Shoryuken::Later::Client.put_item(table, perform_at: time.to_i, shoryuken_args: args,
-                                                     shoryuken_class: self.to_s)
+            Shoryuken::Later::Client.create_item(table, perform_at: time.to_i, shoryuken_args: args,
+                                                        shoryuken_class: self.to_s)
           end
         end
       end

data/shoryuken-later.gemspec

@@ -27,7 +27,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec",   '~> 3.0', '< 3.1'
   spec.add_development_dependency "pry-byebug"
 
-  spec.add_dependency "aws-sdk-v1"
   spec.add_dependency "timers", "~> 4.0.1"
-  spec.add_dependency "shoryuken", "~> 0.0.5"
+  spec.add_dependency "aws-sdk-core", "2.0.21"
+  spec.add_dependency "aws-sdk-resources", "2.0.21.pre"
+  spec.add_dependency "shoryuken", "> 0.0.5"
 end

data/spec/shoryuken/later/client_spec.rb

@@ -1,42 +1,36 @@
 require 'spec_helper'
 
 describe Shoryuken::Later::Client do
-  let(:ddb)              { double 'DynamoDB' }
-  let(:table_collection) { double 'Table Collection' }
-  let(:ddb_table)        { double 'DynamoDb Table' }
-  let(:ddb_items)        { double 'Table Items' }
-  let(:table)            { 'shoryuken_later' }
+  let(:ddb)               { double 'DynamoDB' }
+  let(:table_description) { double 'Table Description' }
+  let(:table)             { 'shoryuken_later' }
 
   before do
     allow(described_class).to receive(:ddb).and_return(ddb)
-    allow(ddb).to receive(:tables).and_return(table_collection)
-    allow(table_collection).to receive(:[]).and_return(ddb_table)
-    allow(ddb_table).to receive(:items).and_return(ddb_items)
-    allow(ddb_table).to receive(:hash_key=)
+    allow(ddb).to receive(:describe_table).and_return(table_description)
   end
 
   describe '.tables' do
-    it 'memoizes tables and sets the hash_key' do
-      expect(table_collection).to receive(:[]).once.with(table).and_return(ddb_table)
-      expect(ddb_table).to receive(:hash_key=).once.with([:id, :string])
+    it 'memoizes tables' do
+      expect(ddb).to receive(:describe_table).once.with(table_name: table).and_return(table_description)
 
-      expect(described_class.tables(table)).to eq(ddb_table)
-      expect(described_class.tables(table)).to eq(ddb_table)
+      expect(described_class.tables(table)).to eq(table_description)
+      expect(described_class.tables(table)).to eq(table_description)
     end
   end
   
-  describe '.put_item' do
+  describe '.create_item' do
     it 'creates an item with a supplied ID' do
-      expect(ddb_items).to receive(:create).with({'id' => 'fubar'}, {unless_exists: 'id'})
+      expect(ddb).to receive(:put_item).with(table_name: table, item: {'id' => 'fubar'}, expected: {id: {exists: false}})
         
-      described_class.put_item(table,'id' => 'fubar')
+      described_class.create_item(table,'id' => 'fubar')
     end
     
     it 'creates an item with a auto-generated ID' do
       expect(SecureRandom).to receive(:uuid).once.and_return('fubar')
-      expect(ddb_items).to receive(:create).with({'id' => 'fubar', 'perform_at' => 1234}, {unless_exists: 'id'})
+      expect(ddb).to receive(:put_item).with(table_name: table, item: {'id' => 'fubar', 'perform_at' => 1234}, expected: {id: {exists: false}})
         
-      described_class.put_item(table,'perform_at' => 1234)
+      described_class.create_item(table,'perform_at' => 1234)
     end
   end
 end

data/spec/shoryuken/later/poller_spec.rb

@@ -2,20 +2,17 @@ require 'spec_helper'
 require 'shoryuken/later/poller'
 
 describe Shoryuken::Later::Poller do
-  let(:ddb_table) { double 'DynamoDb Table' }
-  let(:ddb_items) { double 'Table Items' }
-  let(:table)     { 'shoryuken_later' }
-  
+  let(:ddb)       { double 'DynamoDB' }
   let(:body)      { {'foo' => 'bar'} }  
   let(:json)      { JSON.dump(body: body, options: {}) }
+  let(:table)     { 'shoryuken_later' }
     
   let(:ddb_item)  do
-    double AWS::DynamoDB::Item, delete: nil,
-      attributes: {'id' => 'fubar', 'perform_at' => Time.now + 60, 'shoryuken_args' => json, 'shoryuken_class' => 'TestWorker'}
+    {'id' => 'fubar', 'perform_at' => Time.now + 60, 'shoryuken_args' => json, 'shoryuken_class' => 'TestWorker'}
   end
 
   before do
-    allow(Shoryuken::Later::Client).to receive(:tables).with(table).and_return(ddb_table)
+    allow(Shoryuken::Later::Client).to receive(:ddb).and_return(ddb)
   end
 
   subject do
@@ -42,6 +39,8 @@ describe Shoryuken::Later::Poller do
   
   describe '#process_item' do
     it 'enqueues a message if the item could be deleted' do
+      allow(Shoryuken::Later::Client).to receive(:delete_item).with(table, ddb_item)
+      
       expect(TestWorker).to receive(:perform_in).once do |time,body,options|
         expect(time   ).to be > Time.now
         expect(body   ).to eq(body)
@@ -53,7 +52,7 @@ describe Shoryuken::Later::Poller do
     
     it 'does not enqueue a message if the item could not be deleted' do
       expect(TestWorker).not_to receive(:perform_in)
-      allow(ddb_item).to receive(:delete) { raise AWS::DynamoDB::Errors::ConditionalCheckFailedException }
+      expect(Shoryuken::Later::Client).to receive(:delete_item).with(table, ddb_item){ raise AWS::DynamoDB::Errors::ConditionalCheckFailedException }
       
       subject.send(:process_item, ddb_item)
     end

data/spec/shoryuken/worker_spec.rb

@@ -21,7 +21,7 @@ describe 'Shoryuken::Worker' do
       json = JSON.dump(body: 'message', options: {})
       future = Time.now + 15 * 60 + 1
       
-      expect(Shoryuken::Later::Client).to receive(:put_item) do |table,attrs|
+      expect(Shoryuken::Later::Client).to receive(:create_item) do |table,attrs|
         expect(attrs[:perform_at]).to be >= future.to_i
         expect(attrs[:shoryuken_args]).to eq(json)
         expect(attrs[:shoryuken_class]).to eq('TestWorker')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken-later
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors:
 - Joe Khoobyar
@@ -73,45 +73,59 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-v1
+  name: timers
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.0.1
 - !ruby/object:Gem::Dependency
-  name: timers
+  name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: 2.0.21
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: 2.0.21
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-resources
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.21.pre
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.21.pre
 - !ruby/object:Gem::Dependency
   name: shoryuken
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>'
       - !ruby/object:Gem::Version
         version: 0.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>'
       - !ruby/object:Gem::Version
         version: 0.0.5
 description: "\n    This gem provides a scheduling plugin (using Dynamo DB) for Shoryuken,