toiler 0.5.1.pre7 → 0.5.1.pre9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bef400a1c11a068f9e486a84e02e1afb30fa806
-  data.tar.gz: 768c913f357907d24c0e077cc03a417dfeb7a638
+  metadata.gz: 61292f1454428dae50787cb722201108250b136b
+  data.tar.gz: 004ab4f17d203ca40b4f4f74fac79ba52f0449cd
 SHA512:
-  metadata.gz: 257dc77e9369c11d79b6dfeddb06d5c314d700732d5774b93c1ea13dd49653147199426078be651df506df85870ded3df7fbd5146894e27484232b8eab895a3a
-  data.tar.gz: b72690b4d28ce55a79eef6f022b7bcd077b383d0c490d3f86ac477d353c2a94bf7fcd170b072e17c94014ceeb83784d837f8997b7647d4e68fb79b329056b1d6
+  metadata.gz: d9a8db8fcfd8c5986cf096a4d8f598bdef0a8a35f8da739fdecce50f6dc8976a822cd86f6f5caf8ad4d3d51257eb4fc5591dca29a8b91acc42978af8451711b5
+  data.tar.gz: 1c2552b393d6fde181b4d81fa80a971aa8f2076c44f3f556da2957a9453a5d13df8abaa0a1f90facc097a072592646033f4391682a12756001a5e38c736e2063

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    toiler (0.4.3)
-      aws-sdk-sqs (~> 1.0, >= 1.0.0)
+    toiler (0.5.1.pre7)
+      aws-sdk-sqs (>= 1.0.0, < 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.0)
       concurrent-ruby-edge (~> 0.3, >= 0.3)
 
@@ -11,14 +11,14 @@ GEM
   specs:
     ast (2.4.0)
     aws-eventstream (1.0.1)
-    aws-partitions (1.100.0)
-    aws-sdk-core (3.24.1)
+    aws-partitions (1.105.0)
+    aws-sdk-core (3.31.0)
       aws-eventstream (~> 1.0)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-sqs (1.4.0)
-      aws-sdk-core (~> 3)
+    aws-sdk-sqs (1.7.0)
+      aws-sdk-core (~> 3, >= 3.26.0)
       aws-sigv4 (~> 1.0)
     aws-sigv4 (1.0.3)
     concurrent-ruby (1.0.5)

data/lib/toiler/actor/fetcher.rb

@@ -10,7 +10,7 @@ module Toiler
       FETCH_LIMIT = 10
 
       attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
-                    :executing, :waiting_messages
+                    :executing, :waiting_messages, :concurrency
 
       def initialize(queue, client, count)
         debug "Initializing Fetcher for queue #{queue}..."
@@ -21,6 +21,7 @@ module Toiler
         @visibility_timeout = @queue.visibility_timeout
         @executing = false
         @waiting_messages = 0
+        @concurrency = count
         debug "Finished initializing Fetcher for queue #{queue}"
         tell :poll_messages
       end
@@ -75,6 +76,8 @@ module Toiler
         return unless should_poll?
 
         max_number_of_messages = max_messages
+        return if waiting_messages > 0 && !full_batch?(max_number_of_messages)
+
         @waiting_messages += max_number_of_messages
 
         debug "Fetcher #{queue.name} polling messages..."
@@ -89,13 +92,17 @@ module Toiler
           tell :poll_messages
         end
 
-        tell :poll_messages if should_poll?
+        poll_messages if should_poll?
       end
 
       def should_poll?
         free_processors / 2 > waiting_messages
       end
 
+      def full_batch?(max_number_of_messages)
+        max_number_of_messages == FETCH_LIMIT || max_number_of_messages >= concurrency * 0.1
+      end
+
       def processor_pool
         @processor_pool ||= Toiler.processor_pool queue.name
       end

data/lib/toiler/actor/processor.rb

@@ -94,12 +94,13 @@ module Toiler
 
         interval = [1, queue_visibility / 3].max
         Concurrent::TimerTask.execute execution_interval: interval,
-                                      timeout_interval: interval do
+                                      timeout_interval: interval do |task|
           begin
             sqs_msg.visibility_timeout = queue_visibility
             yield sqs_msg, body if block_given?
           rescue StandardError => e
-            error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
+            error "Processor #{queue} failed to extend visibility of message - #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+            task.shutdown if e.message.include?('ReceiptHandle is invalid')
           end
         end
       end

data/lib/toiler/version.rb

@@ -1,4 +1,4 @@
 # Toiler Version
 module Toiler
-  VERSION = '0.5.1.pre7'.freeze
+  VERSION = '0.5.1.pre9'.freeze
 end

data/spec/models/fetcher_spec.rb

@@ -7,13 +7,15 @@ RSpec.describe Toiler::Actor::Fetcher, type: :model do
 
   before do
     allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:log).and_return(true)
+    allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:tell)
     allow_any_instance_of(Toiler::Aws::Queue).to receive(:visibility_timeout).and_return(100)
     allow(client).to receive(:get_queue_url).with(queue_name: 'default').and_return double(:queue, queue_url: 'http://aws.fake/queue')
   end
 
   describe "#new" do
     it 'completes sucessfully' do
-      fetcher = described_class.new(queue, client)
+      fetcher = described_class.new(queue, client, 1)
+      expect(fetcher).to have_received(:tell).with(:poll_messages)
       expect(fetcher.executing?).to eq(false)
     end
   end

data/spec/models/processor_spec.rb

@@ -7,7 +7,6 @@ RSpec.describe Toiler::Actor::Processor, type: :model do
   describe "#new" do
     it 'initializes properly' do
       allow(Toiler).to receive(:fetcher).and_return(fetcher)
-      expect(fetcher).to receive(:tell).with(:processor_finished)
       processor = described_class.new('default')
       expect(processor.executing?).to eq(false)
     end

data/spec/models/supervisor_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Toiler::Actor::Supervisor, type: :model do
 
       Toiler.options.merge!(active_queues: ['default'])
       expect(::Aws::SQS::Client).to receive(:new).and_return(sqs_client)
-      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client])
+      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client, 1])
       expect(Concurrent::Actor::Utils::Pool).to receive(:spawn!).with(:processor_pool_default, 1)
       supervisor = described_class.new
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.5.1.pre7
+  version: 0.5.1.pre9
 platform: ruby
 authors:
 - Sebastian Schepens
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-17 00:00:00.000000000 Z
+date: 2018-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec