toiler 0.5.1 → 0.6.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1defeb7538e17ae38a0028878f07de278ad6fc34
-  data.tar.gz: 8a233a42c1776663ce3dd04d5f22e318587de752
+  metadata.gz: f3b78eabd0468862d06d5a6755a6cef9abed5200
+  data.tar.gz: 75bcd6cf425bab5ab8bf021bb6dd9c59bc2a9a2c
 SHA512:
-  metadata.gz: bc096c97a2efca04256aaf8d7895d73bde781b2a6c961de35a0a188b7db14a92e0eaabb970040f37a8e1063b8de8801f3bae6fd6d38e4fb42937a9f9e9a06acb
-  data.tar.gz: 41dc4911884dc11ac9df2b1e55d438effda0c9a752b916699e568e206e3ac6fca5cae0d0551ac306f1aa35dc93f1fc7353e17a873f53516f502da8f38a131a8f
+  metadata.gz: 3b710c1dbaf98523837a27eca2cd2ff4a35745846753828e5286cf3830ae066bf08344680bc741cd5dbe30f53a3a7170a55a13ebc8e861dd47c6afc8e2715299
+  data.tar.gz: f11cb35378dac421c0221f7e1ca37a562b31ff89ab9a1ca5bc5de6ed043c0ed234b4eed4447e1c938d07fbf241bd70f59f25d6376e24d9d16607d30110efa86c

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    toiler (0.4.3)
-      aws-sdk-sqs (~> 1.0, >= 1.0.0)
+    toiler (0.5.1.pre7)
+      aws-sdk-sqs (>= 1.0.0, < 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.0)
       concurrent-ruby-edge (~> 0.3, >= 0.3)
 
@@ -11,14 +11,14 @@ GEM
   specs:
     ast (2.4.0)
     aws-eventstream (1.0.1)
-    aws-partitions (1.100.0)
-    aws-sdk-core (3.24.1)
+    aws-partitions (1.105.0)
+    aws-sdk-core (3.31.0)
       aws-eventstream (~> 1.0)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-sqs (1.4.0)
-      aws-sdk-core (~> 3)
+    aws-sdk-sqs (1.7.0)
+      aws-sdk-core (~> 3, >= 3.26.0)
       aws-sigv4 (~> 1.0)
     aws-sigv4 (1.0.3)
     concurrent-ruby (1.0.5)

data/lib/toiler/actor/fetcher.rb

@@ -10,49 +10,38 @@ module Toiler
       FETCH_LIMIT = 10
 
       attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
-                    :scheduled, :executing, :polling
+                    :executing, :waiting_messages, :concurrency
 
-      def initialize(queue, client)
+      def initialize(queue, client, count)
         debug "Initializing Fetcher for queue #{queue}..."
         @queue = Toiler::Aws::Queue.new queue, client
-        @wait = Toiler.options[:wait] || 20
-        @free_processors = Concurrent::AtomicFixnum.new(0)
+        @wait = Toiler.options[:wait] || 60
+        @free_processors = count
         @batch = Toiler.worker_class_registry[queue].batch?
         @visibility_timeout = @queue.visibility_timeout
-        @scheduled = Concurrent::AtomicBoolean.new
-        @executing = Concurrent::AtomicBoolean.new
-        @polling = Concurrent::AtomicBoolean.new
+        @executing = false
+        @waiting_messages = 0
+        @concurrency = count
         debug "Finished initializing Fetcher for queue #{queue}"
+        tell :poll_messages
       end
 
       def default_executor
-        Concurrent.global_io_executor
+        Concurrent.global_fast_executor
       end
 
       def on_message(msg)
-        executing.make_true
+        @executing = true
         method, *args = msg
         send(method, *args)
       rescue StandardError => e
         error "Fetcher #{queue.name} raised exception #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
       ensure
-        executing.make_false
+        @executing = false
       end
 
       def executing?
-        executing.value
-      end
-
-      def polling?
-        polling.value
-      end
-
-      def scheduled?
-        scheduled.value
-      end
-
-      def get_free_processors
-        free_processors.value
+        @executing
       end
 
       private
@@ -63,46 +52,56 @@ module Toiler
 
       def processor_finished
         debug "Fetcher #{queue.name} received processor finished signal..."
-        free_processors.increment
-        schedule_poll
+        @free_processors += 1
+        tell :poll_messages
       end
 
       def max_messages
-        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
+        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors].min
       end
 
-      def poll_future
-
+      def poll_future(max_number_of_messages)
         Concurrent::Promises.future do
-          queue.receive_messages attribute_names: %w(All),
-                                 message_attribute_names: %w(All),
+          queue.receive_messages attribute_names: %w[All],
+                                 message_attribute_names: %w[All],
                                  wait_time_seconds: wait,
-                                 max_number_of_messages: max_messages
+                                 max_number_of_messages: max_number_of_messages
         end
       end
 
+      def release_messages(messages)
+        @waiting_messages -= messages
+      end
+
       def poll_messages
-        return unless polling.make_true
-        poll_future.on_rejection! do
-          polling.make_false
-          scheduled.make_false
-          tell :schedule_poll
+        return unless should_poll?
+
+        max_number_of_messages = max_messages
+        return if waiting_messages > 0 && !full_batch?(max_number_of_messages)
+
+        @waiting_messages += max_number_of_messages
+
+        debug "Fetcher #{queue.name} polling messages..."
+        future = poll_future max_number_of_messages
+        future.on_rejection! do
+          tell [:release_messages, max_number_of_messages]
+          tell :poll_messages
         end
-        poll_future.on_fulfillment! do |msgs|
-          polling.make_false
-          scheduled.make_false
-          if !msgs.nil? && !msgs.empty?
-            tell [:assign_messages, msgs]
-          else
-            tell :schedule_poll
-          end
+        future.on_fulfillment! do |msgs|
+          tell [:assign_messages, msgs] if !msgs.nil? && !msgs.empty?
+          tell [:release_messages, max_number_of_messages]
+          tell :poll_messages
         end
+
+        poll_messages if should_poll?
       end
 
-      def schedule_poll
-        return unless free_processors.value > 0 && scheduled.make_true
-        debug "Fetcher #{queue.name} scheduling polling..."
-        tell :poll_messages
+      def should_poll?
+        free_processors / 2 > waiting_messages
+      end
+
+      def full_batch?(max_number_of_messages)
+        max_number_of_messages == FETCH_LIMIT || max_number_of_messages >= concurrency * 0.1
       end
 
       def processor_pool
@@ -113,10 +112,9 @@ module Toiler
         messages = [messages] if batch?
         messages.each do |m|
           processor_pool.tell [:process, visibility_timeout, m]
-          free_processors.decrement
+          @free_processors -= 1
         end
         debug "Fetcher #{queue.name} assigned #{messages.count} messages"
-        tell :schedule_poll
       end
     end
   end

data/lib/toiler/actor/processor.rb

@@ -13,17 +13,19 @@ module Toiler
       def initialize(queue)
         @queue = queue
         @worker_class = Toiler.worker_class_registry[queue]
-        @fetcher = Toiler.fetcher queue
         @executing = Concurrent::AtomicBoolean.new
         @thread = nil
         init_options
-        processor_finished
       end
 
       def default_executor
         Concurrent.global_io_executor
       end
 
+      def fetcher
+        @fetcher ||= Toiler.fetcher queue
+      end
+
       def on_message(msg)
         method, *args = msg
         send(method, *args)
@@ -89,14 +91,16 @@ module Toiler
 
       def visibility_extender(queue_visibility, sqs_msg, body)
         return unless auto_visibility_timeout?
-        interval = [1,queue_visibility/3].max
+
+        interval = [1, queue_visibility / 3].max
         Concurrent::TimerTask.execute execution_interval: interval,
-                                      timeout_interval: interval do
+                                      timeout_interval: interval do |task|
           begin
             sqs_msg.visibility_timeout = queue_visibility
             yield sqs_msg, body if block_given?
           rescue StandardError => e
-            error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
+            error "Processor #{queue} failed to extend visibility of message - #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+            task.shutdown if e.message.include?('ReceiptHandle is invalid')
           end
         end
       end
@@ -116,7 +120,7 @@ module Toiler
         when :text, nil then sqs_msg.body
         else body_parser.load sqs_msg.body
         end
-      rescue => e
+      rescue StandardError => e
         raise "Error parsing the message body: #{e.message}"
       end
     end

data/lib/toiler/actor/supervisor.rb

@@ -11,8 +11,8 @@ module Toiler
 
       def initialize
         @client = ::Aws::SQS::Client.new
-        spawn_fetchers
         spawn_processors
+        spawn_fetchers
       end
 
       def on_message(_msg)
@@ -20,10 +20,12 @@ module Toiler
       end
 
       def spawn_fetchers
-        Toiler.active_worker_class_registry.each do |queue, _klass|
+        Toiler.active_worker_class_registry.each do |queue, klass|
+          count = klass.concurrency
           begin
             fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
-                                            supervise: true, args: [queue, client]
+                                            supervise: true,
+                                            args: [queue, client, count]
             Toiler.set_fetcher queue, fetcher
           rescue StandardError => e
             error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
@@ -37,8 +39,9 @@ module Toiler
           count = klass.concurrency
           begin
             pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
-                Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
-                                       supervise: true, args: [queue]
+              Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
+                                     supervise: true,
+                                     args: [queue]
             end
             Toiler.set_processor_pool queue, pool
           rescue StandardError => e

data/lib/toiler/cli.rb

@@ -99,7 +99,7 @@ module Toiler
         processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
         busy_processors = processors.count{|pr| pr.executing?}
         message = "Status for [queue:#{queue}]:"
-        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [polling:#{fetcher.polling?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
+        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [waiting_messages:#{fetcher.waiting_messages}] [free_processors:#{fetcher.free_processors}]"
         message += "\n[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
         processors.each do |processor|
           thread = processor.thread

data/lib/toiler/version.rb

@@ -1,4 +1,4 @@
 # Toiler Version
 module Toiler
-  VERSION = '0.5.1'.freeze
+  VERSION = '0.6.0.pre1'.freeze
 end

data/spec/models/fetcher_spec.rb

@@ -7,16 +7,16 @@ RSpec.describe Toiler::Actor::Fetcher, type: :model do
 
   before do
     allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:log).and_return(true)
+    allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:tell)
     allow_any_instance_of(Toiler::Aws::Queue).to receive(:visibility_timeout).and_return(100)
     allow(client).to receive(:get_queue_url).with(queue_name: 'default').and_return double(:queue, queue_url: 'http://aws.fake/queue')
   end
 
   describe "#new" do
     it 'completes sucessfully' do
-      fetcher = described_class.new(queue, client)
-      expect(fetcher.polling?).to eq(false)
+      fetcher = described_class.new(queue, client, 1)
+      expect(fetcher).to have_received(:tell).with(:poll_messages)
       expect(fetcher.executing?).to eq(false)
-      expect(fetcher.scheduled?).to eq(false)
     end
   end
 end

data/spec/models/processor_spec.rb

@@ -7,7 +7,6 @@ RSpec.describe Toiler::Actor::Processor, type: :model do
   describe "#new" do
     it 'initializes properly' do
       allow(Toiler).to receive(:fetcher).and_return(fetcher)
-      expect(fetcher).to receive(:tell).with(:processor_finished)
       processor = described_class.new('default')
       expect(processor.executing?).to eq(false)
     end

data/spec/models/supervisor_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Toiler::Actor::Supervisor, type: :model do
 
       Toiler.options.merge!(active_queues: ['default'])
       expect(::Aws::SQS::Client).to receive(:new).and_return(sqs_client)
-      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client])
+      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client, 1])
       expect(Concurrent::Actor::Utils::Pool).to receive(:spawn!).with(:processor_pool_default, 1)
       supervisor = described_class.new
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0.pre1
 platform: ruby
 authors:
 - Sebastian Schepens
@@ -136,9 +136,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.2.1