toiler 0.5.0 → 0.5.1.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5abd63f186c193943169cb25c4f29498dcf0b7cb
-  data.tar.gz: 1d781bbfb99012e0f3af32ad8caac69fb00e5c10
+  metadata.gz: 3efc28d7e6e93a72eec97541769224ae90bf88aa
+  data.tar.gz: 48c91cc61b0c9624848d3c6b2a5139ca2916566a
 SHA512:
-  metadata.gz: f1c7cacc2c8ba06c9863d056550a36b6429a2c3422e394b14284e0bd63d06d85fbad6976a485c9e3cdde06ef86310a8aba6a5eb3145b99a1197db56eaa06417b
-  data.tar.gz: a1e41500cebe89ef547e86a23f7e647e04dd8704f8808f1ccf8197bf7ccb2423d21172052e60259bcbc0bf980fdef8ba474ba2fb3f5aa2e0520ddd31b77acd05
+  metadata.gz: da792a91216f97a6c7cbee7becc4352d5c9c49a8ecc21cab178adebcb21991a791d72b2ddd93aad434f3533f12ba49e4919f7647c5e8ed1b827802212422a182
+  data.tar.gz: 6d21aed25d630a51f64d61f0520750ca3eba9849277a97f8828cc79830650146de9d90f097d0f26c0abaaf92f1bba368d71fffb5436c8003c7f1ee2758d7834f

data/lib/toiler/actor/fetcher.rb

@@ -10,49 +10,37 @@ module Toiler
       FETCH_LIMIT = 10
 
       attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
-                    :scheduled, :executing, :polling
+                    :executing, :waiting_messages
 
-      def initialize(queue, client)
+      def initialize(queue, client, count)
         debug "Initializing Fetcher for queue #{queue}..."
         @queue = Toiler::Aws::Queue.new queue, client
-        @wait = Toiler.options[:wait] || 20
-        @free_processors = Concurrent::AtomicFixnum.new(0)
+        @wait = Toiler.options[:wait] || 60
+        @free_processors = count
         @batch = Toiler.worker_class_registry[queue].batch?
         @visibility_timeout = @queue.visibility_timeout
-        @scheduled = Concurrent::AtomicBoolean.new
-        @executing = Concurrent::AtomicBoolean.new
-        @polling = Concurrent::AtomicBoolean.new
+        @executing = false
+        @waiting_messages = 0
         debug "Finished initializing Fetcher for queue #{queue}"
+        tell :poll_messages
       end
 
       def default_executor
-        Concurrent.global_io_executor
+        Concurrent.global_fast_executor
       end
 
       def on_message(msg)
-        executing.make_true
+        @executing = true
         method, *args = msg
         send(method, *args)
       rescue StandardError => e
         error "Fetcher #{queue.name} raised exception #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
       ensure
-        executing.make_false
+        @executing = false
       end
 
       def executing?
-        executing.value
-      end
-
-      def polling?
-        polling.value
-      end
-
-      def scheduled?
-        scheduled.value
-      end
-
-      def get_free_processors
-        free_processors.value
+        @executing
       end
 
       private
@@ -63,46 +51,39 @@ module Toiler
 
       def processor_finished
         debug "Fetcher #{queue.name} received processor finished signal..."
-        free_processors.increment
-        schedule_poll
+        @free_processors += 1
+        tell :poll_messages
       end
 
       def max_messages
-        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
+        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors].min
       end
 
-      def poll_future
+      def poll_future(max_number_of_messages)
         Concurrent::Promises.future do
-          queue.receive_messages message_attribute_names: %w(All),
+          queue.receive_messages message_attribute_names: %w[All],
                                  wait_time_seconds: wait,
-                                 max_number_of_messages: max_messages
+                                 max_number_of_messages: max_number_of_messages
         end
       end
 
       def poll_messages
-        return unless polling.make_true
-        poll_future.on_rejection! do
-          polling.make_false
-          scheduled.make_false
-          tell :schedule_poll
+        return unless free_processors / 2 > waiting_messages
+
+        max_number_of_messages = max_messages
+        @waiting_messages += max_number_of_messages
+
+        debug "Fetcher #{queue.name} polling messages..."
+        future = poll_future max_number_of_messages
+        future.on_rejection! do
+          tell :poll_messages
         end
-        poll_future.on_fulfillment! do |msgs|
-          polling.make_false
-          scheduled.make_false
-          if !msgs.nil? && !msgs.empty?
-            tell [:assign_messages, msgs]
-          else
-            tell :schedule_poll
-          end
+        future.on_fulfillment! do |msgs|
+          tell [:assign_messages, msgs] if !msgs.nil? && !msgs.empty?
+          tell :poll_messages
         end
       end
 
-      def schedule_poll
-        return unless free_processors.value > 0 && scheduled.make_true
-        debug "Fetcher #{queue.name} scheduling polling..."
-        tell :poll_messages
-      end
-
       def processor_pool
         @processor_pool ||= Toiler.processor_pool queue.name
       end
@@ -111,10 +92,9 @@ module Toiler
         messages = [messages] if batch?
         messages.each do |m|
           processor_pool.tell [:process, visibility_timeout, m]
-          free_processors.decrement
+          @free_processors -= 1
         end
         debug "Fetcher #{queue.name} assigned #{messages.count} messages"
-        tell :schedule_poll
       end
     end
   end

data/lib/toiler/actor/processor.rb

@@ -17,7 +17,6 @@ module Toiler
         @executing = Concurrent::AtomicBoolean.new
         @thread = nil
         init_options
-        processor_finished
       end
 
       def default_executor
@@ -89,7 +88,8 @@ module Toiler
 
       def visibility_extender(queue_visibility, sqs_msg, body)
         return unless auto_visibility_timeout?
-        interval = [1,queue_visibility/3].max
+
+        interval = [1, queue_visibility / 3].max
         Concurrent::TimerTask.execute execution_interval: interval,
                                       timeout_interval: interval do
           begin
@@ -116,7 +116,7 @@ module Toiler
         when :text, nil then sqs_msg.body
         else body_parser.load sqs_msg.body
         end
-      rescue => e
+      rescue StandardError => e
         raise "Error parsing the message body: #{e.message}"
       end
     end

data/lib/toiler/actor/supervisor.rb

@@ -11,8 +11,8 @@ module Toiler
 
       def initialize
         @client = ::Aws::SQS::Client.new
-        spawn_fetchers
         spawn_processors
+        spawn_fetchers
       end
 
       def on_message(_msg)
@@ -20,10 +20,12 @@ module Toiler
       end
 
       def spawn_fetchers
-        Toiler.active_worker_class_registry.each do |queue, _klass|
+        Toiler.active_worker_class_registry.each do |queue, klass|
+          count = klass.concurrency
           begin
             fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
-                                            supervise: true, args: [queue, client]
+                                            supervise: true,
+                                            args: [queue, client, count]
             Toiler.set_fetcher queue, fetcher
           rescue StandardError => e
             error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
@@ -37,8 +39,9 @@ module Toiler
           count = klass.concurrency
           begin
             pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
-                Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
-                                       supervise: true, args: [queue]
+              Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
+                                     supervise: true,
+                                     args: [queue]
             end
             Toiler.set_processor_pool queue, pool
           rescue StandardError => e

data/lib/toiler/cli.rb

@@ -99,7 +99,7 @@ module Toiler
         processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
         busy_processors = processors.count{|pr| pr.executing?}
         message = "Status for [queue:#{queue}]:"
-        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [polling:#{fetcher.polling?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
+        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [waiting_messages:#{fetcher.waiting_messages}] [free_processors:#{fetcher.free_processors}]"
         message += "\n[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
         processors.each do |processor|
           thread = processor.thread

data/lib/toiler/version.rb

@@ -1,4 +1,4 @@
 # Toiler Version
 module Toiler
-  VERSION = '0.5.0'.freeze
+  VERSION = '0.5.1.pre1'.freeze
 end

data/spec/models/fetcher_spec.rb

@@ -14,9 +14,7 @@ RSpec.describe Toiler::Actor::Fetcher, type: :model do
   describe "#new" do
     it 'completes sucessfully' do
       fetcher = described_class.new(queue, client)
-      expect(fetcher.polling?).to eq(false)
       expect(fetcher.executing?).to eq(false)
-      expect(fetcher.scheduled?).to eq(false)
     end
   end
 end

data/toiler.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'rspec'
 
-  spec.add_dependency 'aws-sdk-sqs', '~> 1.0', '>= 1.0.0'
+  spec.add_dependency 'aws-sdk-sqs', '>= 1.0.0', '< 2.0.0'
   spec.add_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.0'
   spec.add_dependency 'concurrent-ruby-edge', '~> 0.3', '>= 0.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1.pre1
 platform: ruby
 authors:
 - Sebastian Schepens
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-24 00:00:00.000000000 Z
+date: 2018-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -28,22 +28,22 @@ dependencies:
   name: aws-sdk-sqs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -136,9 +136,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.11