toiler 0.5.1.pre9 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61292f1454428dae50787cb722201108250b136b
-  data.tar.gz: 004ab4f17d203ca40b4f4f74fac79ba52f0449cd
+  metadata.gz: 1defeb7538e17ae38a0028878f07de278ad6fc34
+  data.tar.gz: 8a233a42c1776663ce3dd04d5f22e318587de752
 SHA512:
-  metadata.gz: d9a8db8fcfd8c5986cf096a4d8f598bdef0a8a35f8da739fdecce50f6dc8976a822cd86f6f5caf8ad4d3d51257eb4fc5591dca29a8b91acc42978af8451711b5
-  data.tar.gz: 1c2552b393d6fde181b4d81fa80a971aa8f2076c44f3f556da2957a9453a5d13df8abaa0a1f90facc097a072592646033f4391682a12756001a5e38c736e2063
+  metadata.gz: bc096c97a2efca04256aaf8d7895d73bde781b2a6c961de35a0a188b7db14a92e0eaabb970040f37a8e1063b8de8801f3bae6fd6d38e4fb42937a9f9e9a06acb
+  data.tar.gz: 41dc4911884dc11ac9df2b1e55d438effda0c9a752b916699e568e206e3ac6fca5cae0d0551ac306f1aa35dc93f1fc7353e17a873f53516f502da8f38a131a8f

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    toiler (0.5.1.pre7)
-      aws-sdk-sqs (>= 1.0.0, < 2.0.0)
+    toiler (0.4.3)
+      aws-sdk-sqs (~> 1.0, >= 1.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.0)
       concurrent-ruby-edge (~> 0.3, >= 0.3)
 
@@ -11,14 +11,14 @@ GEM
   specs:
     ast (2.4.0)
     aws-eventstream (1.0.1)
-    aws-partitions (1.105.0)
-    aws-sdk-core (3.31.0)
+    aws-partitions (1.100.0)
+    aws-sdk-core (3.24.1)
       aws-eventstream (~> 1.0)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-sqs (1.7.0)
-      aws-sdk-core (~> 3, >= 3.26.0)
+    aws-sdk-sqs (1.4.0)
+      aws-sdk-core (~> 3)
       aws-sigv4 (~> 1.0)
     aws-sigv4 (1.0.3)
     concurrent-ruby (1.0.5)

data/lib/toiler/actor/fetcher.rb

@@ -10,38 +10,49 @@ module Toiler
       FETCH_LIMIT = 10
 
       attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
-                    :executing, :waiting_messages, :concurrency
+                    :scheduled, :executing, :polling
 
-      def initialize(queue, client, count)
+      def initialize(queue, client)
         debug "Initializing Fetcher for queue #{queue}..."
         @queue = Toiler::Aws::Queue.new queue, client
-        @wait = Toiler.options[:wait] || 60
-        @free_processors = count
+        @wait = Toiler.options[:wait] || 20
+        @free_processors = Concurrent::AtomicFixnum.new(0)
         @batch = Toiler.worker_class_registry[queue].batch?
         @visibility_timeout = @queue.visibility_timeout
-        @executing = false
-        @waiting_messages = 0
-        @concurrency = count
+        @scheduled = Concurrent::AtomicBoolean.new
+        @executing = Concurrent::AtomicBoolean.new
+        @polling = Concurrent::AtomicBoolean.new
         debug "Finished initializing Fetcher for queue #{queue}"
-        tell :poll_messages
       end
 
       def default_executor
-        Concurrent.global_fast_executor
+        Concurrent.global_io_executor
       end
 
       def on_message(msg)
-        @executing = true
+        executing.make_true
         method, *args = msg
         send(method, *args)
       rescue StandardError => e
         error "Fetcher #{queue.name} raised exception #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
       ensure
-        @executing = false
+        executing.make_false
       end
 
       def executing?
-        @executing
+        executing.value
+      end
+
+      def polling?
+        polling.value
+      end
+
+      def scheduled?
+        scheduled.value
+      end
+
+      def get_free_processors
+        free_processors.value
       end
 
       private
@@ -52,55 +63,46 @@ module Toiler
 
       def processor_finished
         debug "Fetcher #{queue.name} received processor finished signal..."
-        @free_processors += 1
-        tell :poll_messages
+        free_processors.increment
+        schedule_poll
       end
 
       def max_messages
-        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors].min
+        batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
       end
 
-      def poll_future(max_number_of_messages)
+      def poll_future
+
         Concurrent::Promises.future do
-          queue.receive_messages message_attribute_names: %w[All],
+          queue.receive_messages attribute_names: %w(All),
+                                 message_attribute_names: %w(All),
                                  wait_time_seconds: wait,
-                                 max_number_of_messages: max_number_of_messages
+                                 max_number_of_messages: max_messages
         end
       end
 
-      def release_messages(messages)
-        @waiting_messages -= messages
-      end
-
       def poll_messages
-        return unless should_poll?
-
-        max_number_of_messages = max_messages
-        return if waiting_messages > 0 && !full_batch?(max_number_of_messages)
-
-        @waiting_messages += max_number_of_messages
-
-        debug "Fetcher #{queue.name} polling messages..."
-        future = poll_future max_number_of_messages
-        future.on_rejection! do
-          tell [:release_messages, max_number_of_messages]
-          tell :poll_messages
+        return unless polling.make_true
+        poll_future.on_rejection! do
+          polling.make_false
+          scheduled.make_false
+          tell :schedule_poll
         end
-        future.on_fulfillment! do |msgs|
-          tell [:assign_messages, msgs] if !msgs.nil? && !msgs.empty?
-          tell [:release_messages, max_number_of_messages]
-          tell :poll_messages
+        poll_future.on_fulfillment! do |msgs|
+          polling.make_false
+          scheduled.make_false
+          if !msgs.nil? && !msgs.empty?
+            tell [:assign_messages, msgs]
+          else
+            tell :schedule_poll
+          end
         end
-
-        poll_messages if should_poll?
-      end
-
-      def should_poll?
-        free_processors / 2 > waiting_messages
       end
 
-      def full_batch?(max_number_of_messages)
-        max_number_of_messages == FETCH_LIMIT || max_number_of_messages >= concurrency * 0.1
+      def schedule_poll
+        return unless free_processors.value > 0 && scheduled.make_true
+        debug "Fetcher #{queue.name} scheduling polling..."
+        tell :poll_messages
       end
 
       def processor_pool
@@ -111,9 +113,10 @@ module Toiler
         messages = [messages] if batch?
         messages.each do |m|
           processor_pool.tell [:process, visibility_timeout, m]
-          @free_processors -= 1
+          free_processors.decrement
         end
         debug "Fetcher #{queue.name} assigned #{messages.count} messages"
+        tell :schedule_poll
       end
     end
   end

data/lib/toiler/actor/processor.rb

@@ -13,19 +13,17 @@ module Toiler
       def initialize(queue)
         @queue = queue
         @worker_class = Toiler.worker_class_registry[queue]
+        @fetcher = Toiler.fetcher queue
         @executing = Concurrent::AtomicBoolean.new
         @thread = nil
         init_options
+        processor_finished
       end
 
       def default_executor
         Concurrent.global_io_executor
       end
 
-      def fetcher
-        @fetcher ||= Toiler.fetcher queue
-      end
-
       def on_message(msg)
         method, *args = msg
         send(method, *args)
@@ -91,16 +89,14 @@ module Toiler
 
       def visibility_extender(queue_visibility, sqs_msg, body)
         return unless auto_visibility_timeout?
-
-        interval = [1, queue_visibility / 3].max
+        interval = [1,queue_visibility/3].max
         Concurrent::TimerTask.execute execution_interval: interval,
-                                      timeout_interval: interval do |task|
+                                      timeout_interval: interval do
           begin
             sqs_msg.visibility_timeout = queue_visibility
             yield sqs_msg, body if block_given?
           rescue StandardError => e
-            error "Processor #{queue} failed to extend visibility of message - #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
-            task.shutdown if e.message.include?('ReceiptHandle is invalid')
+            error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
           end
         end
       end
@@ -120,7 +116,7 @@ module Toiler
         when :text, nil then sqs_msg.body
         else body_parser.load sqs_msg.body
         end
-      rescue StandardError => e
+      rescue => e
         raise "Error parsing the message body: #{e.message}"
       end
     end

data/lib/toiler/actor/supervisor.rb

@@ -11,8 +11,8 @@ module Toiler
 
       def initialize
         @client = ::Aws::SQS::Client.new
-        spawn_processors
         spawn_fetchers
+        spawn_processors
       end
 
       def on_message(_msg)
@@ -20,12 +20,10 @@ module Toiler
       end
 
       def spawn_fetchers
-        Toiler.active_worker_class_registry.each do |queue, klass|
-          count = klass.concurrency
+        Toiler.active_worker_class_registry.each do |queue, _klass|
           begin
             fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
-                                            supervise: true,
-                                            args: [queue, client, count]
+                                            supervise: true, args: [queue, client]
             Toiler.set_fetcher queue, fetcher
           rescue StandardError => e
             error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
@@ -39,9 +37,8 @@ module Toiler
           count = klass.concurrency
           begin
             pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
-              Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
-                                     supervise: true,
-                                     args: [queue]
+                Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
+                                       supervise: true, args: [queue]
             end
             Toiler.set_processor_pool queue, pool
           rescue StandardError => e

data/lib/toiler/cli.rb

@@ -99,7 +99,7 @@ module Toiler
         processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
         busy_processors = processors.count{|pr| pr.executing?}
         message = "Status for [queue:#{queue}]:"
-        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [waiting_messages:#{fetcher.waiting_messages}] [free_processors:#{fetcher.free_processors}]"
+        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [polling:#{fetcher.polling?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
         message += "\n[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
         processors.each do |processor|
           thread = processor.thread

data/lib/toiler/version.rb

@@ -1,4 +1,4 @@
 # Toiler Version
 module Toiler
-  VERSION = '0.5.1.pre9'.freeze
+  VERSION = '0.5.1'.freeze
 end

data/spec/models/fetcher_spec.rb

@@ -7,16 +7,16 @@ RSpec.describe Toiler::Actor::Fetcher, type: :model do
 
   before do
     allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:log).and_return(true)
-    allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:tell)
     allow_any_instance_of(Toiler::Aws::Queue).to receive(:visibility_timeout).and_return(100)
     allow(client).to receive(:get_queue_url).with(queue_name: 'default').and_return double(:queue, queue_url: 'http://aws.fake/queue')
   end
 
   describe "#new" do
     it 'completes sucessfully' do
-      fetcher = described_class.new(queue, client, 1)
-      expect(fetcher).to have_received(:tell).with(:poll_messages)
+      fetcher = described_class.new(queue, client)
+      expect(fetcher.polling?).to eq(false)
       expect(fetcher.executing?).to eq(false)
+      expect(fetcher.scheduled?).to eq(false)
     end
   end
 end

data/spec/models/processor_spec.rb

@@ -7,6 +7,7 @@ RSpec.describe Toiler::Actor::Processor, type: :model do
   describe "#new" do
     it 'initializes properly' do
       allow(Toiler).to receive(:fetcher).and_return(fetcher)
+      expect(fetcher).to receive(:tell).with(:processor_finished)
       processor = described_class.new('default')
       expect(processor.executing?).to eq(false)
     end

data/spec/models/supervisor_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Toiler::Actor::Supervisor, type: :model do
 
       Toiler.options.merge!(active_queues: ['default'])
       expect(::Aws::SQS::Client).to receive(:new).and_return(sqs_client)
-      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client, 1])
+      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client])
       expect(Concurrent::Actor::Utils::Pool).to receive(:spawn!).with(:processor_pool_default, 1)
       supervisor = described_class.new
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.5.1.pre9
+  version: 0.5.1
 platform: ruby
 authors:
 - Sebastian Schepens
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-25 00:00:00.000000000 Z
+date: 2018-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -136,12 +136,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.5.2.1
 signing_key: 
 specification_version: 4
 summary: Toiler is a super efficient AWS SQS thread based message processor