eventboss 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcfcef49dffcc6459e8138b6b2aa2c29ebbf78e9fb186b747c2e56a92750d0d1
-  data.tar.gz: faba8a61381425fd43c9314aa863f937b1c9746b6ff5104a24c3c2a260b0aab6
+  metadata.gz: 63a1b730db853663d1a26cc59b1935a8176fb9aa0bde0be6e5761499a8127109
+  data.tar.gz: cae74a3c182b09e6c2a1ee34149388263428ddb3a9db5a8b7058de2d304b8544
 SHA512:
-  metadata.gz: 2e962c91d54e9156459fe98b1250b92c2d9394c1b64c002960f8b3b21f17c3789f157238fc6bd4bec23646c1f5ef84626fc2ecbb5a3dd5d14e6a45ca77f90f99
-  data.tar.gz: 4a92c6de07b5d1509b97d2f2a73730cd35e298a903ab36baad6c115b57d03ea4e02e121b345f933bead9d8954bd6ba8b5501f40b921e662ba18aba9331225932
+  metadata.gz: e52a3a0aafc32e09c275157bfcb9325167be90b8a6ab4cda02ddeccc5577c25e8477c64492751b24b3a1222640ce147a311b01c598500b9f6b1a7a1b67f1b88d
+  data.tar.gz: d8ade264acf142b682e05ce90a4ca32464f76a301e2e2c0440b4bf52e30d9d22f8ca84f58d02024c48c06c36f726b3b0fc050505f77ab262928f47ae5399cbd5

data/.github/workflows/ruby.yml

@@ -0,0 +1,20 @@
+name: Ruby
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - name: Build and test with Rake
+      run: |
+        gem install bundler
+        bundle install --jobs 4 --retry 3
+        bundle exec rake

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    eventboss (1.1.0)
+    eventboss (1.1.1)
       aws-sdk-sns (>= 1.1.0)
       aws-sdk-sqs (>= 1.3.0)
       concurrent-ruby (~> 1.0, >= 1.0.5)
@@ -11,23 +11,23 @@ GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.0.3)
-    aws-partitions (1.189.0)
-    aws-sdk-core (3.59.0)
+    aws-partitions (1.206.0)
+    aws-sdk-core (3.64.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-sns (1.18.0)
-      aws-sdk-core (~> 3, >= 3.58.0)
+    aws-sdk-sns (1.19.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.18.0)
-      aws-sdk-core (~> 3, >= 3.58.0)
+    aws-sdk-sqs (1.21.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.1.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
     concurrent-ruby (1.1.5)
     diff-lcs (1.3)
-    dotenv (2.7.4)
+    dotenv (2.7.5)
     jmespath (1.4.0)
     rake (12.3.1)
     rspec (3.7.0)

data/README.md

@@ -102,24 +102,29 @@ AWS_SNS_ENDPOINT=http://localhost:4575 # when using with localstack
 AWS_SQS_ENDPOINT=http://localhost:4576 # when using with localstack
 ```
 
-### Logging and error handling
-To have more verbose logging, set `log_level` in configuration (default is `info`).
-
-Logger is used as default error handler. There is Airbrake handler available, to use it ensure you have `airbrake` or `airbrake-ruby` gem and add it to error handlers stack:
+Be aware that `eventbus:deadletter:reload` rake task won't load your configuration if you are not using ENVs
+ in non Rails app, although to make it work you can extend your `Rakefile` with:
 
 ```ruby
-Eventboss.configure do |config|
-  config.error_handlers << Eventboss::ErrorHandlers::Airbrake.new
+load File.join(Gem::Specification.find_by_name('eventboss').gem_dir, 'lib', 'tasks', 'eventboss.rake')
+
+task :environment do
+  # Load your environment
+  # Example:
+  # require_relative 'config/application'
 end
+
+task 'eventboss:deadletter:reload': :environment
 ```
 
-### Polling strategy
+### Logging and error handling
+To have more verbose logging, set `log_level` in configuration (default is `info`).
 
-Default is `Eventboss::Polling::Basic`. See `eventboss/polling/*` for other options. The configuration should be a `lambda` like so:
+Logger is used as default error handler. There is Airbrake handler available, to use it ensure you have `airbrake` or `airbrake-ruby` gem and add it to error handlers stack:
 
 ```ruby
 Eventboss.configure do |config|
-  config.polling_strategy = lambda { |queues| Eventboss::Polling::TimedRoundRobin.new(queues) }
+  config.error_handlers << Eventboss::ErrorHandlers::Airbrake.new
 end
 ```
 

data/lib/eventboss/configuration.rb

@@ -11,7 +11,6 @@ module Eventboss
                 :eventboss_account_id,
                 :aws_access_key_id,
                 :aws_secret_access_key,
-                :polling_strategy,
                 :aws_sns_endpoint,
                 :aws_sqs_endpoint,
                 :sns_sqs_name_infix
@@ -81,12 +80,6 @@ module Eventboss
       defined_or_default('aws_sns_endpoint') { ENV['AWS_SNS_ENDPOINT'] }
     end
 
-    def polling_strategy
-      defined_or_default('polling_strategy') do
-        lambda { |queues| Eventboss::Polling::Basic.new(queues) }
-      end
-    end
-
     def sns_sqs_name_infix
       defined_or_default('sns_sqs_name_infix') { ENV['EVENTBUS_SQS_SNS_NAME_INFIX'] || 'eventboss' }
     end

data/lib/eventboss/railtie.rb

@@ -1,5 +1,9 @@
 class Eventboss::Railtie < Rails::Railtie
   rake_tasks do
     load 'tasks/eventboss.rake'
+
+    # Load rails environment before executing reload.
+    # It makes sure to load configuration file.
+    task 'eventboss:deadletter:reload': :environment
   end
 end

data/lib/eventboss/runner.rb

@@ -23,38 +23,6 @@ module Eventboss
         end
       end
 
-      def start
-        configuration = Eventboss.configuration
-
-        queue_listeners = Eventboss::QueueListener.list
-        Eventboss::Instrumentation.add(queue_listeners)
-        polling_strategy = configuration.polling_strategy.call(queue_listeners.keys)
-
-        fetcher = Eventboss::Fetcher.new(configuration)
-        executor = Concurrent.global_io_executor
-
-        manager = Eventboss::Manager.new(
-          fetcher,
-          polling_strategy,
-          executor,
-          queue_listeners,
-          configuration.concurrency,
-          configuration.error_handlers
-        )
-
-        manager.start
-
-        self_read = setup_signals([:SIGTERM])
-
-        begin
-          handle_signals(self_read)
-        rescue Interrupt
-          executor.shutdown
-          executor.wait_for_termination
-          exit 0
-        end
-      end
-
       private
 
       def setup_signals(signals)
@@ -69,7 +37,6 @@ module Eventboss
         self_read
       end
 
-
       def handle_signals(self_read, launcher)
         while readable_io = IO.select([self_read])
           signal = readable_io.first[0].gets.strip

data/lib/eventboss/version.rb

@@ -1,3 +1,3 @@
 module Eventboss
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

data/lib/eventboss.rb

@@ -19,11 +19,8 @@ require 'eventboss/worker'
 require 'eventboss/fetcher'
 require 'eventboss/publisher'
 require 'eventboss/sender'
-require 'eventboss/manager'
 require 'eventboss/runner'
 require 'eventboss/logger'
-require 'eventboss/polling/basic'
-require 'eventboss/polling/timed_round_robin'
 require 'eventboss/extensions'
 
 # For Rails use railtie, for plain Ruby apps use custom scripts loader

data/lib/tasks/eventboss.rake

@@ -3,7 +3,7 @@ require 'rake'
 namespace :eventboss do
   namespace :deadletter do
     desc 'Reload deadletter queue'
-    task :reload, [:event_name, :source_app, :max_messages] do |_, args|
+    task :reload, [:event_name, :source_app, :max_messages] do |task, args|
       source_app = args[:source_app]
       event_name = args[:event_name]
 
@@ -13,7 +13,7 @@ namespace :eventboss do
       # Ensure we don't fetch more than 10 messages from SQS
       batch_size = max_messages == 0 ? 10 : [10, max_messages].min
 
-      abort 'At least event name should be passed as argument' unless event_name
+      abort "[#{task.name}] At least event name should be passed as argument" unless event_name
 
       queue_name = [
         Eventboss.configuration.eventboss_app_name,
@@ -21,14 +21,14 @@ namespace :eventboss do
         source_app,
         event_name,
         Eventboss.env
-      ].join('-')
+      ].compact.join('-')
+      puts "[#{task.name}] Reloading #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
       queue = Eventboss::Queue.new("#{queue_name}-deadletter")
       send_queue = Eventboss::Queue.new(queue_name)
 
-      puts "Reloading deadletter (max: #{ max_messages }, batch: #{ batch_size })"
-      puts "  #{queue.url}"
-      puts '  to'
-      puts "  #{send_queue.url}"
+      puts "[#{task.name}] #{queue.url}"
+      puts "[#{task.name}] to"
+      puts "[#{task.name}] #{send_queue.url}"
 
       fetcher = Eventboss::Fetcher.new(Eventboss.configuration)
       client = fetcher.client
@@ -38,7 +38,7 @@ namespace :eventboss do
         break if messages.count.zero?
 
         messages.each do |message|
-          puts "Publishing message: #{message.body}"
+          puts "[#{task.name}] Publishing message: #{message.body}"
           client.send_message(queue_url: send_queue.url, message_body: message.body)
           fetcher.delete(queue, message)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventboss
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - AirHelp
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-30 00:00:00.000000000 Z
+date: 2019-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -128,6 +128,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -151,9 +152,6 @@ files:
 - lib/eventboss/logger.rb
 - lib/eventboss/logging.rb
 - lib/eventboss/long_poller.rb
-- lib/eventboss/manager.rb
-- lib/eventboss/polling/basic.rb
-- lib/eventboss/polling/timed_round_robin.rb
 - lib/eventboss/publisher.rb
 - lib/eventboss/queue.rb
 - lib/eventboss/queue_listener.rb

data/lib/eventboss/manager.rb

@@ -1,116 +0,0 @@
-module Eventboss
-  class Manager
-    MIN_DISPATCH_INTERVAL = 0.1
-
-    def initialize(fetcher, polling_strategy, executor, queue_listeners, concurrency, error_handlers)
-      @fetcher          = fetcher
-      @polling_strategy = polling_strategy
-      @max_processors   = concurrency
-      @busy_processors  = Concurrent::AtomicFixnum.new(0)
-      @executor         = executor
-      @queue_listeners  = queue_listeners
-      @error_handlers   = Array(error_handlers)
-    end
-
-    def start
-      Eventboss::Logger.debug('Starting dispatch loop...')
-
-      dispatch_loop
-    end
-
-    private
-
-    def running?
-      @executor.running?
-    end
-
-    def dispatch_loop
-      return unless running?
-
-      Eventboss::Logger.debug('Posting task to executor')
-
-      @executor.post { dispatch }
-    end
-
-    def dispatch
-      return unless running?
-
-      if ready <= 0 || (queue = @polling_strategy.next_queue).nil?
-        return sleep(MIN_DISPATCH_INTERVAL)
-      end
-      dispatch_single_messages(queue)
-    rescue => ex
-      handle_dispatch_error(ex)
-    ensure
-      Eventboss::Logger.debug('Ensuring dispatch loop')
-      dispatch_loop
-    end
-
-    def busy
-      @busy_processors.value
-    end
-
-    def ready
-      @max_processors - busy
-    end
-
-    def processor_done(processor)
-      Eventboss::Logger.info("Success", processor.jid)
-      @busy_processors.decrement
-    end
-
-    def processor_error(processor, exception)
-      @error_handlers.each { |handler| handler.call(exception, processor) }
-      @busy_processors.decrement
-    end
-
-    def assign(queue, sqs_msg)
-      return unless running?
-
-      @busy_processors.increment
-      processor = @queue_listeners[queue].new
-
-      Concurrent::Promise.execute(executor: @executor) do
-        body = JSON.parse(sqs_msg.body) rescue sqs_msg.body
-        Eventboss::Logger.info("Started", processor.jid)
-        processor.receive(body)
-      end.then do
-        cleanup(processor)
-        postpone_if_needed(queue, sqs_msg, processor) || delete_from_queue(queue, sqs_msg)
-        processor_done(processor)
-      end.rescue do |e|
-        cleanup(processor)
-        postpone_if_needed(queue, sqs_msg, processor)
-        processor_error(processor, e)
-      end
-    end
-
-    def cleanup(_processor)
-      if defined?(ActiveRecord)
-        ::ActiveRecord::Base.clear_active_connections!
-      end
-    end
-
-    def delete_from_queue(queue, sqs_msg)
-      @fetcher.delete(queue, sqs_msg)
-    end
-
-    def postpone_if_needed(queue, sqs_msg, processor)
-      return false unless processor.postponed_by
-      @fetcher.change_message_visibility(queue, sqs_msg, processor.postponed_by)
-    rescue => error
-      Eventboss::Logger.info("Could not postpone message #{error.message}", processor.jid)
-    end
-
-    def dispatch_single_messages(queue)
-      messages = @fetcher.fetch(queue, ready)
-      @polling_strategy.messages_found(queue, messages.size)
-      messages.each { |message| assign(queue, message) }
-    end
-
-    def handle_dispatch_error(ex)
-      Eventboss::Logger.error("Error dispatching #{ex.message}")
-      Process.kill('USR1', Process.pid)
-    end
-  end
-end

data/lib/eventboss/polling/basic.rb

@@ -1,68 +0,0 @@
-module Eventboss
-  module Polling
-    class Basic
-      PAUSE_AFTER_EMPTY = 2 # seconds
-
-      def initialize(queues, timer = Time)
-        @queues = queues.to_a
-        @timer = timer
-        @paused_until = @queues.each_with_object(Hash.new) do |queue, hash|
-          hash[queue] = @timer.at(0)
-        end
-
-        reset_next_queue
-      end
-
-      def next_queue
-        next_active_queue
-      end
-
-      def messages_found(queue, messages_count)
-        if messages_count == 0
-          pause(queue)
-        else
-          reset_next_queue
-        end
-      end
-
-      def active_queues
-        @queues.reject { |q, _| queue_paused?(q) }
-      end
-
-      private
-
-      def next_active_queue
-        reset_next_queue if queues_unpaused_since?
-
-        size = @queues.length
-        size.times do
-          queue = @queues[@next_queue_index]
-          @next_queue_index = (@next_queue_index + 1) % size
-          return queue unless queue_paused?(queue)
-        end
-
-        nil
-      end
-
-      def queues_unpaused_since?
-        last = @last_unpause_check
-        now = @last_unpause_check = @timer.now
-
-        last && @paused_until.values.any? { |t| t > last && t <= now }
-      end
-
-      def reset_next_queue
-        @next_queue_index = 0
-      end
-
-      def queue_paused?(queue)
-        @paused_until[queue] > @timer.now
-      end
-
-      def pause(queue)
-        return unless PAUSE_AFTER_EMPTY > 0
-        @paused_until[queue] = @timer.now + PAUSE_AFTER_EMPTY
-      end
-    end
-  end
-end

data/lib/eventboss/polling/timed_round_robin.rb

@@ -1,42 +0,0 @@
-module Eventboss
-  module Polling
-    class TimedRoundRobin
-      PAUSE_AFTER_EMPTY = 2 # seconds
-
-      def initialize(queues, timer = Time)
-        @queues = queues.to_a
-        @timer = timer
-        @next_queue_index = 0
-        @paused_until = @queues.each_with_object(Hash.new) do |queue, hash|
-          hash[queue] = @timer.at(0)
-        end
-      end
-
-      def next_queue
-        size = @queues.length
-        size.times do
-          queue = @queues[@next_queue_index]
-          @next_queue_index = (@next_queue_index + 1) % size
-          return queue unless queue_paused?(queue)
-        end
-
-        nil
-      end
-
-      def messages_found(queue, messages_count)
-        pause(queue) if messages_count == 0
-      end
-
-      private
-
-      def queue_paused?(queue)
-        @paused_until[queue] > @timer.now
-      end
-
-      def pause(queue)
-        return unless PAUSE_AFTER_EMPTY > 0
-        @paused_until[queue] = @timer.now + PAUSE_AFTER_EMPTY
-      end
-    end
-  end
-end