eventboss 1.4.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e7d223c4834db900777f804ac21d03e023e952fa2df455d90768d845cc806e1
-  data.tar.gz: bcc796c5866521aeea6a33c2920ab1634956e5f3b5aeb367c81a17b657df19a3
+  metadata.gz: 96d1bb368bb5d3178aba2cf5a00bec49444f24e3a0b768d3128651d14947248e
+  data.tar.gz: 98c2d5c2e7c64a629ef7ceedb31bfb74798c6a61d781fea22c52e36f132fd2b4
 SHA512:
-  metadata.gz: c84d52bb6344340c0cc17f8e7387db07e1a46fa1eb5d04b67d9d577e879ddc294fa71c150e0c3110244ee046a278e3fd43c855e3bd28f31e0477c01e8c5c8603
-  data.tar.gz: de26d5a46fe898d785edc6cd3a3436a75350907c10bf3ac3fd3d2bb9366c390776372dc9b39475cab57529df0c5aaf6df01c6ce1575043f5d947f806b8d056dc
+  metadata.gz: c54b9a6375dc06ce7dd516513b5e09c0f8eda8c79e796694cb9fbceaa348bff1bc196e83d400f2355b323b85b344018b13c8194505c37773b04c7196d6d6c012
+  data.tar.gz: a64d2636b7d9a8dbb910d85f36eedd873abdeb7b7b7aa078c19fcb67772d6de7f6dfc353117cf1b0ea84505c3f5aa5f87d48a4deb2c68beb504b4a6cfe24333c

data/.github/workflows/verify.yml

@@ -0,0 +1,40 @@
+name: Verify
+# Source: https://github.com/actions/setup-ruby
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ["2.6", "2.7", "3.0"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          # runs 'bundle install' and caches installed gems automatically
+          bundler-cache: true
+      - name: Run tests
+        run: bundle exec rspec
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.0"
+      - name: Install bundle audit
+        run: gem install bundle-audit
+      - name: Update Ruby Advisory Database
+        run: bundle-audit update
+      - name: Run audit
+        run: bundle audit

data/Gemfile.lock

@@ -1,55 +1,57 @@
 PATH
   remote: .
   specs:
-    eventboss (1.4.1)
+    eventboss (1.8.0)
       aws-sdk-sns (>= 1.1.0)
       aws-sdk-sqs (>= 1.3.0)
       dotenv (~> 2.1, >= 2.1.1)
+      rexml (~> 3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.1.0)
-    aws-partitions (1.350.0)
-    aws-sdk-core (3.104.3)
+    aws-eventstream (1.2.0)
+    aws-partitions (1.568.0)
+    aws-sdk-core (3.130.0)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
+      aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-sns (1.28.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-sns (1.53.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.30.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-sqs (1.51.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.1)
+    aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
-    diff-lcs (1.3)
+    diff-lcs (1.5.0)
     dotenv (2.7.6)
-    jmespath (1.4.0)
-    rake (13.0.1)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.0)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    jmespath (1.6.1)
+    rake (13.0.6)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.13)
+  bundler (>= 1)
   eventboss!
   rake (>= 10.0)
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.17.3
+   2.3.7

data/README.md

@@ -100,8 +100,8 @@ EVENTBOSS_ENV=env_name # production/staging/test
 EVENTBOSS_REGION=aws_region # i.e. eu-west-1
 EVENTBOSS_CONCURRENCY=10 # default is 25
 
-AWS_SNS_ENDPOINT=http://localhost:4575 # when using with localstack
-AWS_SQS_ENDPOINT=http://localhost:4576 # when using with localstack
+AWS_SNS_ENDPOINT=http://localhost:4566 # when using with localstack
+AWS_SQS_ENDPOINT=http://localhost:4566 # when using with localstack
 ```
 Use fixed account ID for localstack setup:
 ```

data/eventboss.gemspec

@@ -23,8 +23,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk-sqs", ">= 1.3.0"
   spec.add_dependency "aws-sdk-sns", ">= 1.1.0"
   spec.add_dependency "dotenv", "~> 2.1", ">= 2.1.1"
+  spec.add_dependency "rexml", "~> 3.0"
 
-  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "bundler", ">= 1"
   spec.add_development_dependency 'rake', '>= 10.0'
   spec.add_development_dependency "rspec", "~> 3.0"
 end

data/lib/eventboss/configuration.rb

@@ -25,14 +25,13 @@ module Eventboss
       :sns_sqs_name_infix,
       :listeners
 
-
     def raise_on_missing_configuration
       defined_or_default('raise_on_missing_configuration') { (ENV['EVENTBOSS_RAISE_ON_MISSING_CONFIGURATION'] || ENV['EVENTBUS_RAISE_ON_MISSING_CONFIGURATION'])&.downcase == 'true' }
     end
 
     def error_handlers
       defined_or_default('error_handlers') do
-        [ErrorHandlers::Logger.new].tap do |handlers|
+        [ErrorHandlers::Logger.new, ErrorHandlers::NonExistentQueueHandler.new].tap do |handlers|
           handlers << ErrorHandlers::DbConnectionDropHandler.new if defined?(::ActiveRecord::StatementInvalid)
           handlers << ErrorHandlers::DbConnectionNotEstablishedHandler.new if defined?(::ActiveRecord::ConnectionNotEstablished)
         end

data/lib/eventboss/development_mode.rb

@@ -17,6 +17,9 @@ module Eventboss
           logger.info('development-mode') { "Creating queue #{queue.name}..." }
           sqs_client.create_queue(queue_name: queue.name)
 
+          logger.info('development-mode') { "Creating deadletter queue #{queue.name}-deadletter..." }
+          sqs_client.create_queue(queue_name: "#{queue.name}-deadletter")
+
           logger.info('development-mode') { "Setting up queue #{queue.name} policy..." }
           policy = queue_policy(queue.arn, topic.topic_arn)
           sqs_client.set_queue_attributes(queue_url: queue.url, attributes: { Policy: policy.to_json })

data/lib/eventboss/error_handlers/non_existent_queue_handler.rb

@@ -0,0 +1,12 @@
+module Eventboss
+  module ErrorHandlers
+    class NonExistentQueueHandler
+      def call(exception, context = {})
+        if exception.class == ::Aws::SQS::Errors::NonExistentQueue
+          queue = context.fetch(:poller_id, "").sub('poller-', '')
+          Eventboss.logger.error("Queue  doesn't exist: " + queue)
+        end
+      end
+    end
+  end
+end

data/lib/eventboss/error_handlers/rollbar.rb

@@ -0,0 +1,11 @@
+module Eventboss
+  module ErrorHandlers
+    class Rollbar
+      def call(exception, context = {})
+        eventboss_context = { component: 'eventboss' }
+        eventboss_context[:action] = context[:processor].class.to_s if context[:processor]
+        ::Rollbar.error(exception, eventboss_context.merge(context))
+      end
+    end
+  end
+end

data/lib/eventboss/extensions.rb

@@ -1,4 +1,6 @@
 require 'eventboss/error_handlers/logger'
 require 'eventboss/error_handlers/airbrake'
+require 'eventboss/error_handlers/rollbar'
 require 'eventboss/error_handlers/db_connection_drop_handler'
 require 'eventboss/error_handlers/db_connection_not_established_handler'
+require 'eventboss/error_handlers/non_existent_queue_handler'

data/lib/eventboss/launcher.rb

@@ -34,6 +34,7 @@ module Eventboss
       @bus.clear
       @pollers.each(&:terminate)
       @workers.each(&:terminate)
+      @bus.close
 
       wait_for_shutdown
       hard_shutdown
@@ -70,7 +71,7 @@ module Eventboss
     end
 
     def new_worker(id)
-      Worker.new(self, id, @bus)
+      Worker.new(self, "worker-#{id}", @bus)
     end
 
     def new_poller(queue, listener)

data/lib/eventboss/long_poller.rb

@@ -45,6 +45,8 @@ module Eventboss
       fetch_messages.each do |message|
         logger.debug(id) { "enqueueing message #{message.message_id}" }
         @bus << UnitOfWork.new(@client, queue, listener, message)
+      rescue ClosedQueueError
+        logger.info(id) { "skip message #{message.message_id} enqueuing due to closed queue" }
       end
     end
 
@@ -53,6 +55,9 @@ module Eventboss
       @launcher.poller_stopped(self)
     rescue Eventboss::Shutdown
       @launcher.poller_stopped(self)
+    rescue Aws::SQS::Errors::NonExistentQueue => exception
+      handle_exception(exception, poller_id: id)
+      @launcher.poller_stopped(self)
     rescue StandardError => exception
       handle_exception(exception, poller_id: id)
       # Give a chance for temporary AWS errors to be resolved

data/lib/eventboss/version.rb

@@ -1,3 +1,3 @@
 module Eventboss
-  VERSION = "1.4.1"
+  VERSION = "1.8.0"
 end

data/lib/eventboss/worker.rb

@@ -7,7 +7,7 @@ module Eventboss
     attr_reader :id
 
     def initialize(launcher, id, bus, restart_on: [Exception])
-      @id = "worker-#{id}"
+      @id = id
       @launcher = launcher
       @bus = bus
       @thread = nil
@@ -45,7 +45,6 @@ module Eventboss
     end
 
     def kill(wait = false)
-      stop_token
       return unless @thread
       @thread.raise Eventboss::Shutdown
       @thread.value if wait

data/lib/tasks/eventboss.rake

@@ -6,6 +6,9 @@ namespace :eventboss do
     task :reload, [:event_name, :source_app, :max_messages] do |task, args|
       source_app = args[:source_app]
       event_name = args[:event_name]
+      start_time = Time.now
+
+      Eventboss.logger.info "[#{task.name}] Start task"
 
       # Zero means: fetch all messages
       max_messages = args[:max_messages].to_i
@@ -17,13 +20,11 @@ namespace :eventboss do
 
       queue_name = compose_queue_name(source_app, event_name)
 
-      puts "[#{task.name}] Reloading #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
+      Eventboss.logger.info "[#{task.name}] Reloading #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
       queue = Eventboss::Queue.new("#{queue_name}-deadletter")
       send_queue = Eventboss::Queue.new(queue_name)
 
-      puts "[#{task.name}] #{queue.url}"
-      puts "[#{task.name}] to"
-      puts "[#{task.name}] #{send_queue.url}"
+      Eventboss.logger.info "[#{task.name}] #{queue.url} to #{send_queue.url}"
 
       fetcher = Eventboss::Fetcher.new(Eventboss.configuration)
       client = fetcher.client
@@ -42,16 +43,24 @@ namespace :eventboss do
 
         break if max_messages > 0 && total >= max_messages
       end
+      Eventboss.logger.info <<~HEREDOC
+        [#{task.name}] Task done
+        total messages: #{total}
+        total time: #{Time.now - start_time}s
+      HEREDOC
     end
 
     desc 'Purge deadletter queue'
     task :purge, [:event_name, :source_app, :max_messages] do |task, args|
       source_app = args[:source_app]
       event_name = args[:event_name]
+      start_time = Time.now
 
       # Zero means: fetch all messages
       max_messages = args[:max_messages].to_i
 
+      Eventboss.logger.info "[#{task.name}] Start task"
+
       # Ensure we don't fetch more than 10 messages from SQS
       batch_size = max_messages == 0 ? 10 : [10, max_messages].min
 
@@ -59,9 +68,9 @@ namespace :eventboss do
 
       queue_name = compose_queue_name(source_app, event_name)
 
-      puts "[#{task.name}] Purging #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
+      Eventboss.logger.info "[#{task.name}] Purging #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
       queue = Eventboss::Queue.new("#{queue_name}-deadletter")
-      puts "[#{task.name}] #{queue.url}"
+      Eventboss.logger.info "[#{task.name}] #{queue.url}"
 
       fetcher = Eventboss::Fetcher.new(Eventboss.configuration)
       total = 0
@@ -78,6 +87,12 @@ namespace :eventboss do
 
         break if max_messages > 0 && total >= max_messages
       end
+
+      Eventboss.logger.info <<~HEREDOC
+        [#{task.name}] Task done
+        total messages: #{total}
+        total time: #{Time.now - start_time}s
+      HEREDOC
     end
 
     def compose_queue_name(source_app, event_name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventboss
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.8.0
 platform: ruby
 authors:
 - AirHelp
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-11 00:00:00.000000000 Z
+date: 2022-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -59,19 +59,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 2.1.1
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rexml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
+        version: '3.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/verify.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -128,6 +143,8 @@ files:
 - lib/eventboss/error_handlers/db_connection_drop_handler.rb
 - lib/eventboss/error_handlers/db_connection_not_established_handler.rb
 - lib/eventboss/error_handlers/logger.rb
+- lib/eventboss/error_handlers/non_existent_queue_handler.rb
+- lib/eventboss/error_handlers/rollbar.rb
 - lib/eventboss/extensions.rb
 - lib/eventboss/fetcher.rb
 - lib/eventboss/instrumentation.rb
@@ -171,7 +188,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Eventboss Ruby Client.