eventboss 1.3.1 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fbfa930452474047053b798205334360c07c6a90e912a90cb3012e9b64c75113
-  data.tar.gz: 9b4a1361b3bc0f0e72986394cd8055ec79983aee97dc128c7b3645d54179784d
+  metadata.gz: 4e7d223c4834db900777f804ac21d03e023e952fa2df455d90768d845cc806e1
+  data.tar.gz: bcc796c5866521aeea6a33c2920ab1634956e5f3b5aeb367c81a17b657df19a3
 SHA512:
-  metadata.gz: a483280c7c418e23e85d1e0727de6f8ec50b231e6c4dee3ca513526bb6ed653a3d6a8935c9dcaad8706ed1dd394bed37e33e26ad0f7e145a1e22624ff6d7b8d0
-  data.tar.gz: 211225253db25ec12e0d8f60de4b6abd077c8cec38b9771b0558854d68eba018ea0f0f4c3cb83e580000b99acfde5f0acfa3b138f2f3f8fb239ece14ff590353
+  metadata.gz: c84d52bb6344340c0cc17f8e7387db07e1a46fa1eb5d04b67d9d577e879ddc294fa71c150e0c3110244ee046a278e3fd43c855e3bd28f31e0477c01e8c5c8603
+  data.tar.gz: de26d5a46fe898d785edc6cd3a3436a75350907c10bf3ac3fd3d2bb9366c390776372dc9b39475cab57529df0c5aaf6df01c6ce1575043f5d947f806b8d056dc

data/.travis.yml

@@ -0,0 +1,5 @@
+lang: ruby
+
+script: bundle exec rake
+
+rvm: 2.6.5

data/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0] - 2020-04-18
+
+- Introduce server middlewares (#31)
+
 ## [1.1.0] - 2019-07-16
 
 ### Added

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    eventboss (1.3.1)
+    eventboss (1.4.1)
       aws-sdk-sns (>= 1.1.0)
       aws-sdk-sqs (>= 1.3.0)
       dotenv (~> 2.1, >= 2.1.1)
@@ -9,25 +9,25 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.0.3)
-    aws-partitions (1.232.0)
-    aws-sdk-core (3.73.0)
-      aws-eventstream (~> 1.0, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.228.0)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.350.0)
+    aws-sdk-core (3.104.3)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-sns (1.20.0)
-      aws-sdk-core (~> 3, >= 3.71.0)
+    aws-sdk-sns (1.28.0)
+      aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.23.0)
-      aws-sdk-core (~> 3, >= 3.71.0)
+    aws-sdk-sqs (1.30.0)
+      aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.1.0)
-      aws-eventstream (~> 1.0, >= 1.0.2)
+    aws-sigv4 (1.2.1)
+      aws-eventstream (~> 1, >= 1.0.2)
     diff-lcs (1.3)
-    dotenv (2.7.5)
+    dotenv (2.7.6)
     jmespath (1.4.0)
-    rake (12.3.1)
+    rake (13.0.1)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)

data/README.md

@@ -1,6 +1,7 @@
 # Eventboss
 
 [![Gem Version](https://badge.fury.io/rb/eventboss.svg)](https://badge.fury.io/rb/eventboss)
+[![Build Status](https://travis-ci.org/AirHelp/eventboss.svg?branch=master)](https://travis-ci.org/AirHelp/eventboss)
 
 AWS based Pub/Sub implementation in Ruby.
 
@@ -16,10 +17,10 @@ AWS based Pub/Sub implementation in Ruby.
 * [x] support multiple environments in the same AWS account
 * [x] pluggable error handlers (airbrake, newrelic)
 * [x] utility tasks (deadletter reload)
-* [x] localstack compatible
+* [x] [localstack](https://github.com/localstack/localstack) compatible
 * [x] rails support (preloads rails environment)
+* [x] development mode (creates missing SNS/SQS on the fly)
 * [ ] terraform pub/sub scripts
-* [ ] development mode (creates missing SNS/SQS on the fly)
 * [ ] alternative infrastructure (redis?, kafka?)
 * [ ] message compression
 * [ ] alternative serialization (protobuf)
@@ -93,15 +94,19 @@ end
 Using ENVs:
 
 ```
-EVENTBUS_ACCOUNT_ID=12345676
-EVENTBUS_APP_NAME=application_name
-EVENTBUS_ENV=env_name # production/staging/test
-EVENTBUS_REGION=aws_region # i.e. eu-west-1
-EVENTBUS_CONCURRENCY=10 # default is 25
+EVENTBOSS_ACCOUNT_ID=12345676
+EVENTBOSS_APP_NAME=application_name
+EVENTBOSS_ENV=env_name # production/staging/test
+EVENTBOSS_REGION=aws_region # i.e. eu-west-1
+EVENTBOSS_CONCURRENCY=10 # default is 25
 
 AWS_SNS_ENDPOINT=http://localhost:4575 # when using with localstack
 AWS_SQS_ENDPOINT=http://localhost:4576 # when using with localstack
 ```
+Use fixed account ID for localstack setup:
+```
+EVENTBUS_ACCOUNT_ID=000000000000
+```
 
 Be aware that `eventbus:deadletter:reload` rake task won't load your configuration if you are not using ENVs
  in non Rails app, although to make it work you can extend your `Rakefile` with:
@@ -128,12 +133,12 @@ listeners:
   exclude:
     - OtherListener # When include option is not set it will run all listeners except listed here (OtherListener). When include is set it will run only included (but not excluded) listeners.
 ```
-Yml config is optional and by default is loaded from `'./config/eventboss.yml'`.
+YAML config is optional and by default is loaded from `'./config/eventboss.yml'`.
 You can also pass config path as an argument:
-```bash
+```sh
 eventboss -C my/custom/path/to/config.yml
 ```
-Yml config content is merged to configuration last, which means it overwrites ENVs and `.configure`.
+YAML config content is merged to configuration last, which means it overwrites ENVs and `.configure`.
 
 ### Logging and error handling
 To have more verbose logging, set `log_level` in configuration (default is `info`).
@@ -146,9 +151,65 @@ Eventboss.configure do |config|
 end
 ```
 
+### Middlewares
+
+Server middlewares intercept the execution of your `Listeners`. You can use to extract and run common functions on every message received.
+
+Define a middleware in the following way:
+
+```ruby
+class LogMiddleware < Eventboss::Middleware::Base
+  def call(_work)
+    yield
+    logger.debug 'finished with success'
+  rescue StandardError => _error
+    logger.error 'finished with error'
+    raise
+  end
+
+  private
+
+  def logger
+    @logger ||= @options.fetch(:logger)
+  end
+end
+```
+
+And configure your logger as such:
+
+```ruby
+Eventboss.configure do |config|
+  config.server_middleware.add LogMiddleware, logger: Logger.new
+end
+```
+
+## Development mode
+
+In the _development mode_ you don't need to create the infrastructure required by the application - Eventboss will take care of this.
+
+It works on AWS and [localstack](https://github.com/localstack/localstack).
+
+Following resources are created:
+* SNS topics - created when application starts and when message is published
+* SQS queues (with SendMessage policy) - created when application starts
+* subscriptions for topics and queues - created when application starts
+
+Just enable it via environment variable...
+```
+EVENTBOSS_DEVELOPMENT_MODE=true
+```
+use fixed account ID for localstack setup...
+```
+EVENTBUS_ACCOUNT_ID=000000000000 # or set it via YAML
+```
+...and you're good to go:
+```shell script
+bundle exec eventboss
+```
+
 ## Topics & Queues naming convention
 
-The SNSes should be name in the following pattern:
+The SNSes should be named in the following pattern:
 ```
 eventboss-{src_app_name}-{event_name}-{environment}
 ```
@@ -158,7 +219,7 @@ i.e.
 eventboss-srcapp-transaction_change-staging
 ```
 
-The corresponding SQSes should be name like:
+The corresponding SQSes should be named like:
 ```
 {dest_app_name}-eventboss-{src_app_name}-{event_name}-{environment}
 {dest_app_name}-eventboss-{src_app_name}-{event_name}-{environment}-deadletter

data/lib/eventboss.rb

@@ -13,13 +13,16 @@ require 'eventboss/logging'
 require 'eventboss/safe_thread'
 require 'eventboss/launcher'
 require 'eventboss/long_poller'
+require 'eventboss/middleware'
 require 'eventboss/unit_of_work'
 require 'eventboss/worker'
 require 'eventboss/fetcher'
 require 'eventboss/publisher'
 require 'eventboss/sender'
+require 'eventboss/topic'
 require 'eventboss/runner'
 require 'eventboss/extensions'
+require 'eventboss/development_mode'
 
 # For Rails use railtie, for plain Ruby apps use custom scripts loader
 if defined?(Rails)
@@ -33,21 +36,34 @@ module Eventboss
 
   class << self
     def publisher(event_name, opts = {})
-      Publisher.new(event_name, configuration.sns_client, configuration, opts)
+      sns_client = configuration.sns_client
+
+      if configuration.development_mode?
+        source_app = configuration.eventboss_app_name unless opts[:generic]
+        topic_name = Topic.build_name(event_name: event_name, source_app: source_app)
+        sns_client.create_topic(name: topic_name)
+      end
+
+      Publisher.new(event_name, sns_client, configuration, opts)
     end
 
-    def sender(event_name, destination_app, options = {})
-      queue_name = Queue.build_name(
-        destination: destination_app,
-        source: configuration.eventboss_app_name,
-        event: event_name,
-        env: env,
-        generic: options[:generic]
+    def sender(event_name, destination, options = {})
+      source_app = configuration.eventboss_app_name unless options[:generic]
+      queue = Queue.build(
+        destination: destination,
+        source_app: source_app,
+        event_name: event_name,
+        env: env
       )
+      sqs_client = configuration.sqs_client
+
+      if configuration.development_mode?
+        sqs_client.create_queue(queue_name: queue.name)
+      end
 
       Sender.new(
-        client: configuration.sqs_client,
-        queue: Queue.new(queue_name)
+        client: sqs_client,
+        queue: queue
       )
     end
 
@@ -56,7 +72,7 @@ module Eventboss
     end
 
     def env
-      @env ||= ENV['EVENTBUS_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV']
+      @env ||= ENV['EVENTBOSS_ENV'] || ENV['EVENTBUS_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV']
     end
 
     def configure

data/lib/eventboss/configuration.rb

@@ -1,43 +1,49 @@
+# frozen_string_literal: true
+
 module Eventboss
   class Configuration
     OPTS_ALLOWED_IN_CONFIG_FILE = %i[
       concurrency
       sns_sqs_name_infix
       listeners
-    ]
+    ].freeze
 
     attr_writer :raise_on_missing_configuration,
-                :error_handlers,
-                :concurrency,
-                :log_level,
-                :logger,
-                :sns_client,
-                :sqs_client,
-                :eventboss_region,
-                :eventboss_app_name,
-                :eventboss_account_id,
-                :aws_access_key_id,
-                :aws_secret_access_key,
-                :aws_sns_endpoint,
-                :aws_sqs_endpoint,
-                :sns_sqs_name_infix,
-                :listeners
+      :error_handlers,
+      :concurrency,
+      :log_level,
+      :logger,
+      :sns_client,
+      :sqs_client,
+      :eventboss_region,
+      :eventboss_app_name,
+      :eventboss_account_id,
+      :aws_access_key_id,
+      :aws_secret_access_key,
+      :aws_sns_endpoint,
+      :aws_sqs_endpoint,
+      :sns_sqs_name_infix,
+      :listeners
 
 
     def raise_on_missing_configuration
-      defined_or_default('raise_on_missing_configuration') { ENV['EVENTBUS_RAISE_ON_MISSING_CONFIGURATION']&.downcase == 'true' }
+      defined_or_default('raise_on_missing_configuration') { (ENV['EVENTBOSS_RAISE_ON_MISSING_CONFIGURATION'] || ENV['EVENTBUS_RAISE_ON_MISSING_CONFIGURATION'])&.downcase == 'true' }
     end
 
     def error_handlers
       defined_or_default('error_handlers') do
         [ErrorHandlers::Logger.new].tap do |handlers|
           handlers << ErrorHandlers::DbConnectionDropHandler.new if defined?(::ActiveRecord::StatementInvalid)
+          handlers << ErrorHandlers::DbConnectionNotEstablishedHandler.new if defined?(::ActiveRecord::ConnectionNotEstablished)
         end
       end
     end
 
     def concurrency
-      defined_or_default('concurrency') { ENV['EVENTBUS_CONCURRENCY'] ? ENV['EVENTBUS_CONCURRENCY'].to_i : 25 }
+      defined_or_default('concurrency') do
+        concurrency = ENV['EVENTBOSS_CONCURRENCY'] || ENV['EVENTBUS_CONCURRENCY']
+        concurrency ? concurrency.to_i : 25
+      end
     end
 
     def log_level
@@ -72,15 +78,15 @@ module Eventboss
     end
 
     def eventboss_region
-      defined_or_default('eventboss_region') { ENV['EVENTBUS_REGION'] }
+      defined_or_default('eventboss_region') { ENV['EVENTBOSS_REGION'] || ENV['EVENTBUS_REGION'] }
     end
 
     def eventboss_app_name
-      defined_or_default('eventboss_app_name') { ENV['EVENTBUS_APP_NAME'] }
+      defined_or_default('eventboss_app_name') { ENV['EVENTBOSS_APP_NAME'] || ENV['EVENTBUS_APP_NAME'] }
     end
 
     def eventboss_account_id
-      defined_or_default('eventboss_account_id') { ENV['EVENTBUS_ACCOUNT_ID'] }
+      defined_or_default('eventboss_account_id') { ENV['EVENTBOSS_ACCOUNT_ID'] || ENV['EVENTBUS_ACCOUNT_ID'] }
     end
 
     def aws_access_key_id
@@ -100,13 +106,23 @@ module Eventboss
     end
 
     def sns_sqs_name_infix
-      defined_or_default('sns_sqs_name_infix') { ENV['EVENTBUS_SQS_SNS_NAME_INFIX'] || 'eventboss' }
+      defined_or_default('sns_sqs_name_infix') { ENV['EVENTBOSS_SQS_SNS_NAME_INFIX'] || ENV['EVENTBUS_SQS_SNS_NAME_INFIX'] || 'eventboss' }
     end
 
     def listeners
       defined_or_default('listeners') { {} }
     end
 
+    def development_mode?
+      defined_or_default('development_mode') do
+        (ENV['EVENTBOSS_DEVELOPMENT_MODE']&.downcase || ENV['EVENTBUS_DEVELOPMENT_MODE'])&.downcase == 'true'
+      end
+    end
+
+    def server_middleware
+      @server_middleware ||= Middleware::Chain.new
+    end
+
     private
 
     def defined_or_default(variable_name)

data/lib/eventboss/development_mode.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Eventboss
+  module DevelopmentMode
+    extend Logging
+
+    class << self
+      def setup_infrastructure(queues)
+        sns_client = Eventboss.configuration.sns_client
+        sqs_client = Eventboss.configuration.sqs_client
+
+        queues.each do |queue, listener|
+          topic_name = Eventboss::Topic.build_name(**listener.options)
+          logger.info('development-mode') { "Creating topic #{topic_name}..." }
+          topic = sns_client.create_topic(name: topic_name)
+
+          logger.info('development-mode') { "Creating queue #{queue.name}..." }
+          sqs_client.create_queue(queue_name: queue.name)
+
+          logger.info('development-mode') { "Setting up queue #{queue.name} policy..." }
+          policy = queue_policy(queue.arn, topic.topic_arn)
+          sqs_client.set_queue_attributes(queue_url: queue.url, attributes: { Policy: policy.to_json })
+
+          logger.info('development-mode') { "Creating subscription for topic #{topic.topic_arn} and #{queue.arn}..." }
+          sns_client.create_subscription(topic_arn: topic.topic_arn, queue_arn: queue.arn)
+        end
+      end
+
+      def queue_policy(queue_arn, topic_arn)
+        {
+          "Version": "2012-10-17",
+          "Statement": [{
+            "Sid": "queue-policy-#{queue_arn}-#{topic_arn}",
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": ["SQS:SendMessage"],
+            "Resource": "#{queue_arn}",
+            "Condition": {
+              "ArnEquals": {
+                "aws:SourceArn": "#{topic_arn}"
+              }
+            }
+          }]
+        }
+      end
+    end
+  end
+end

data/lib/eventboss/error_handlers/db_connection_not_established_handler.rb

@@ -0,0 +1,11 @@
+module Eventboss
+  module ErrorHandlers
+    class DbConnectionNotEstablishedHandler
+      def call(exception, _context = {})
+        if exception.class == ::ActiveRecord::ConnectionNotEstablished
+          ::ActiveRecord::Base.connection.reconnect!
+        end
+      end
+    end
+  end
+end

data/lib/eventboss/error_handlers/logger.rb

@@ -6,7 +6,7 @@ module Eventboss
         notice[:jid] = notice[:processor].jid if notice[:processor]
         notice[:processor] = notice[:processor].class.to_s if notice[:processor]
         Eventboss.logger.error(notice) do
-          "Failure processing request #{exception.message}"
+          "Failure processing request: #{exception.message}"
         end
       end
     end

data/lib/eventboss/extensions.rb

@@ -1,3 +1,4 @@
 require 'eventboss/error_handlers/logger'
 require 'eventboss/error_handlers/airbrake'
 require 'eventboss/error_handlers/db_connection_drop_handler'
+require 'eventboss/error_handlers/db_connection_not_established_handler'

data/lib/eventboss/launcher.rb

@@ -70,7 +70,7 @@ module Eventboss
     end
 
     def new_worker(id)
-      Worker.new(self, id, @client, @bus)
+      Worker.new(self, id, @bus)
     end
 
     def new_poller(queue, listener)

data/lib/eventboss/listener.rb

@@ -17,15 +17,12 @@ module Eventboss
     end
 
     module ClassMethods
-      def eventboss_options(opts)
-        source_app = opts[:source_app] ? "#{opts[:source_app]}-" : ""
-        event_name = opts[:event_name]
-        destination_app = opts[:destination_app]
+      attr_reader :options
 
-        ACTIVE_LISTENERS["#{source_app}#{event_name}"] = {
-          listener: self,
-          destination_app: destination_app
-        }
+      def eventboss_options(options)
+        @options = options.compact
+
+        ACTIVE_LISTENERS[@options] = self
       end
     end
   end

data/lib/eventboss/long_poller.rb

@@ -44,7 +44,7 @@ module Eventboss
     def fetch_and_dispatch
       fetch_messages.each do |message|
         logger.debug(id) { "enqueueing message #{message.message_id}" }
-        @bus << UnitOfWork.new(queue, listener, message)
+        @bus << UnitOfWork.new(@client, queue, listener, message)
       end
     end
 

data/lib/eventboss/middleware.rb

@@ -0,0 +1,57 @@
+module Eventboss
+  module Middleware
+    class Chain
+      attr_reader :entries
+
+      def initialize
+        @entries = []
+      end
+
+      def add(klass, options = {})
+        @entries << Entry.new(klass, options)
+      end
+
+      def invoke(*args)
+        chain = @entries.map(&:build).reverse!
+
+        invoke_lambda = lambda do
+          if (mid = chain.pop)
+            mid.call(*args, &invoke_lambda)
+          else
+            yield
+          end
+        end
+        invoke_lambda.call
+      end
+
+      def clear
+        @entries.clear
+      end
+    end
+
+    class Base
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      def call
+        raise 'Not implemented'
+      end
+    end
+
+    class Entry
+      attr_reader :klass, :options
+
+      def initialize(klass, options)
+        @klass = klass
+        @options = options
+      end
+
+      def build
+        @klass.new(options)
+      end
+    end
+  end
+end

data/lib/eventboss/publisher.rb

@@ -1,32 +1,28 @@
+# frozen_string_literal: true
+
 module Eventboss
   class Publisher
     def initialize(event_name, sns_client, configuration, opts = {})
       @event_name = event_name
       @sns_client = sns_client
       @configuration = configuration
-      @generic = opts[:generic]
+      @source = configuration.eventboss_app_name unless opts[:generic]
     end
 
     def publish(payload)
-      sns_client.publish({
+      topic_arn = Topic.build_arn(event_name: event_name, source_app: source)
+      sns_client.publish(
         topic_arn: topic_arn,
         message: json_payload(payload)
-      })
+      )
     end
 
     private
 
-    attr_reader :event_name, :sns_client, :configuration
+    attr_reader :event_name, :sns_client, :configuration, :source
 
     def json_payload(payload)
       payload.is_a?(String) ? payload : payload.to_json
     end
-
-    def topic_arn
-      src_selector = @generic ? "" : "-#{configuration.eventboss_app_name}"
-
-      "arn:aws:sns:#{configuration.eventboss_region}:#{configuration.eventboss_account_id}:\
-#{Eventboss.configuration.sns_sqs_name_infix}#{src_selector}-#{event_name}-#{Eventboss.env}"
-    end
   end
 end

data/lib/eventboss/queue.rb

@@ -1,21 +1,34 @@
+# frozen_string_literal: true
+
 module Eventboss
   class Queue
     include Comparable
     attr_reader :name
 
-    def self.build_name(source:, destination:, event:, env:, generic:)
-      source =
-        if generic
-          ''
-        else
-          "#{source}-"
-        end
+    class << self
+      def build_name(destination:, event_name:, env:, source_app: nil)
+        [
+          destination,
+          Eventboss.configuration.sns_sqs_name_infix,
+          source_app,
+          event_name,
+          env
+        ].compact.join('-')
+      end
 
-      "#{destination}-#{Eventboss.configuration.sns_sqs_name_infix}-#{source}#{event}-#{env}"
+      def build(destination:, event_name:, env:, source_app: nil)
+        name = build_name(
+          destination: destination,
+          event_name: event_name,
+          env: env,
+          source_app: source_app
+        )
+        Queue.new(name)
+      end
     end
 
-    def initialize(name, configuration = Eventboss.configuration)
-      @client = configuration.sqs_client
+    def initialize(name)
+      @client = Eventboss.configuration.sqs_client
       @name = name
     end
 
@@ -23,6 +36,15 @@ module Eventboss
       @url ||= client.get_queue_url(queue_name: name).queue_url
     end
 
+    def arn
+      [
+        'arn:aws:sqs',
+        Eventboss.configuration.eventboss_region,
+        Eventboss.configuration.eventboss_account_id,
+        name
+      ].join(':')
+    end
+
     def <=>(another_queue)
       name <=> another_queue&.name
     end
@@ -35,6 +57,10 @@ module Eventboss
       name.hash
     end
 
+    def to_s
+      "<Eventboss::Queue: #{name}>"
+    end
+
     private
 
     attr_reader :client

data/lib/eventboss/queue_listener.rb

@@ -1,7 +1,7 @@
 module Eventboss
   class QueueListener
     class << self
-      def select(include:, exclude:)
+      def select(include: nil, exclude: nil)
         listeners = list.values.map(&:name)
 
         listeners &= include if include
@@ -13,19 +13,15 @@ module Eventboss
       private
 
       def list
-        Hash[Eventboss::Listener::ACTIVE_LISTENERS.map do |src_app_event, opts|
-          [
-            Eventboss::Queue.new(
-              [
-                opts[:destination_app] || Eventboss.configuration.eventboss_app_name,
-                Eventboss.configuration.sns_sqs_name_infix,
-                src_app_event,
-                Eventboss.env
-              ].join('-')
-            ),
-            opts[:listener]
-          ]
-        end]
+        Eventboss::Listener::ACTIVE_LISTENERS.each_with_object({}) do |(eventboss_options, listener), queue_listeners|
+          queue = Eventboss::Queue.build(
+            destination: eventboss_options[:destination_app] || Eventboss.configuration.eventboss_app_name,
+            source_app: eventboss_options[:source_app],
+            event_name: eventboss_options[:event_name],
+            env: Eventboss.env
+          )
+          queue_listeners[queue] = listener
+        end
       end
     end
   end

data/lib/eventboss/railtie.rb

@@ -2,8 +2,9 @@ class Eventboss::Railtie < Rails::Railtie
   rake_tasks do
     load 'tasks/eventboss.rake'
 
-    # Load rails environment before executing reload.
+    # Load rails environment before executing reload and purge.
     # It makes sure to load configuration file.
     task 'eventboss:deadletter:reload': :environment
+    task 'eventboss:deadletter:purge': :environment
   end
 end

data/lib/eventboss/runner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Eventboss
   class Runner
     extend Logging
@@ -17,8 +19,11 @@ module Eventboss
 
         self_read = setup_signals([:SIGTERM])
 
-        logger.info('Active Listeners:')
-        logger.info(queues.to_s)
+        logger.info('Active listeners:')
+        queues.each { |queue, listener| logger.info("#{queue}: #{listener}") }
+
+        Eventboss::DevelopmentMode.setup_infrastructure(queues) if config.development_mode?
+
         begin
           launcher.start
           handle_signals(self_read, launcher)
@@ -45,7 +50,7 @@ module Eventboss
       def handle_signals(self_read, launcher)
         while readable_io = IO.select([self_read])
           signal = readable_io.first[0].gets.strip
-          logger.info('runner') { "Received #{ signal } signal, gracefully shutdowning..." }
+          logger.info('runner') { "Received #{signal} signal, gracefully shutting down..." }
 
           launcher.stop
           exit 0

data/lib/eventboss/sns_client.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 module Eventboss
-  class NotConfigured < StandardError; end
+  class NotConfigured < StandardError;
+  end
 
   class SnsClient
     def initialize(configuration)
@@ -10,10 +13,31 @@ module Eventboss
       backend.publish(payload)
     end
 
+    def create_topic(name:)
+      backend.create_topic(name: name)
+    end
+
+    def create_subscription(topic_arn:, queue_arn:)
+      subscription = backend.subscribe(
+        topic_arn: topic_arn,
+        endpoint: queue_arn,
+        protocol: 'sqs'
+      )
+      set_raw_message_delivery(subscription)
+    end
+
     private
 
     attr_reader :configuration
 
+    def set_raw_message_delivery(subscription)
+      backend.set_subscription_attributes(
+        subscription_arn: subscription.subscription_arn,
+        attribute_name: 'RawMessageDelivery',
+        attribute_value: 'true'
+      )
+    end
+
     def backend
       if configured?
         options = {
@@ -26,11 +50,10 @@ module Eventboss
         if configuration.aws_sns_endpoint
           options[:endpoint] = configuration.aws_sns_endpoint
         end
-        Aws::SNS::Client.new(
-          options
-        )
+
+        Aws::SNS::Client.new(options)
       elsif configuration.raise_on_missing_configuration
-        raise NotConfigured, 'Eventboss is not configured'
+        raise NotConfigured, 'Eventboss is not configured.'
       else
         Mock.new
       end

data/lib/eventboss/topic.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Eventboss
+  class Topic
+    class << self
+      def build_arn(event_name:, source_app: nil)
+        [
+          'arn:aws:sns',
+          Eventboss.configuration.eventboss_region,
+          Eventboss.configuration.eventboss_account_id,
+          build_name(
+            event_name: event_name,
+            source_app: source_app
+          )
+        ].join(':')
+      end
+
+      def build_name(event_name:, source_app: nil)
+        [
+          Eventboss.configuration.sns_sqs_name_infix,
+          source_app,
+          event_name,
+          Eventboss.env
+        ].compact.join('-')
+      end
+    end
+  end
+end

data/lib/eventboss/unit_of_work.rb

@@ -6,14 +6,15 @@ module Eventboss
 
     attr_accessor :queue, :listener, :message
 
-    def initialize(queue, listener, message)
+    def initialize(client, queue, listener, message)
+      @client = client
       @queue = queue
       @listener = listener
       @message = message
       @logger = logger
     end
 
-    def run(client)
+    def run
       logger.debug(@message.message_id) { 'Started' }
       processor = @listener.new
       processor.receive(JSON.parse(@message.body))
@@ -21,21 +22,21 @@ module Eventboss
     rescue StandardError => exception
       handle_exception(exception, processor: processor, message_id: @message.message_id)
     else
-      cleanup(client) unless processor.postponed_by
+      cleanup unless processor.postponed_by
     ensure
-      change_message_visibility(client, processor.postponed_by) if processor.postponed_by
+      change_message_visibility(processor.postponed_by) if processor.postponed_by
     end
 
-    def change_message_visibility(client, postponed_by)
-      client.change_message_visibility(
+    def change_message_visibility(postponed_by)
+      @client.change_message_visibility(
         queue_url: @queue.url,
         receipt_handle: @message.receipt_handle,
         visibility_timeout: postponed_by
       )
     end
 
-    def cleanup(client)
-      client.delete_message(
+    def cleanup
+      @client.delete_message(
         queue_url: @queue.url, receipt_handle: @message.receipt_handle
       )
       logger.debug(@message.message_id) { 'Deleting' }

data/lib/eventboss/version.rb

@@ -1,3 +1,3 @@
 module Eventboss
-  VERSION = "1.3.1"
+  VERSION = "1.4.1"
 end

data/lib/eventboss/worker.rb

@@ -6,12 +6,12 @@ module Eventboss
 
     attr_reader :id
 
-    def initialize(launcher, id, client, bus)
+    def initialize(launcher, id, bus, restart_on: [Exception])
       @id = "worker-#{id}"
       @launcher = launcher
-      @client = client
       @bus = bus
       @thread = nil
+      @restart_on = restart_on
     end
 
     def start
@@ -20,18 +20,24 @@ module Eventboss
 
     def run
       while (work = @bus.pop)
-        work.run(@client)
+        run_work(work)
       end
       @launcher.worker_stopped(self)
     rescue Eventboss::Shutdown
       @launcher.worker_stopped(self)
-    rescue Exception => exception
+    rescue *@restart_on => exception
       handle_exception(exception, worker_id: id)
       # Restart the worker in case of hard exception
       # Message won't be delete from SQS and will be visible later
       @launcher.worker_stopped(self, restart: true)
     end
 
+    def run_work(work)
+      server_middleware.invoke(work) do
+        work.run
+      end
+    end
+
     def terminate(wait = false)
       stop_token
       return unless @thread
@@ -51,5 +57,9 @@ module Eventboss
     def stop_token
       @bus << nil
     end
+
+    def server_middleware
+      Eventboss.configuration.server_middleware
+    end
   end
 end

data/lib/generators/eventboss/listener/eventboss_listener.rb.erb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class <%= event_name.camelize %>Listener
+  include Eventboss::Listener
+
+  eventboss_options event_name: '<%= event_name %>'<%= source_app ? ", source_app: '#{ source_app }'" : "" %>
+
+  def receive(payload)
+    Rails.logger.tagged(jid) { Rails.logger.info("payload: #{ payload }") }
+  end
+end

data/lib/generators/eventboss/listener/listener_generator.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Creates the Eventboss listener scaffold
+#
+# @example Invocation from terminal
+#   rails generate eventboss:listener get_well air-helper
+#
+module Eventboss
+  class ListenerGenerator < Rails::Generators::Base
+    source_root File.expand_path(__dir__)
+
+    argument :event_name, required: true
+    argument :source_app, required: false
+
+    desc 'Creates the Eventboss listener scaffold'
+    def create_listener_scaffold
+      template 'eventboss_listener.rb.erb', "app/listeners/#{ event_name }_listener.rb"
+    end
+  end
+end

data/lib/tasks/eventboss.rake

@@ -7,7 +7,7 @@ namespace :eventboss do
       source_app = args[:source_app]
       event_name = args[:event_name]
 
-      # Zero means, fetch all messages
+      # Zero means: fetch all messages
       max_messages = args[:max_messages].to_i
 
       # Ensure we don't fetch more than 10 messages from SQS
@@ -15,13 +15,8 @@ namespace :eventboss do
 
       abort "[#{task.name}] At least event name should be passed as argument" unless event_name
 
-      queue_name = [
-        Eventboss.configuration.eventboss_app_name,
-        Eventboss.configuration.sns_sqs_name_infix,
-        source_app,
-        event_name,
-        Eventboss.env
-      ].compact.join('-')
+      queue_name = compose_queue_name(source_app, event_name)
+
       puts "[#{task.name}] Reloading #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
       queue = Eventboss::Queue.new("#{queue_name}-deadletter")
       send_queue = Eventboss::Queue.new(queue_name)
@@ -38,7 +33,6 @@ namespace :eventboss do
         break if messages.count.zero?
 
         messages.each do |message|
-          puts "[#{task.name}] Publishing message: #{message.body}"
           client.send_message(queue_url: send_queue.url, message_body: message.body)
           fetcher.delete(queue, message)
 
@@ -49,5 +43,51 @@ namespace :eventboss do
         break if max_messages > 0 && total >= max_messages
       end
     end
+
+    desc 'Purge deadletter queue'
+    task :purge, [:event_name, :source_app, :max_messages] do |task, args|
+      source_app = args[:source_app]
+      event_name = args[:event_name]
+
+      # Zero means: fetch all messages
+      max_messages = args[:max_messages].to_i
+
+      # Ensure we don't fetch more than 10 messages from SQS
+      batch_size = max_messages == 0 ? 10 : [10, max_messages].min
+
+      abort "[#{task.name}] At least event name should be passed as argument" unless event_name
+
+      queue_name = compose_queue_name(source_app, event_name)
+
+      puts "[#{task.name}] Purging #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
+      queue = Eventboss::Queue.new("#{queue_name}-deadletter")
+      puts "[#{task.name}] #{queue.url}"
+
+      fetcher = Eventboss::Fetcher.new(Eventboss.configuration)
+      total = 0
+      loop do
+        messages = fetcher.fetch(queue, batch_size)
+        break if messages.count.zero?
+
+        messages.each do |message|
+          fetcher.delete(queue, message)
+
+          total += 1
+          break if max_messages > 0 && total >= max_messages
+        end
+
+        break if max_messages > 0 && total >= max_messages
+      end
+    end
+
+    def compose_queue_name(source_app, event_name)
+      [
+        Eventboss.configuration.eventboss_app_name,
+        Eventboss.configuration.sns_sqs_name_infix,
+        source_app,
+        event_name,
+        Eventboss.env
+      ].compact.join('-')
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventboss
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.1
 platform: ruby
 authors:
 - AirHelp
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-05 00:00:00.000000000 Z
+date: 2020-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -108,9 +108,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
@@ -123,8 +123,10 @@ files:
 - lib/eventboss.rb
 - lib/eventboss/cli.rb
 - lib/eventboss/configuration.rb
+- lib/eventboss/development_mode.rb
 - lib/eventboss/error_handlers/airbrake.rb
 - lib/eventboss/error_handlers/db_connection_drop_handler.rb
+- lib/eventboss/error_handlers/db_connection_not_established_handler.rb
 - lib/eventboss/error_handlers/logger.rb
 - lib/eventboss/extensions.rb
 - lib/eventboss/fetcher.rb
@@ -133,6 +135,7 @@ files:
 - lib/eventboss/listener.rb
 - lib/eventboss/logging.rb
 - lib/eventboss/long_poller.rb
+- lib/eventboss/middleware.rb
 - lib/eventboss/publisher.rb
 - lib/eventboss/queue.rb
 - lib/eventboss/queue_listener.rb
@@ -142,15 +145,18 @@ files:
 - lib/eventboss/scripts.rb
 - lib/eventboss/sender.rb
 - lib/eventboss/sns_client.rb
+- lib/eventboss/topic.rb
 - lib/eventboss/unit_of_work.rb
 - lib/eventboss/version.rb
 - lib/eventboss/worker.rb
+- lib/generators/eventboss/listener/eventboss_listener.rb.erb
+- lib/generators/eventboss/listener/listener_generator.rb
 - lib/tasks/eventboss.rake
 homepage: https://github.com/AirHelp/eventboss
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -166,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Eventboss Ruby Client.
 test_files: []

data/.github/workflows/ruby.yml

@@ -1,20 +0,0 @@
-name: Ruby
-
-on: [push]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Ruby 2.6
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.6.x
-    - name: Build and test with Rake
-      run: |
-        gem install bundler
-        bundle install --jobs 4 --retry 3
-        bundle exec rake