railway-ipc 2.0.3 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 444dfeb8a9729ca23276ce6f4a899589a646c93d3671fb517ab81d0fdc85958e
-  data.tar.gz: 9ede7408f2a428cf3345ef3b8aa684c6aa3fe43a6d460ef7eb6866a99f46fe71
+  metadata.gz: 5002df116daa50da3bfcefc3a966901a49c04671e869f23a645482ef9cf4c619
+  data.tar.gz: de49b0064f5b57f77893f46701007928a3e1c0fa4644a34041de50c6c4113f18
 SHA512:
-  metadata.gz: d15afcbeec8272d7e9b3c9e589f7ed88ffc8fb50fbf370cbaabf596a753e435bd7c67980668c7ce27d13b1b4dff7adf2b4b11f3b7c2f28ec224295bfc7f8ec69
-  data.tar.gz: 1d39cf3abce42366a096a83d15f8a287e32fbb7ef239fd17f04707ef0eed9fc7c5e3c06e129b4e4345871def82914a740fc6cab5039cc5d2934886342207aa67
+  metadata.gz: 9ed5098e317e039d3e4ed0305aca7f571be35f1ace0d88a513ef702ec381f10fe8291378b00ca181a6d23cfcbc7882c367dacedfb68d458ff71f9c8f808128d3
+  data.tar.gz: a30c8d4795fcd839588a8e9fc375c027b4a36e48b64eee9800a659fa4380f2d89d66f89c838c80fb70bf2072917146bcb6bd4ad4917141fa4dcfc0cfdf89db14

data/.gitignore

@@ -8,6 +8,9 @@
 /tmp/
 Gemfile.lock
 
+# built gems
+railway-ipc-*.gem
+
 # rspec failure tracking
 .rspec_status
 

data/CHANGELOG.md

@@ -10,6 +10,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 ### Fixed
 
+## [3.0.0] - 2020-12-07
+### Changed
+* Consumers _will no longer crash_ when an exception is raised. Instead, consumers will move the message that caused the exception to a single dead-letter exchange called 'ipc:errors'. Railway will configure the dead-letter exchange automatically.
+
+## [2.2.2] - 2020-11-20
+### Fixed
+* Fixed Publisher class channel leak. Channels were being created on each instantiation of a Publisher instead of being re-used.
+
+## [2.2.1] - 2020-10-20
+### Added
+* Logging to indicate when options passed via `listen_to`/`from_queue` are overriding the defaults.
+
+## [2.2.0] - 2020-10-20
+### Added
+* The ability to configure workers to handle different workloads via `rake railway_ipc::consumers:spawn`
+
+## [2.1.0] - 2020-10-19
+### Added
+* :options parameter to `listen_to` which passes keys along to Sneaker's `from_queue` method.
+
 ## [2.0.3] - 2020-09-02
 ### Fixed
 * Fix RPC server. RPC servers need to conform to the Sneaker worker API (i.e. their initializers need to be able to accept queue name / pool and they require a `stop` method.
@@ -66,7 +86,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Correlation ID and message UUID are auto generated for messages for IDs are not passed in [#23](https://github.com/learn-co/railway_ipc_gem/pull/23)
 
-[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.3...HEAD
+[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v30.0.0...HEAD
+[3.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.2...v3.0.0
+[2.2.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.1...v2.2.2
+[2.2.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.0...v2.2.1
+[2.2.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.1.0...v2.2.0
+[2.1.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.3...v2.1.0
 [2.0.3]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.2...v2.0.3
 [2.0.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.1...v2.0.2
 [2.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.0...v2.0.1

data/README.md

@@ -52,6 +52,26 @@ Then, run your consumers
 bundle exec rake railway_ipc:consumers:start CONSUMERS=YourConsumer,YourOtherConsumer
 ```
 
+You may also configure your consumers more granularly using:
+
+`./config/sneaker_worker_groups.yml`
+```yaml
+HighPriority:
+    classes: YourConsumer,YourOtherConsumer
+    workers: 5
+LowPriority:
+    classes: YourThirdConsumer
+    workers: 1
+```
+
+```bash
+bundle exec rake railway_ipc:consumers:spawn
+```
+
+By default, `spawn` will map to `./config/sneaker_worker_groups.yml` but you can override it by using `WORKER_GROUP_CONFIG`.
+
+See the [Sneaker Documentation](https://github.com/jondot/sneakers/wiki/Handling-different-workloads) for more information.
+
 # Request/Response
 
 Define your server, client and responder. Docs coming soon.

data/lib/railway_ipc.rb

@@ -2,9 +2,10 @@
 
 require 'railway_ipc/version'
 require 'sneakers'
+require 'sneakers/spawner'
 require 'bunny'
 require 'active_record'
-require 'railway_ipc/version'
+require 'singleton'
 require 'railway_ipc/logger'
 require 'railway_ipc/unhandled_message_error'
 require 'railway_ipc/response'
@@ -14,6 +15,7 @@ require 'railway_ipc/rabbitmq/adapter'
 require 'railway_ipc/handler'
 require 'railway_ipc/handler_store'
 require 'railway_ipc/incoming_message'
+require 'railway_ipc/connection_manager'
 require 'railway_ipc/publisher'
 require 'railway_ipc/responder'
 require 'railway_ipc/rpc/rpc'
@@ -29,12 +31,16 @@ module RailwayIpc
     Rake::Task['sneakers:run'].invoke
   end
 
-  def self.configure(log_device=STDOUT, level=::Logger::INFO, log_formatter=nil)
+  def self.spawn
+    Sneakers::Spawner.spawn
+  end
+
+  def self.configure(log_device=$stdout, level=::Logger::INFO, log_formatter=nil)
     @logger = RailwayIpc::Logger.new(log_device, level, log_formatter)
   end
 
   def self.logger
-    @logger || RailwayIpc::Logger.new(STDOUT)
+    @logger || RailwayIpc::Logger.new($stdout)
   end
 
   def self.bunny_connection

data/lib/railway_ipc/connection_manager.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module RailwayIpc
+  # RabbitMQ connection manager. Ensures there is a single RabbitMQ
+  # connection and channel per thread, which prevents channel leaks.
+  #
+  class ConnectionManager
+    include Singleton
+
+    def initialize
+      establish_connection
+    end
+
+    def establish_connection
+      @connection = Bunny.new(
+        host: settings[:host],
+        user: settings[:user],
+        pass: settings[:pass],
+        port: settings[:port],
+        vhost: settings[:vhost] || '/',
+        logger: RailwayIpc.logger
+      )
+      @connection.start
+      @channel = @connection.create_channel
+
+      @connection
+    end
+
+    def channel
+      return @channel if connected?
+
+      establish_connection
+      @channel
+    end
+
+    def connected?
+      @connection&.connected? && @channel&.open?
+    end
+
+    private
+
+    def amqp_url
+      @amqp_url ||= ENV.fetch('RAILWAY_RABBITMQ_CONNECTION_URL', 'amqp://guest:guest@localhost:5672')
+    end
+
+    def settings
+      @settings ||= AMQ::Settings.parse_amqp_url(amqp_url)
+    end
+  end
+end

data/lib/railway_ipc/consumer/consumer.rb

@@ -3,7 +3,10 @@
 module RailwayIpc
   class Consumer
     include Sneakers::Worker
+
     def self.inherited(base)
+      super
+
       base.instance_eval do
         def handlers
           @handlers ||= RailwayIpc::HandlerStore.new
@@ -11,13 +14,27 @@ module RailwayIpc
       end
     end
 
-    def self.listen_to(queue:, exchange:)
-      from_queue queue,
-                 exchange: exchange,
-                 durable: true,
-                 exchange_type: :fanout,
-                 connection: RailwayIpc.bunny_connection
+    # rubocop:disable Metrics/MethodLength
+    def self.listen_to(queue:, exchange:, options: {})
+      unless options.empty?
+        RailwayIpc.logger.info(
+          "Overriding configuration for #{queue} with new options",
+          feature: 'railway_ipc_consumer',
+          options: options
+        )
+      end
+
+      from_queue queue, {
+        exchange: exchange,
+        durable: true,
+        exchange_type: :fanout,
+        arguments: {
+          'x-dead-letter-exchange' => 'ipc:errors'
+        },
+        connection: RailwayIpc.bunny_connection
+      }.merge(options)
     end
+    # rubocop:enable Metrics/MethodLength
 
     def self.handle(message_type, with:)
       handlers.register(message: message_type, handler: with)
@@ -52,7 +69,7 @@ module RailwayIpc
         error: e.class,
         payload: payload
       )
-      raise e
+      reject!
     end
 
     def get_handler(type)

data/lib/railway_ipc/logger.rb

@@ -21,7 +21,7 @@ module RailwayIpc
   # logger = RailwayIpc::Logger.new(STDOUT, Logger::INFO, OjFormatter)
   #
   class Logger
-    def initialize(device=STDOUT, level=::Logger::INFO, formatter=nil)
+    def initialize(device=$stdout, level=::Logger::INFO, formatter=nil)
       @logger = ::Logger.new(device)
       @logger.level = level
       @logger.formatter = formatter if formatter
@@ -32,7 +32,7 @@ module RailwayIpc
         data.merge!(feature: 'railway_ipc') unless data.key?(:feature)
         return logger.send(level, data.merge(message: message)) unless block
 
-        data = message.merge(data) if message&.is_a?(Hash)
+        data = message.merge(data) if message.is_a?(Hash)
         data.merge!(message: block.call)
 
         # This is for compatability w/ Ruby's Logger. Ruby's Logger class
@@ -41,7 +41,7 @@ module RailwayIpc
         # is assumed to be the `progname`.
         #
         # https://github.com/ruby/logger/blob/master/lib/logger.rb#L471
-        data.merge!(progname: message) if message&.is_a?(String)
+        data.merge!(progname: message) if message.is_a?(String)
         logger.send(level, data)
       end
     end

data/lib/railway_ipc/publisher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'singleton'
-
 module RailwayIpc
   class SingletonPublisher < Sneakers::Publisher
     include ::Singleton
@@ -53,19 +51,12 @@ module RailwayIpc
 end
 
 module RailwayIpc
-  class Publisher < Sneakers::Publisher
+  class Publisher
     attr_reader :exchange_name, :message_store
 
     def initialize(opts={})
       @exchange_name = opts.fetch(:exchange_name)
       @message_store = opts.fetch(:message_store, RailwayIpc::PublishedMessage)
-      connection = opts.fetch(:connection, nil)
-      options = {
-        exchange: exchange_name,
-        connection: connection,
-        exchange_type: :fanout
-      }.compact
-      super(options)
     end
 
     # rubocop:disable Metrics/AbcSize
@@ -75,7 +66,7 @@ module RailwayIpc
       RailwayIpc.logger.info('Publishing message', log_message_options(message))
 
       stored_message = message_store.store_message(exchange_name, message)
-      super(RailwayIpc::Rabbitmq::Payload.encode(message))
+      exchange.publish(RailwayIpc::Rabbitmq::Payload.encode(message))
     rescue RailwayIpc::InvalidProtobuf => e
       RailwayIpc.logger.error('Invalid protobuf', log_message_options(message))
       raise e
@@ -88,8 +79,16 @@ module RailwayIpc
     end
     # rubocop:enable Metrics/AbcSize
 
+    def exchange
+      @exchange ||= channel.exchange(exchange_name, type: :fanout, durable: true, auto_delete: false, arguments: {})
+    end
+
     private
 
+    def channel
+      RailwayIpc::ConnectionManager.instance.channel
+    end
+
     def log_message_options(message)
       {
         feature: 'railway_ipc_publisher',

data/lib/railway_ipc/rabbitmq/adapter.rb

@@ -16,7 +16,7 @@ module RailwayIpc
                      :port,
                      :user
 
-      def initialize(amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], exchange_name:, queue_name: '', options: {})
+      def initialize(exchange_name:, amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], queue_name: '', options: {})
         @queue_name = queue_name
         @exchange_name = exchange_name
         settings = AMQ::Settings.parse_amqp_url(amqp_url)

data/lib/railway_ipc/rpc/concerns/message_observation_configurable.rb

@@ -3,7 +3,7 @@
 module RailwayIpc
   module RPC
     module MessageObservationConfigurable
-      def listen_to(exchange:, queue:)
+      def listen_to(queue:, exchange:)
         @exchange_name = exchange
         @queue_name = queue
       end

data/lib/railway_ipc/tasks/start_consumers.rake

@@ -2,9 +2,15 @@
 
 namespace :railway_ipc do
   namespace :consumers do
+    desc 'Start consumers via explicitly defining them'
     task :start do
       ENV['WORKERS'] = ENV['CONSUMERS']
       RailwayIpc.start
     end
+
+    desc 'Start consumers via ./config/sneaker_worker_groups.yml'
+    task :spawn do
+      RailwayIpc.spawn
+    end
   end
 end

data/lib/railway_ipc/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailwayIpc
-  VERSION = '2.0.3'
+  VERSION = '3.0.0'
 end

data/railway_ipc.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '2.0.1'
+  spec.add_development_dependency 'bundler', '2.1.4'
   spec.add_development_dependency 'factory_bot', '~> 5.1'
   spec.add_development_dependency 'google-protobuf', '~> 3.9'
   spec.add_development_dependency 'rake', '>= 10.0.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: railway-ipc
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 3.0.0
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-02 00:00:00.000000000 Z
+date: 2020-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 2.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 2.1.4
 - !ruby/object:Gem::Dependency
   name: factory_bot
   requirement: !ruby/object:Gem::Requirement
@@ -225,6 +225,7 @@ files:
 - bin/setup
 - lib/railway_ipc.rb
 - lib/railway_ipc/Rakefile
+- lib/railway_ipc/connection_manager.rb
 - lib/railway_ipc/consumer/consumer.rb
 - lib/railway_ipc/consumer/process_incoming_message.rb
 - lib/railway_ipc/errors.rb
@@ -278,8 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key:
 specification_version: 4
 summary: IPC components for Rails