railway-ipc 2.0.2 → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38510e51a7320246c8351c026b49b77a72c3886d96f39735749bb2d8b0c580e8
-  data.tar.gz: '06182d40648d84f3c1a421073fc566fae21c8d966f241602ab29d260aea74d8e'
+  metadata.gz: a407ec255b81d23755164c8a2a45393092bd7d5eb2d39caa9e89125974f1822b
+  data.tar.gz: 4fd881a27222a11f767f64908aff24851c03a09a3f23cd1dfcf32ea7295c1971
 SHA512:
-  metadata.gz: 95f03fe787853fa46d71adba1bb452cbd6f514bae0c25a9c068df148769ac6aa1f22237162bdbbc6c099cc4f81623086be76d8b53083661eddfd3427835be5ba
-  data.tar.gz: f8a5736baf4ca3d84c57896d30db7deefcd26068b3c56c108b5f673d9e62368fea48fb9c9004479e92282edcdf77e86c0523374966f4e627530a665c6fec8f6f
+  metadata.gz: 79135cc5e3a2ee00ec1c79b00f168c5aa4ff7b9bf36c3aba6564c66a5644da1fe787f077443ebf5020660280cd98b58dbd2cc638b64962cc91663226ca87a359
+  data.tar.gz: 0ff544182b441fafc71d9be9bfdfc8dfc344dbe9142812559570d7f01cd57cc81d4311599d0a160951d45da9b81309f843f18ae57a3d4010282ba619034b1967

data/.gitignore

@@ -8,6 +8,9 @@
 /tmp/
 Gemfile.lock
 
+# built gems
+railway-ipc-*.gem
+
 # rspec failure tracking
 .rspec_status
 

data/CHANGELOG.md

@@ -10,6 +10,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 ### Fixed
 
+## [2.2.2] - 2020-11-20
+### Fixed
+* Fixed Publisher class channel leak. Channels were being created on each
+  instantiation of a Publisher instead of being re-used.
+
+## [2.2.1] - 2020-10-20
+### Added
+* Logging to indicate when options passed via `listen_to`/`from_queue` are overriding the defaults.
+
+## [2.2.0] - 2020-10-20
+### Added
+* The ability to configure workers to handle different workloads via `rake railway_ipc::consumers:spawn`
+
+## [2.1.0] - 2020-10-19
+### Added
+* :options parameter to `listen_to` which passes keys along to Sneaker's `from_queue` method.
+
+## [2.0.3] - 2020-09-02
+### Fixed
+* Fix RPC server. RPC servers need to conform to the Sneaker worker API (i.e. their initializers need to be able to accept queue name / pool and they require a `stop` method.
+
 ## [2.0.2] - 2020-08-27
 ### Changed
 * RPC `RailwayIpc::Client` does not need to log the queue name.
@@ -62,7 +83,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Correlation ID and message UUID are auto generated for messages for IDs are not passed in [#23](https://github.com/learn-co/railway_ipc_gem/pull/23)
 
-[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.2...HEAD
+[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.2...HEAD
+[2.2.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.1...v2.2.2
+[2.2.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.0...v2.2.1
+[2.2.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.1.0...v2.2.0
+[2.1.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.3...v2.1.0
+[2.0.3]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.2...v2.0.3
 [2.0.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.1...v2.0.2
 [2.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.0...v2.0.1
 [2.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v1.1.0...v2.0.0

data/README.md

@@ -52,6 +52,26 @@ Then, run your consumers
 bundle exec rake railway_ipc:consumers:start CONSUMERS=YourConsumer,YourOtherConsumer
 ```
 
+You may also configure your consumers more granularly using:
+
+`./config/sneaker_worker_groups.yml`
+```yaml
+HighPriority:
+    classes: YourConsumer,YourOtherConsumer
+    workers: 5
+LowPriority:
+    classes: YourThirdConsumer
+    workers: 1
+```
+
+```bash
+bundle exec rake railway_ipc:consumers:spawn
+```
+
+By default, `spawn` will map to `./config/sneaker_worker_groups.yml` but you can override it by using `WORKER_GROUP_CONFIG`.
+
+See the [Sneaker Documentation](https://github.com/jondot/sneakers/wiki/Handling-different-workloads) for more information.
+
 # Request/Response
 
 Define your server, client and responder. Docs coming soon.

data/lib/railway_ipc.rb

@@ -2,9 +2,10 @@
 
 require 'railway_ipc/version'
 require 'sneakers'
+require 'sneakers/spawner'
 require 'bunny'
 require 'active_record'
-require 'railway_ipc/version'
+require 'singleton'
 require 'railway_ipc/logger'
 require 'railway_ipc/unhandled_message_error'
 require 'railway_ipc/response'
@@ -14,6 +15,7 @@ require 'railway_ipc/rabbitmq/adapter'
 require 'railway_ipc/handler'
 require 'railway_ipc/handler_store'
 require 'railway_ipc/incoming_message'
+require 'railway_ipc/connection_manager'
 require 'railway_ipc/publisher'
 require 'railway_ipc/responder'
 require 'railway_ipc/rpc/rpc'
@@ -29,12 +31,16 @@ module RailwayIpc
     Rake::Task['sneakers:run'].invoke
   end
 
-  def self.configure(log_device=STDOUT, level=::Logger::INFO, log_formatter=nil)
+  def self.spawn
+    Sneakers::Spawner.spawn
+  end
+
+  def self.configure(log_device=$stdout, level=::Logger::INFO, log_formatter=nil)
     @logger = RailwayIpc::Logger.new(log_device, level, log_formatter)
   end
 
   def self.logger
-    @logger || RailwayIpc::Logger.new(STDOUT)
+    @logger || RailwayIpc::Logger.new($stdout)
   end
 
   def self.bunny_connection

data/lib/railway_ipc/connection_manager.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module RailwayIpc
+  # RabbitMQ connection manager. Ensures there is a single RabbitMQ
+  # connection and channel per thread, which prevents channel leaks.
+  #
+  class ConnectionManager
+    include Singleton
+
+    def initialize
+      establish_connection
+    end
+
+    def establish_connection
+      @connection = Bunny.new(
+        host: settings[:host],
+        user: settings[:user],
+        pass: settings[:pass],
+        port: settings[:port],
+        vhost: settings[:vhost] || '/',
+        logger: RailwayIpc.logger
+      )
+      @connection.start
+      @channel = @connection.create_channel
+
+      @connection
+    end
+
+    def channel
+      return @channel if connected?
+
+      establish_connection
+      @channel
+    end
+
+    def connected?
+      @connection&.connected? && @channel&.open?
+    end
+
+    private
+
+    def amqp_url
+      @amqp_url ||= ENV.fetch('RAILWAY_RABBITMQ_CONNECTION_URL', 'amqp://guest:guest@localhost:5672')
+    end
+
+    def settings
+      @settings ||= AMQ::Settings.parse_amqp_url(amqp_url)
+    end
+  end
+end

data/lib/railway_ipc/consumer/consumer.rb

@@ -3,7 +3,10 @@
 module RailwayIpc
   class Consumer
     include Sneakers::Worker
+
     def self.inherited(base)
+      super
+
       base.instance_eval do
         def handlers
           @handlers ||= RailwayIpc::HandlerStore.new
@@ -11,12 +14,21 @@ module RailwayIpc
       end
     end
 
-    def self.listen_to(queue:, exchange:)
-      from_queue queue,
-                 exchange: exchange,
-                 durable: true,
-                 exchange_type: :fanout,
-                 connection: RailwayIpc.bunny_connection
+    def self.listen_to(queue:, exchange:, options: {})
+      unless options.empty?
+        RailwayIpc.logger.info(
+          "Overriding configuration for #{queue} with new options",
+          feature: 'railway_ipc_consumer',
+          options: options
+        )
+      end
+
+      from_queue queue, {
+        exchange: exchange,
+        durable: true,
+        exchange_type: :fanout,
+        connection: RailwayIpc.bunny_connection
+      }.merge(options)
     end
 
     def self.handle(message_type, with:)

data/lib/railway_ipc/logger.rb

@@ -21,7 +21,7 @@ module RailwayIpc
   # logger = RailwayIpc::Logger.new(STDOUT, Logger::INFO, OjFormatter)
   #
   class Logger
-    def initialize(device=STDOUT, level=::Logger::INFO, formatter=nil)
+    def initialize(device=$stdout, level=::Logger::INFO, formatter=nil)
       @logger = ::Logger.new(device)
       @logger.level = level
       @logger.formatter = formatter if formatter
@@ -32,7 +32,7 @@ module RailwayIpc
         data.merge!(feature: 'railway_ipc') unless data.key?(:feature)
         return logger.send(level, data.merge(message: message)) unless block
 
-        data = message.merge(data) if message&.is_a?(Hash)
+        data = message.merge(data) if message.is_a?(Hash)
         data.merge!(message: block.call)
 
         # This is for compatability w/ Ruby's Logger. Ruby's Logger class
@@ -41,7 +41,7 @@ module RailwayIpc
         # is assumed to be the `progname`.
         #
         # https://github.com/ruby/logger/blob/master/lib/logger.rb#L471
-        data.merge!(progname: message) if message&.is_a?(String)
+        data.merge!(progname: message) if message.is_a?(String)
         logger.send(level, data)
       end
     end

data/lib/railway_ipc/publisher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'singleton'
-
 module RailwayIpc
   class SingletonPublisher < Sneakers::Publisher
     include ::Singleton
@@ -53,19 +51,12 @@ module RailwayIpc
 end
 
 module RailwayIpc
-  class Publisher < Sneakers::Publisher
+  class Publisher
     attr_reader :exchange_name, :message_store
 
     def initialize(opts={})
       @exchange_name = opts.fetch(:exchange_name)
       @message_store = opts.fetch(:message_store, RailwayIpc::PublishedMessage)
-      connection = opts.fetch(:connection, nil)
-      options = {
-        exchange: exchange_name,
-        connection: connection,
-        exchange_type: :fanout
-      }.compact
-      super(options)
     end
 
     # rubocop:disable Metrics/AbcSize
@@ -75,7 +66,7 @@ module RailwayIpc
       RailwayIpc.logger.info('Publishing message', log_message_options(message))
 
       stored_message = message_store.store_message(exchange_name, message)
-      super(RailwayIpc::Rabbitmq::Payload.encode(message))
+      exchange.publish(RailwayIpc::Rabbitmq::Payload.encode(message))
     rescue RailwayIpc::InvalidProtobuf => e
       RailwayIpc.logger.error('Invalid protobuf', log_message_options(message))
       raise e
@@ -88,8 +79,16 @@ module RailwayIpc
     end
     # rubocop:enable Metrics/AbcSize
 
+    def exchange
+      @exchange ||= channel.exchange(exchange_name, type: :fanout, durable: true, auto_delete: false, arguments: {})
+    end
+
     private
 
+    def channel
+      RailwayIpc::ConnectionManager.instance.channel
+    end
+
     def log_message_options(message)
       {
         feature: 'railway_ipc_publisher',

data/lib/railway_ipc/rabbitmq/adapter.rb

@@ -16,7 +16,7 @@ module RailwayIpc
                      :port,
                      :user
 
-      def initialize(amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], exchange_name:, queue_name: '', options: {})
+      def initialize(exchange_name:, amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], queue_name: '', options: {})
         @queue_name = queue_name
         @exchange_name = exchange_name
         settings = AMQ::Settings.parse_amqp_url(amqp_url)

data/lib/railway_ipc/rpc/concerns/message_observation_configurable.rb

@@ -3,7 +3,7 @@
 module RailwayIpc
   module RPC
     module MessageObservationConfigurable
-      def listen_to(exchange:, queue:)
+      def listen_to(queue:, exchange:)
         @exchange_name = exchange
         @queue_name = queue
       end

data/lib/railway_ipc/rpc/server/server.rb

@@ -14,7 +14,7 @@ module RailwayIpc
       RailwayIpc::RPC::ServerResponseHandlers.instance.register(handler: with, message: message_type)
     end
 
-    def initialize(opts={ automatic_recovery: true }, rabbit_adapter: RailwayIpc::Rabbitmq::Adapter)
+    def initialize(_queue, _pool, opts={ automatic_recovery: true }, rabbit_adapter: RailwayIpc::Rabbitmq::Adapter)
       @rabbit_connection = rabbit_adapter.new(
         queue_name: self.class.queue_name,
         exchange_name: self.class.exchange_name,
@@ -31,6 +31,10 @@ module RailwayIpc
       subscribe_to_queue
     end
 
+    def stop
+      rabbit_connection.disconnect
+    end
+
     # rubocop:disable Metrics/AbcSize
     # rubocop:disable Metrics/MethodLength
     def work(payload)

data/lib/railway_ipc/tasks/start_consumers.rake

@@ -2,9 +2,15 @@
 
 namespace :railway_ipc do
   namespace :consumers do
+    desc 'Start consumers via explicitly defining them'
     task :start do
       ENV['WORKERS'] = ENV['CONSUMERS']
       RailwayIpc.start
     end
+
+    desc 'Start consumers via ./config/sneaker_worker_groups.yml'
+    task :spawn do
+      RailwayIpc.spawn
+    end
   end
 end

data/lib/railway_ipc/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailwayIpc
-  VERSION = '2.0.2'
+  VERSION = '2.2.2'
 end

data/railway_ipc.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '2.0.1'
+  spec.add_development_dependency 'bundler', '2.1.4'
   spec.add_development_dependency 'factory_bot', '~> 5.1'
   spec.add_development_dependency 'google-protobuf', '~> 3.9'
   spec.add_development_dependency 'rake', '>= 10.0.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: railway-ipc
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.2.2
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-27 00:00:00.000000000 Z
+date: 2020-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 2.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 2.1.4
 - !ruby/object:Gem::Dependency
   name: factory_bot
   requirement: !ruby/object:Gem::Requirement
@@ -225,6 +225,7 @@ files:
 - bin/setup
 - lib/railway_ipc.rb
 - lib/railway_ipc/Rakefile
+- lib/railway_ipc/connection_manager.rb
 - lib/railway_ipc/consumer/consumer.rb
 - lib/railway_ipc/consumer/process_incoming_message.rb
 - lib/railway_ipc/errors.rb