railway-ipc 2.1.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b65c6b026a85e5cd4ae6693711ec05d9efe714517c2a05c3bfd7111bde92eab
-  data.tar.gz: 944093374956d0ad82460ebbddb61f952d3ac70dbdb8294f032dd63ee2e0df9e
+  metadata.gz: 7f349dc76232b1cef6492bb2dc09cdbd58bf2ad22be64bfd1771e4e6d7192b16
+  data.tar.gz: 3e04b076735ca1bcc8d1cc376925311b1547b8b13a554691726f348a62625cfc
 SHA512:
-  metadata.gz: 7fad19c0a2c6bc99a7e56242eaa70b6c5f449d9c641372cc4088cc5aaba1f8ac3cfcc45f6e8b003c34713c65342b676114dd7ada579ed8d8bf3b31a94838d614
-  data.tar.gz: 27e95ff1100792b27e07414ee681062ceb54f12c7c31308e022045184031c15793f5f187b9dfbe51746ffe0bd4cb4ab0bf1d81163cb04126598fba15d720a6ec
+  metadata.gz: d0dd3263f9ca6c57da0fd98c25b720f1f7bad7ec87bb4c34cd110d51824b350ca255d9a467170d420aeb323faff081f03ea62c18c080f3a52fd72f0c687be49a
+  data.tar.gz: 74d6f196446603d93d42050e2c672ccac22ec49cedbc2d026cf8f8f61eb0bc4d6537b179ad2d60cd37426ad6b60ab9e428d40b21f69ce21426cc86cbeefaacaf

data/.gitignore

@@ -4,10 +4,14 @@
 /coverage/
 /doc/
 /pkg/
+/rake
 /spec/reports/
 /tmp/
 Gemfile.lock
 
+# built gems
+railway-ipc-*.gem
+
 # rspec failure tracking
 .rspec_status
 

data/.ruby-version

@@ -0,0 +1 @@
+2.6.5

data/CHANGELOG.md

@@ -10,6 +10,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 ### Fixed
 
+## [4.0.0] - 2021-01-11
+### Added
+* JSON decoder for consumers that can handle JSON encoded Protobufs. Note that the publishers do not (yet) have the option of encoding the messages as JSON.
+
+### Changed
+* (Breaking Change) Rename `Consumer#work` to `Consumer#work_with_params`. This was necessary so that we can support specifying different message encodings via metadata in the future. If the message encoding cannot be determined from the message metadata fall back to a default decoder (binary protobufs).
+
+### Fixed
+* `./bin/console` script was broken because Pry wasn't a dependency; added Pry as a development dependency only.
+
+## [3.0.0] - 2020-12-07
+### Changed
+* Consumers _will no longer crash_ when an exception is raised. Instead, consumers will move the message that caused the exception to a single dead-letter exchange called 'ipc:errors'. Railway will configure the dead-letter exchange automatically.
+
+## [2.2.2] - 2020-11-20
+### Fixed
+* Fixed Publisher class channel leak. Channels were being created on each instantiation of a Publisher instead of being re-used.
+
+## [2.2.1] - 2020-10-20
+### Added
+* Logging to indicate when options passed via `listen_to`/`from_queue` are overriding the defaults.
+
+## [2.2.0] - 2020-10-20
+### Added
+* The ability to configure workers to handle different workloads via `rake railway_ipc::consumers:spawn`
+
+## [2.1.0] - 2020-10-19
+### Added
+* :options parameter to `listen_to` which passes keys along to Sneaker's `from_queue` method.
+
 ## [2.0.3] - 2020-09-02
 ### Fixed
 * Fix RPC server. RPC servers need to conform to the Sneaker worker API (i.e. their initializers need to be able to accept queue name / pool and they require a `stop` method.
@@ -66,7 +96,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Correlation ID and message UUID are auto generated for messages for IDs are not passed in [#23](https://github.com/learn-co/railway_ipc_gem/pull/23)
 
-[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.3...HEAD
+[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v4.0.0...HEAD
+[4.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v3.0.0...v4.0.0
+[3.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.2...v3.0.0
+[2.2.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.1...v2.2.2
+[2.2.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.0...v2.2.1
+[2.2.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.1.0...v2.2.0
+[2.1.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.3...v2.1.0
 [2.0.3]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.2...v2.0.3
 [2.0.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.1...v2.0.2
 [2.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.0...v2.0.1

data/README.md

@@ -52,6 +52,26 @@ Then, run your consumers
 bundle exec rake railway_ipc:consumers:start CONSUMERS=YourConsumer,YourOtherConsumer
 ```
 
+You may also configure your consumers more granularly using:
+
+`./config/sneaker_worker_groups.yml`
+```yaml
+HighPriority:
+    classes: YourConsumer,YourOtherConsumer
+    workers: 5
+LowPriority:
+    classes: YourThirdConsumer
+    workers: 1
+```
+
+```bash
+bundle exec rake railway_ipc:consumers:spawn
+```
+
+By default, `spawn` will map to `./config/sneaker_worker_groups.yml` but you can override it by using `WORKER_GROUP_CONFIG`.
+
+See the [Sneaker Documentation](https://github.com/jondot/sneakers/wiki/Handling-different-workloads) for more information.
+
 # Request/Response
 
 Define your server, client and responder. Docs coming soon.

data/lib/railway_ipc.rb

@@ -2,9 +2,10 @@
 
 require 'railway_ipc/version'
 require 'sneakers'
+require 'sneakers/spawner'
 require 'bunny'
 require 'active_record'
-require 'railway_ipc/version'
+require 'singleton'
 require 'railway_ipc/logger'
 require 'railway_ipc/unhandled_message_error'
 require 'railway_ipc/response'
@@ -13,7 +14,9 @@ require 'railway_ipc/unknown_message.pb'
 require 'railway_ipc/rabbitmq/adapter'
 require 'railway_ipc/handler'
 require 'railway_ipc/handler_store'
+require 'railway_ipc/message_decoders'
 require 'railway_ipc/incoming_message'
+require 'railway_ipc/connection_manager'
 require 'railway_ipc/publisher'
 require 'railway_ipc/responder'
 require 'railway_ipc/rpc/rpc'
@@ -29,12 +32,16 @@ module RailwayIpc
     Rake::Task['sneakers:run'].invoke
   end
 
-  def self.configure(log_device=STDOUT, level=::Logger::INFO, log_formatter=nil)
+  def self.spawn
+    Sneakers::Spawner.spawn
+  end
+
+  def self.configure(log_device=$stdout, level=::Logger::INFO, log_formatter=nil)
     @logger = RailwayIpc::Logger.new(log_device, level, log_formatter)
   end
 
   def self.logger
-    @logger || RailwayIpc::Logger.new(STDOUT)
+    @logger || RailwayIpc::Logger.new($stdout)
   end
 
   def self.bunny_connection

data/lib/railway_ipc/connection_manager.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module RailwayIpc
+  # RabbitMQ connection manager. Ensures there is a single RabbitMQ
+  # connection and channel per thread, which prevents channel leaks.
+  #
+  class ConnectionManager
+    include Singleton
+
+    def initialize
+      establish_connection
+    end
+
+    def establish_connection
+      @connection = Bunny.new(
+        host: settings[:host],
+        user: settings[:user],
+        pass: settings[:pass],
+        port: settings[:port],
+        vhost: settings[:vhost] || '/',
+        logger: RailwayIpc.logger
+      )
+      @connection.start
+      @channel = @connection.create_channel
+
+      @connection
+    end
+
+    def channel
+      return @channel if connected?
+
+      establish_connection
+      @channel
+    end
+
+    def connected?
+      @connection&.connected? && @channel&.open?
+    end
+
+    private
+
+    def amqp_url
+      @amqp_url ||= ENV.fetch('RAILWAY_RABBITMQ_CONNECTION_URL', 'amqp://guest:guest@localhost:5672')
+    end
+
+    def settings
+      @settings ||= AMQ::Settings.parse_amqp_url(amqp_url)
+    end
+  end
+end

data/lib/railway_ipc/consumer/consumer.rb

@@ -5,6 +5,8 @@ module RailwayIpc
     include Sneakers::Worker
 
     def self.inherited(base)
+      super
+
       base.instance_eval do
         def handlers
           @handlers ||= RailwayIpc::HandlerStore.new
@@ -12,14 +14,27 @@ module RailwayIpc
       end
     end
 
+    # rubocop:disable Metrics/MethodLength
     def self.listen_to(queue:, exchange:, options: {})
+      unless options.empty?
+        RailwayIpc.logger.info(
+          "Overriding configuration for #{queue} with new options",
+          feature: 'railway_ipc_consumer',
+          options: options
+        )
+      end
+
       from_queue queue, {
         exchange: exchange,
         durable: true,
         exchange_type: :fanout,
+        arguments: {
+          'x-dead-letter-exchange' => 'ipc:errors'
+        },
         connection: RailwayIpc.bunny_connection
       }.merge(options)
     end
+    # rubocop:enable Metrics/MethodLength
 
     def self.handle(message_type, with:)
       handlers.register(message: message_type, handler: with)
@@ -41,8 +56,14 @@ module RailwayIpc
       queue.opts[:exchange]
     end
 
-    def work(payload)
-      message = RailwayIpc::IncomingMessage.new(payload)
+    # REFACTOR: Long term we should think about not leaking Sneakers
+    # methods as part of Railway's public API since clients can (and do)
+    # override them. -BN
+    def work_with_params(payload, _delivery_info, metadata)
+      message_format = metadata.fetch('headers', {})
+                               .fetch('message_format', 'protobuf_binary')
+
+      message = RailwayIpc::IncomingMessage.new(payload, message_format: message_format)
       RailwayIpc::ProcessIncomingMessage.call(self, message)
       ack!
     rescue StandardError => e
@@ -54,7 +75,7 @@ module RailwayIpc
         error: e.class,
         payload: payload
       )
-      raise e
+      reject!
     end
 
     def get_handler(type)

data/lib/railway_ipc/incoming_message.rb

@@ -2,9 +2,10 @@
 
 module RailwayIpc
   class IncomingMessage
-    attr_reader :type, :payload, :parsed_payload, :errors
+    attr_reader :type, :message_format, :payload, :parsed_payload, :errors
 
-    def initialize(payload)
+    def initialize(payload, message_format: nil)
+      @message_format = message_format
       @parsed_payload = JSON.parse(payload)
       @type = parsed_payload['type']
       @payload = payload
@@ -32,20 +33,23 @@ module RailwayIpc
     end
 
     def decoded
-      @decoded ||=
-        begin
-          protobuf_msg = Base64.decode64(parsed_payload['encoded_message'])
-          decoder = Kernel.const_get(type)
-          decoder.decode(protobuf_msg)
-        rescue Google::Protobuf::ParseError => e
-          raise RailwayIpc::IncomingMessage::ParserError.new(e)
-        rescue NameError
-          RailwayIpc::Messages::Unknown.decode(protobuf_msg)
-        end
+      @decoded ||= \
+        get_decoder(message_format).call(type, parsed_payload['encoded_message'])
     end
 
     def stringify_errors
       errors.values.join(', ')
     end
+
+    private
+
+    DEFAULT_DECODER = RailwayIpc::MessageDecoders::ProtobufBinaryDecoder
+
+    def get_decoder(name)
+      {
+        'protobuf_binary' => RailwayIpc::MessageDecoders::ProtobufBinaryDecoder,
+        'protobuf_json' => RailwayIpc::MessageDecoders::ProtobufJsonDecoder
+      }.fetch(name, DEFAULT_DECODER)
+    end
   end
 end

data/lib/railway_ipc/logger.rb

@@ -21,7 +21,7 @@ module RailwayIpc
   # logger = RailwayIpc::Logger.new(STDOUT, Logger::INFO, OjFormatter)
   #
   class Logger
-    def initialize(device=STDOUT, level=::Logger::INFO, formatter=nil)
+    def initialize(device=$stdout, level=::Logger::INFO, formatter=nil)
       @logger = ::Logger.new(device)
       @logger.level = level
       @logger.formatter = formatter if formatter
@@ -32,7 +32,7 @@ module RailwayIpc
         data.merge!(feature: 'railway_ipc') unless data.key?(:feature)
         return logger.send(level, data.merge(message: message)) unless block
 
-        data = message.merge(data) if message&.is_a?(Hash)
+        data = message.merge(data) if message.is_a?(Hash)
         data.merge!(message: block.call)
 
         # This is for compatability w/ Ruby's Logger. Ruby's Logger class
@@ -41,7 +41,7 @@ module RailwayIpc
         # is assumed to be the `progname`.
         #
         # https://github.com/ruby/logger/blob/master/lib/logger.rb#L471
-        data.merge!(progname: message) if message&.is_a?(String)
+        data.merge!(progname: message) if message.is_a?(String)
         logger.send(level, data)
       end
     end

data/lib/railway_ipc/message_decoders.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RailwayIpc
+  module MessageDecoders
+    ProtobufBinaryDecoder = lambda do |type, encoded_message|
+      protobuf_msg = Base64.decode64(encoded_message)
+      protobuf_klass = Kernel.const_get(type)
+      protobuf_klass.decode(protobuf_msg)
+    rescue Google::Protobuf::ParseError => e
+      raise RailwayIpc::IncomingMessage::ParserError.new(e)
+    rescue NameError
+      RailwayIpc::Messages::Unknown.decode(protobuf_msg)
+    end
+
+    ProtobufJsonDecoder = lambda do |type, message_hash|
+      protobuf_klass = Kernel.const_get(type)
+      protobuf_klass.new(message_hash)
+    rescue ArgumentError => e
+      raise RailwayIpc::IncomingMessage::ParserError.new(e)
+    rescue NameError
+      # NOTE: I didn't realize this until I made this ProtobufJsonDecoder, but
+      # the ProtobufBinaryDecoder will ignore any unknown keys -- which is
+      # probably not what we want. I'm coding this the same way as the binary
+      # protobuf version for consistency, but we should re-think how we want to
+      # handle this situation. -BN
+      RailwayIpc::Messages::Unknown.new(
+        user_uuid: message_hash.fetch(:user_uuid, ''),
+        correlation_id: message_hash.fetch(:correlation_id, ''),
+        uuid: message_hash.fetch(:uuid, ''),
+        context: message_hash.fetch(:context, {})
+      )
+    end
+  end
+end

data/lib/railway_ipc/publisher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'singleton'
-
 module RailwayIpc
   class SingletonPublisher < Sneakers::Publisher
     include ::Singleton
@@ -53,19 +51,12 @@ module RailwayIpc
 end
 
 module RailwayIpc
-  class Publisher < Sneakers::Publisher
+  class Publisher
     attr_reader :exchange_name, :message_store
 
     def initialize(opts={})
       @exchange_name = opts.fetch(:exchange_name)
       @message_store = opts.fetch(:message_store, RailwayIpc::PublishedMessage)
-      connection = opts.fetch(:connection, nil)
-      options = {
-        exchange: exchange_name,
-        connection: connection,
-        exchange_type: :fanout
-      }.compact
-      super(options)
     end
 
     # rubocop:disable Metrics/AbcSize
@@ -75,7 +66,7 @@ module RailwayIpc
       RailwayIpc.logger.info('Publishing message', log_message_options(message))
 
       stored_message = message_store.store_message(exchange_name, message)
-      super(RailwayIpc::Rabbitmq::Payload.encode(message))
+      exchange.publish(RailwayIpc::Rabbitmq::Payload.encode(message))
     rescue RailwayIpc::InvalidProtobuf => e
       RailwayIpc.logger.error('Invalid protobuf', log_message_options(message))
       raise e
@@ -88,8 +79,16 @@ module RailwayIpc
     end
     # rubocop:enable Metrics/AbcSize
 
+    def exchange
+      @exchange ||= channel.exchange(exchange_name, type: :fanout, durable: true, auto_delete: false, arguments: {})
+    end
+
     private
 
+    def channel
+      RailwayIpc::ConnectionManager.instance.channel
+    end
+
     def log_message_options(message)
       {
         feature: 'railway_ipc_publisher',

data/lib/railway_ipc/rabbitmq/adapter.rb

@@ -16,7 +16,7 @@ module RailwayIpc
                      :port,
                      :user
 
-      def initialize(amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], exchange_name:, queue_name: '', options: {})
+      def initialize(exchange_name:, amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], queue_name: '', options: {})
         @queue_name = queue_name
         @exchange_name = exchange_name
         settings = AMQ::Settings.parse_amqp_url(amqp_url)

data/lib/railway_ipc/tasks/start_consumers.rake

@@ -2,9 +2,15 @@
 
 namespace :railway_ipc do
   namespace :consumers do
+    desc 'Start consumers via explicitly defining them'
     task :start do
       ENV['WORKERS'] = ENV['CONSUMERS']
       RailwayIpc.start
     end
+
+    desc 'Start consumers via ./config/sneaker_worker_groups.yml'
+    task :spawn do
+      RailwayIpc.spawn
+    end
   end
 end

data/lib/railway_ipc/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailwayIpc
-  VERSION = '2.1.0'
+  VERSION = '4.0.0'
 end

data/railway_ipc.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'database_cleaner', '~> 1.7'
   spec.add_development_dependency 'listen', '~> 3.0.5'
   spec.add_development_dependency 'pg', '~> 0.18'
+  spec.add_development_dependency 'pry', '~> 0.13'
   spec.add_development_dependency 'rails', '~> 5.0.7'
   spec.add_development_dependency 'rspec-rails'
   spec.add_development_dependency 'shoulda-matchers', '~> 4.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: railway-ipc
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 4.0.0
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-19 00:00:00.000000000 Z
+date: 2021-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -213,6 +227,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -225,6 +240,7 @@ files:
 - bin/setup
 - lib/railway_ipc.rb
 - lib/railway_ipc/Rakefile
+- lib/railway_ipc/connection_manager.rb
 - lib/railway_ipc/consumer/consumer.rb
 - lib/railway_ipc/consumer/process_incoming_message.rb
 - lib/railway_ipc/errors.rb
@@ -232,6 +248,7 @@ files:
 - lib/railway_ipc/handler_store.rb
 - lib/railway_ipc/incoming_message.rb
 - lib/railway_ipc/logger.rb
+- lib/railway_ipc/message_decoders.rb
 - lib/railway_ipc/models/consumed_message.rb
 - lib/railway_ipc/models/published_message.rb
 - lib/railway_ipc/publisher.rb