railway-ipc 2.2.1 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8153f89992dd3713ca4b6e8792a69d91e80738ae69d26bd3325616c30d854bf2
-  data.tar.gz: d6680bb1330b885172730f7e9110a255896931d2e627cb50c2b0384cd4a423b2
+  metadata.gz: 5d520b8d56b0ef067e1a02228e1982541de4abce283d83ccdf7d168ee3cbb52f
+  data.tar.gz: b4b3eb85928de3ee6dd2a86f3e93e89567b37996af3e2476baa074de3cf5dc47
 SHA512:
-  metadata.gz: 404ec939662fbc9f4f8e571b95f3d86222c2d8039afd46346ed10b4319b2f5d8a307d4a0047cc75e041b17393031d0a0551088d2f291d4bf70ecd28baf98b449
-  data.tar.gz: efb5cb9aa4635e7d4a07344a61de6bd1f61d39fecbd8445e7baa2d375413603b7e9398cf391feac8b04a7a7c6fc46f5e9cd56791188ed6fb3e233a856a8e1caa
+  metadata.gz: 1190af3d7a5e4bb953007b9c2f31668ba0e6048b9958af0927c9be59d01ce4f3f8139c205354e2b931bc08c831c8f9a6e40e94bc40410b9329dc709bc212c5a4
+  data.tar.gz: 24f7eba242e5879ec229b80e2d4b24af920bcfa4bfef2a5b0b12b1c8c9e408011b2bd7f0954fcfd27de64d4c7429b185dc9945073a78e1192868ea414a2cfc02

data/.gitignore

@@ -4,6 +4,7 @@
 /coverage/
 /doc/
 /pkg/
+/rake
 /spec/reports/
 /tmp/
 Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+2.6.5

data/CHANGELOG.md

@@ -1,8 +1,7 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 ### Added
@@ -10,6 +9,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 ### Fixed
 
+## [5.0.0] - 2021-02-17
+### Added
+* Message encoders. Messages can now be encoded using either binary protobufs (the default) or JSON protobufs.
+* `RailwayIpc::OutgoingMessage` abstraction that encapsulates everything about a message to be published.
+
+### Changed
+* `Publisher#publish` now takes an optional `format` parameter that specifies how the message should be encoded. It is added to the message header when the message is published. The default format is `binary_protobuf`.
+* (Breaking change) `Publisher#publish` now returns an `OutgoingMessage` instead of a `Bunny::Exchange`.
+* Refactor `PublishedMessage#store_message` to take an `OutgoingMessage`.
+
+### Removed
+* (Breaking change) Remove deprecated `SingletonPublisher`
+
+## [4.0.1] - 2021-01-12
+### Fixed
+* Fixed `undefined method fetch for Bunny::MessageProperties` error. `Bunny::MessageProperties` isn't really a Hash, it wraps one (and doesn't provide a `#fetch` method).
+
+## [4.0.0] - 2021-01-11
+### Added
+* JSON decoder for consumers that can handle JSON encoded Protobufs. Note that the publishers do not (yet) have the option of encoding the messages as JSON.
+
+### Changed
+* (Breaking Change) Rename `Consumer#work` to `Consumer#work_with_params`. This was necessary so that we can support specifying different message encodings via metadata in the future. If the message encoding cannot be determined from the message metadata fall back to a default decoder (binary protobufs).
+
+### Fixed
+* `./bin/console` script was broken because Pry wasn't a dependency; added Pry as a development dependency only.
+
+## [3.0.0] - 2020-12-07
+### Changed
+* Consumers _will no longer crash_ when an exception is raised. Instead, consumers will move the message that caused the exception to a single dead-letter exchange called 'ipc:errors'. Railway will configure the dead-letter exchange automatically.
+
+## [2.2.2] - 2020-11-20
+### Fixed
+* Fixed Publisher class channel leak. Channels were being created on each instantiation of a Publisher instead of being re-used.
+
+## [2.2.1] - 2020-10-20
+### Added
+* Logging to indicate when options passed via `listen_to`/`from_queue` are overriding the defaults.
+
 ## [2.2.0] - 2020-10-20
 ### Added
 * The ability to configure workers to handle different workloads via `rake railway_ipc::consumers:spawn`
@@ -74,7 +112,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Correlation ID and message UUID are auto generated for messages for IDs are not passed in [#23](https://github.com/learn-co/railway_ipc_gem/pull/23)
 
-[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.3...HEAD
+[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v5.0.0...HEAD
+[5.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v4.0.1...v5.0.0
+[4.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v4.0.0...v4.0.1
+[4.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v3.0.0...v4.0.0
+[3.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.2...v3.0.0
+[2.2.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.1...v2.2.2
+[2.2.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.2.0...v2.2.1
+[2.2.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.1.0...v2.2.0
+[2.1.0]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.3...v2.1.0
 [2.0.3]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.2...v2.0.3
 [2.0.2]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.1...v2.0.2
 [2.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.0...v2.0.1

data/lib/railway_ipc.rb

@@ -5,6 +5,7 @@ require 'sneakers'
 require 'sneakers/spawner'
 require 'bunny'
 require 'active_record'
+require 'singleton'
 require 'railway_ipc/logger'
 require 'railway_ipc/unhandled_message_error'
 require 'railway_ipc/response'
@@ -13,7 +14,11 @@ require 'railway_ipc/unknown_message.pb'
 require 'railway_ipc/rabbitmq/adapter'
 require 'railway_ipc/handler'
 require 'railway_ipc/handler_store'
+require 'railway_ipc/message_encoders'
+require 'railway_ipc/message_decoders'
 require 'railway_ipc/incoming_message'
+require 'railway_ipc/outgoing_message'
+require 'railway_ipc/connection_manager'
 require 'railway_ipc/publisher'
 require 'railway_ipc/responder'
 require 'railway_ipc/rpc/rpc'

data/lib/railway_ipc/connection_manager.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module RailwayIpc
+  # RabbitMQ connection manager. Ensures there is a single RabbitMQ
+  # connection and channel per thread, which prevents channel leaks.
+  #
+  class ConnectionManager
+    include Singleton
+
+    def initialize
+      establish_connection
+    end
+
+    def establish_connection
+      @connection = Bunny.new(
+        host: settings[:host],
+        user: settings[:user],
+        pass: settings[:pass],
+        port: settings[:port],
+        vhost: settings[:vhost] || '/',
+        logger: RailwayIpc.logger
+      )
+      @connection.start
+      @channel = @connection.create_channel
+
+      @connection
+    end
+
+    def channel
+      return @channel if connected?
+
+      establish_connection
+      @channel
+    end
+
+    def connected?
+      @connection&.connected? && @channel&.open?
+    end
+
+    private
+
+    def amqp_url
+      @amqp_url ||= ENV.fetch('RAILWAY_RABBITMQ_CONNECTION_URL', 'amqp://guest:guest@localhost:5672')
+    end
+
+    def settings
+      @settings ||= AMQ::Settings.parse_amqp_url(amqp_url)
+    end
+  end
+end

data/lib/railway_ipc/consumer/consumer.rb

@@ -14,6 +14,7 @@ module RailwayIpc
       end
     end
 
+    # rubocop:disable Metrics/MethodLength
     def self.listen_to(queue:, exchange:, options: {})
       unless options.empty?
         RailwayIpc.logger.info(
@@ -27,9 +28,13 @@ module RailwayIpc
         exchange: exchange,
         durable: true,
         exchange_type: :fanout,
+        arguments: {
+          'x-dead-letter-exchange' => 'ipc:errors'
+        },
         connection: RailwayIpc.bunny_connection
       }.merge(options)
     end
+    # rubocop:enable Metrics/MethodLength
 
     def self.handle(message_type, with:)
       handlers.register(message: message_type, handler: with)
@@ -51,8 +56,14 @@ module RailwayIpc
       queue.opts[:exchange]
     end
 
-    def work(payload)
-      message = RailwayIpc::IncomingMessage.new(payload)
+    # REFACTOR: Long term we should think about not leaking Sneakers
+    # methods as part of Railway's public API since clients can (and do)
+    # override them. -BN
+    def work_with_params(payload, _delivery_info, metadata)
+      headers = metadata.headers || {}
+      message_format = headers.fetch('message_format', 'binary_protobuf')
+
+      message = RailwayIpc::IncomingMessage.new(payload, message_format: message_format)
       RailwayIpc::ProcessIncomingMessage.call(self, message)
       ack!
     rescue StandardError => e
@@ -64,7 +75,7 @@ module RailwayIpc
         error: e.class,
         payload: payload
       )
-      raise e
+      reject!
     end
 
     def get_handler(type)

data/lib/railway_ipc/incoming_message.rb

@@ -2,9 +2,10 @@
 
 module RailwayIpc
   class IncomingMessage
-    attr_reader :type, :payload, :parsed_payload, :errors
+    attr_reader :type, :message_format, :payload, :parsed_payload, :errors
 
-    def initialize(payload)
+    def initialize(payload, message_format: nil)
+      @message_format = message_format
       @parsed_payload = JSON.parse(payload)
       @type = parsed_payload['type']
       @payload = payload
@@ -32,20 +33,23 @@ module RailwayIpc
     end
 
     def decoded
-      @decoded ||=
-        begin
-          protobuf_msg = Base64.decode64(parsed_payload['encoded_message'])
-          decoder = Kernel.const_get(type)
-          decoder.decode(protobuf_msg)
-        rescue Google::Protobuf::ParseError => e
-          raise RailwayIpc::IncomingMessage::ParserError.new(e)
-        rescue NameError
-          RailwayIpc::Messages::Unknown.decode(protobuf_msg)
-        end
+      @decoded ||= \
+        get_decoder(message_format).call(type, parsed_payload['encoded_message'])
     end
 
     def stringify_errors
       errors.values.join(', ')
     end
+
+    private
+
+    DEFAULT_DECODER = RailwayIpc::MessageDecoders::ProtobufBinaryDecoder
+
+    def get_decoder(name)
+      {
+        'binary_protobuf' => RailwayIpc::MessageDecoders::ProtobufBinaryDecoder,
+        'json_protobuf' => RailwayIpc::MessageDecoders::ProtobufJsonDecoder
+      }.fetch(name, DEFAULT_DECODER)
+    end
   end
 end

data/lib/railway_ipc/message_decoders.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RailwayIpc
+  module MessageDecoders
+    ProtobufBinaryDecoder = lambda do |type, encoded_message|
+      protobuf_msg = Base64.decode64(encoded_message)
+      protobuf_klass = Kernel.const_get(type)
+      protobuf_klass.decode(protobuf_msg)
+    rescue Google::Protobuf::ParseError => e
+      raise RailwayIpc::IncomingMessage::ParserError.new(e)
+    rescue NameError
+      RailwayIpc::Messages::Unknown.decode(protobuf_msg)
+    end
+
+    ProtobufJsonDecoder = lambda do |type, message_hash|
+      protobuf_klass = Kernel.const_get(type)
+      protobuf_klass.new(message_hash)
+    rescue ArgumentError => e
+      raise RailwayIpc::IncomingMessage::ParserError.new(e)
+    rescue NameError
+      # NOTE: I didn't realize this until I made this ProtobufJsonDecoder, but
+      # the ProtobufBinaryDecoder will ignore any unknown keys -- which is
+      # probably not what we want. I'm coding this the same way as the binary
+      # protobuf version for consistency, but we should re-think how we want to
+      # handle this situation. -BN
+      RailwayIpc::Messages::Unknown.new(
+        user_uuid: message_hash.fetch(:user_uuid, ''),
+        correlation_id: message_hash.fetch(:correlation_id, ''),
+        uuid: message_hash.fetch(:uuid, ''),
+        context: message_hash.fetch(:context, {})
+      )
+    end
+  end
+end

data/lib/railway_ipc/message_encoders.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RailwayIpc
+  module MessageEncoders
+    ProtobufBinaryEncoder = lambda do |message|
+      {
+        type: message.type,
+        encoded_message: Base64.encode64(message.proto.class.encode(message.proto))
+      }.to_json
+    rescue NoMethodError
+      raise RailwayIpc::InvalidProtobuf.new("Message #{message} is not a valid protobuf")
+    end
+
+    ProtobufJsonEncoder = lambda do |message|
+      {
+        type: message.type,
+        encoded_message: message.proto.to_h
+      }.to_json
+    rescue NoMethodError
+      raise RailwayIpc::InvalidProtobuf.new("Message #{message} is not a valid protobuf")
+    end
+  end
+end

data/lib/railway_ipc/models/published_message.rb

@@ -7,16 +7,15 @@ module RailwayIpc
 
     validates :uuid, :status, presence: true
 
-    def self.store_message(exchange_name, message)
-      encoded_message = RailwayIpc::Rabbitmq::Payload.encode(message)
+    def self.store_message(outgoing_message)
       create!(
-        uuid: message.uuid,
-        message_type: message.class.to_s,
-        user_uuid: message.user_uuid,
-        correlation_id: message.correlation_id,
-        encoded_message: encoded_message,
+        uuid: outgoing_message.uuid,
+        message_type: outgoing_message.type,
+        user_uuid: outgoing_message.user_uuid,
+        correlation_id: outgoing_message.correlation_id,
+        encoded_message: outgoing_message.encoded,
         status: 'sent',
-        exchange: exchange_name
+        exchange: outgoing_message.exchange
       )
     end
 

data/lib/railway_ipc/outgoing_message.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module RailwayIpc
+  class OutgoingMessage
+    extend Forwardable
+
+    attr_reader :proto, :exchange, :format
+
+    def_delegators :@proto, :uuid, :user_uuid, :correlation_id
+
+    def initialize(proto, exchange, format=nil)
+      proto.uuid = SecureRandom.uuid if proto.uuid.blank?
+      proto.correlation_id = SecureRandom.uuid if proto.correlation_id.blank?
+      @proto = proto
+      @exchange = exchange
+      @format = format
+    end
+
+    def type
+      proto.class.to_s
+    end
+
+    def encoded
+      @encoded ||= encoder.call(self)
+    end
+
+    private
+
+    DEFAULT_ENCODER = RailwayIpc::MessageEncoders::ProtobufBinaryEncoder
+
+    def encoder
+      {
+        'binary_protobuf' => RailwayIpc::MessageEncoders::ProtobufBinaryEncoder,
+        'json_protobuf' => RailwayIpc::MessageEncoders::ProtobufJsonEncoder
+      }.fetch(format, DEFAULT_ENCODER)
+    end
+  end
+end

data/lib/railway_ipc/publisher.rb

@@ -1,81 +1,21 @@
 # frozen_string_literal: true
 
-require 'singleton'
-
-module RailwayIpc
-  class SingletonPublisher < Sneakers::Publisher
-    include ::Singleton
-
-    def self.exchange(exchange)
-      @exchange_name = exchange
-    end
-
-    def self.exchange_name
-      raise 'Subclass must set the exchange' unless @exchange_name
-
-      @exchange_name
-    end
-
-    def initialize
-      super(exchange: self.class.exchange_name, exchange_type: :fanout)
-    end
-
-    def publish(message, published_message_store=RailwayIpc::PublishedMessage)
-      RailwayIpc.logger.warn('DEPRECATED: Use new PublisherInstance class', log_message_options)
-      ensure_message_uuid(message)
-      ensure_correlation_id(message)
-      RailwayIpc.logger.info('Publishing message', log_message_options(message))
-      result = super(RailwayIpc::Rabbitmq::Payload.encode(message))
-      published_message_store.store_message(self.class.exchange_name, message)
-      result
-    rescue RailwayIpc::InvalidProtobuf => e
-      RailwayIpc.logger.error('Invalid protobuf', log_message_options(message))
-      raise e
-    end
-
-    private
-
-    def ensure_message_uuid(message)
-      message.uuid = SecureRandom.uuid if message.uuid.blank?
-      message
-    end
-
-    def ensure_correlation_id(message)
-      message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
-      message
-    end
-
-    def log_message_options(message=nil)
-      options = { feature: 'railway_ipc_publisher', exchange: self.class.exchange_name }
-      message.nil? ? options : options.merge(protobuf: { type: message.class, data: message })
-    end
-  end
-end
-
 module RailwayIpc
-  class Publisher < Sneakers::Publisher
+  class Publisher
     attr_reader :exchange_name, :message_store
 
     def initialize(opts={})
       @exchange_name = opts.fetch(:exchange_name)
       @message_store = opts.fetch(:message_store, RailwayIpc::PublishedMessage)
-      connection = opts.fetch(:connection, nil)
-      options = {
-        exchange: exchange_name,
-        connection: connection,
-        exchange_type: :fanout
-      }.compact
-      super(options)
     end
 
     # rubocop:disable Metrics/AbcSize
-    def publish(message)
-      message.uuid = SecureRandom.uuid if message.uuid.blank?
-      message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
+    def publish(message, format='binary_protobuf')
+      outgoing_message = OutgoingMessage.new(message, exchange_name, format)
+      stored_message = message_store.store_message(outgoing_message)
       RailwayIpc.logger.info('Publishing message', log_message_options(message))
-
-      stored_message = message_store.store_message(exchange_name, message)
-      super(RailwayIpc::Rabbitmq::Payload.encode(message))
+      exchange.publish(outgoing_message.encoded, headers: { message_format: format })
+      outgoing_message
     rescue RailwayIpc::InvalidProtobuf => e
       RailwayIpc.logger.error('Invalid protobuf', log_message_options(message))
       raise e
@@ -88,8 +28,16 @@ module RailwayIpc
     end
     # rubocop:enable Metrics/AbcSize
 
+    def exchange
+      @exchange ||= channel.exchange(exchange_name, type: :fanout, durable: true, auto_delete: false, arguments: {})
+    end
+
     private
 
+    def channel
+      RailwayIpc::ConnectionManager.instance.channel
+    end
+
     def log_message_options(message)
       {
         feature: 'railway_ipc_publisher',

data/lib/railway_ipc/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailwayIpc
-  VERSION = '2.2.1'
+  VERSION = '5.0.0'
 end

data/railway_ipc.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'database_cleaner', '~> 1.7'
   spec.add_development_dependency 'listen', '~> 3.0.5'
   spec.add_development_dependency 'pg', '~> 0.18'
+  spec.add_development_dependency 'pry', '~> 0.13'
   spec.add_development_dependency 'rails', '~> 5.0.7'
   spec.add_development_dependency 'rspec-rails'
   spec.add_development_dependency 'shoulda-matchers', '~> 4.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: railway-ipc
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 5.0.0
 platform: ruby
 authors:
 - ''
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-21 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -213,6 +227,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -225,6 +240,7 @@ files:
 - bin/setup
 - lib/railway_ipc.rb
 - lib/railway_ipc/Rakefile
+- lib/railway_ipc/connection_manager.rb
 - lib/railway_ipc/consumer/consumer.rb
 - lib/railway_ipc/consumer/process_incoming_message.rb
 - lib/railway_ipc/errors.rb
@@ -232,8 +248,11 @@ files:
 - lib/railway_ipc/handler_store.rb
 - lib/railway_ipc/incoming_message.rb
 - lib/railway_ipc/logger.rb
+- lib/railway_ipc/message_decoders.rb
+- lib/railway_ipc/message_encoders.rb
 - lib/railway_ipc/models/consumed_message.rb
 - lib/railway_ipc/models/published_message.rb
+- lib/railway_ipc/outgoing_message.rb
 - lib/railway_ipc/publisher.rb
 - lib/railway_ipc/rabbitmq/adapter.rb
 - lib/railway_ipc/rabbitmq/payload.rb