railway-ipc 0.1.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 110c82fedd170e47574c5cce683f08a1bad9b6fe27ce4ed91428c04153b4c317
-  data.tar.gz: 70da4ff747fbb8ca79f161c3e8148d7034ed1975ad5a9aa5f5f619d67e30dd31
+  metadata.gz: 11e49c11ff3e5815f7c8050ad5fcdfde1510cf7646d10cfa5d1104a5c1868422
+  data.tar.gz: 3fbdeab2e555f9c4b3402cf27e3b072d33c833f85659eaccc09f31d11fddc219
 SHA512:
-  metadata.gz: ffa82d046eacfd17e63bd27c99a3078ed0fa7a5e9521e28b87e29bbda3da81167ec6d8940657c4d5fc298c814872910c3bbe8e11f82e17333fc7aa547a95c151
-  data.tar.gz: c332dfdb14d28209fe2f07ded7784bd5a82c98b5cb2810a547ddfb3f406d8879a9aab3f74ff44c0d8555f99de0a947e96fba3333626d2f5e7efe00e024118101
+  metadata.gz: 70d3834ac2a0db7b1a9a53c52edb3718b34a821d3b0d42e49ebe01f1c76c00188b928734a5969ef621e9609a3852117fd8190706566777c870b2b8b3d9592ccd
+  data.tar.gz: 297bbf2e92f9a77d85661849f3bbada5b18eff08b213ec6a48089dea112ab65b70a47b196e8ddc59b475e61cb9e6ecfc8648bb5dcbcd2836e341079d81b2fd92

data/.gitignore

@@ -6,14 +6,15 @@
 /pkg/
 /spec/reports/
 /tmp/
+Gemfile.lock
 
 # rspec failure tracking
 .rspec_status
 
 # rails support app files
-/spec/support/rails_app/.bundle
-/spec/support/rails_app/log/*
-/spec/support/rails_app/tmp/*
 /spec/support/rails_app!/log/.keep
 /spec/support/rails_app!/tmp/.keep
 /spec/support/rails_app.byebug_history
+/spec/support/rails_app/.bundle
+/spec/support/rails_app/log/*
+/spec/support/rails_app/tmp/*

data/CHANGELOG.md

@@ -0,0 +1,61 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+### Added
+### Changed
+### Removed
+### Fixed
+
+## [2.0.0] - 2020-08-20
+### Added
+* Several additions to internal logging:
+  - Log messages now include a `feature` key. This can be used by logging aggregator tools to group log messages across different systems that use the gem. If one isn't provided a default value of `railway_ipc` is used.
+  - Protobufs are logged as a sub-hash which contains both the protobuf type and payload.
+  - Exchange and queue names are logged where applicable.
+  - The internal Bunny connection now uses the `RailwayIpc::Logger` instead of a generic `Logger`.
+
+### Changed
+* *Breaking Change* `RailwayIpc.configure` now takes `device`, `level`, and `formatter` instead of a complete `Logger` instance. The instance is now managed internally by Railway. This is a breaking change to the `RailwayIpc.configure` API, clients will need to be updated to use the new syntax.
+
+## [1.1.0] - 2020-08-07
+### Changed
+* allow multiple consumers to handle the same message
+* consumed messages table requires its own primary key due to ActiveRecord not having support for composite primary keys
+
+## [1.0.1] - 2020-07-23
+### Fixed
+* Fix publisher connection by using default connection if one isn't provided
+
+## [1.0.0] - 2020-07-20
+### Added
+* CircleCI build that runs the specs
+* Rubocop (also ran by CircleCI)
+* New error types for incoming messages
+* RailwayIpc::Messages::Unknown protobuf
+
+### Changed
+* Refactored worker to use ProcessIncomingMessage and IncomingMessage abstractions
+* Moved decoding logic from ConsumedMessage to IncomingMessage
+* Removed STATUSES constant from ConsumedMessage
+* Publisher is no longer a Singleton; kept a Singleton version of the Publisher for backwards compatibility that gives a "deprecated" warning
+
+### Removed
+* Removed `BaseMessage` protobuf
+* NullMessage and NullHandler were removed
+
+### Fixed
+* Fixed all Rubocop warnings and errors
+
+## [0.1.7] - 2020-06-29
+### Added
+- Correlation ID and message UUID are auto generated for messages for IDs are not passed in [#23](https://github.com/learn-co/railway_ipc_gem/pull/23)
+
+[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v1.1.0...HEAD
+[1.1.0]: https://github.com/learn-co/railway_ipc_gem/compare/v1.0.1...v1.1.0
+[1.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v1.0.0...v1.0.1
+[1.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v0.1.7...v1.0.0
+[0.1.7]: https://github.com/learn-co/railway_ipc_gem/releases/tag/v0.1.7

data/Gemfile

@@ -1,5 +1,5 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 gemspec

data/README.md

@@ -19,7 +19,7 @@ And then execute:
 ```ruby
 # config/initializers/railway_ipc.rb
 
-RailwayIpc.configure(logger: Rails.logger)
+RailwayIpc.configure(STDOUT, Logger::INFO, MyFormatter)
 ```
 
 * Load the rake tasks in your Rakefile
@@ -32,7 +32,7 @@ require "railway_ipc"
 * Create RabbitMQ connection credentials for Railway and set the environment variable:
 
 ```
-RABBITMQ_CONNECTION_URL=amqp://<railway_user>:<railway_password>@localhost:5672
+RAILWAY_RABBITMQ_CONNECTION_URL=amqp://<railway_user>:<railway_password>@localhost:5672
 ```
 
 * Load table migrations and migrate by executing:

data/Rakefile

@@ -1,7 +1,13 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
-task :test => :spec
+task default: %i[spec lint]
+
+desc 'Lint code with Rubocop'
+task :lint do
+  exec('./bin/rubocop .')
+end

data/bin/console

@@ -1,10 +1,10 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "railway_ipc"
+require 'bundler/setup'
+require 'railway_ipc'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-require "pry"
+require 'pry'
 Pry.start

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/rubocop

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rubocop' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rubocop", "rubocop")

data/lib/railway_ipc.rb

@@ -1,42 +1,40 @@
+# frozen_string_literal: true
+
 require 'railway_ipc/version'
 require 'sneakers'
 require 'bunny'
 require 'active_record'
 require 'railway_ipc/version'
-require 'railway_ipc/errors'
 require 'railway_ipc/logger'
 require 'railway_ipc/unhandled_message_error'
 require 'railway_ipc/response'
 require 'railway_ipc/rabbitmq/payload'
-require 'railway_ipc/null_message'
-require 'railway_ipc/base_message.pb'
+require 'railway_ipc/unknown_message.pb'
 require 'railway_ipc/rabbitmq/adapter'
 require 'railway_ipc/handler'
 require 'railway_ipc/handler_store'
+require 'railway_ipc/incoming_message'
 require 'railway_ipc/publisher'
-require 'railway_ipc/null_handler'
 require 'railway_ipc/responder'
 require 'railway_ipc/rpc/rpc'
 require 'railway_ipc/consumer/consumer'
+require 'railway_ipc/consumer/process_incoming_message'
 require 'railway_ipc/models/published_message'
 require 'railway_ipc/models/consumed_message'
 require 'railway_ipc/railtie' if defined?(Rails)
+require 'railway_ipc/errors'
 
 module RailwayIpc
   def self.start
     Rake::Task['sneakers:run'].invoke
   end
 
-  def self.configure(logger: ::Logger.new(STDOUT))
-    @logger = RailwayIpc::Logger.new(logger)
+  def self.configure(log_device=STDOUT, level=::Logger::INFO, log_formatter=nil)
+    @logger = RailwayIpc::Logger.new(log_device, level, log_formatter)
   end
 
   def self.logger
-    @logger || RailwayIpc::Logger.new(::Logger.new(STDOUT))
-  end
-
-  def self.bunny_logger
-    logger.logger
+    @logger || RailwayIpc::Logger.new(STDOUT)
   end
 
   def self.bunny_connection

data/lib/railway_ipc/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'railway_ipc'
 
 path = File.expand_path(__dir__)

data/lib/railway_ipc/consumer/consumer.rb

@@ -1,11 +1,15 @@
-require "json"
-require "base64"
-require "railway_ipc/consumer/consumer_response_handlers"
+# frozen_string_literal: true
 
 module RailwayIpc
   class Consumer
     include Sneakers::Worker
-    attr_reader :message, :handler, :protobuf_message, :delivery_info, :decoded_payload, :encoded_message
+    def self.inherited(base)
+      base.instance_eval do
+        def handlers
+          @handlers ||= RailwayIpc::HandlerStore.new
+        end
+      end
+    end
 
     def self.listen_to(queue:, exchange:)
       from_queue queue,
@@ -16,88 +20,44 @@ module RailwayIpc
     end
 
     def self.handle(message_type, with:)
-      ConsumerResponseHandlers.instance.register(message: message_type, handler: with)
+      handlers.register(message: message_type, handler: with)
     end
 
-    def registered_handlers
-      ConsumerResponseHandlers.instance.registered
+    def handlers
+      self.class.handlers
     end
 
-    def work_with_params(payload, delivery_info, _metadata)
-      @delivery_info = delivery_info
-      @decoded_payload = RailwayIpc::Rabbitmq::Payload.decode(payload)
-      @encoded_message = payload
-
-      case decoded_payload.type
-      when *registered_handlers
-        @handler = handler_for(decoded_payload)
-        message_klass = message_handler_for(decoded_payload)
-        @protobuf_message = message_klass.decode(decoded_payload.message)
-        process_known_message_type
-      else
-        @protobuf_message = RailwayIpc::BaseMessage.decode(decoded_payload.message)
-        process_unknown_message_type
-      end
-      ack!
-    rescue StandardError => e
-      RailwayIpc.logger.log_exception(
-        feature: "railway_consumer",
-        error: e.class,
-        error_message: e.message,
-        payload: payload,
-      )
-      raise e
-    end
-
-    private
-
-    def process_protobuf!(message)
-      response = handler.handle(protobuf_message)
-      message.status = RailwayIpc::ConsumedMessage.response_to_status(response)
-
-      message.save!
+    def registered_handlers
+      handlers.registered
     end
 
-    def process_known_message_type
-      message = RailwayIpc::ConsumedMessage.find_by(uuid: protobuf_message.uuid)
-
-      return if message && message.processed?
-
-      if message && !message.processed?
-        message.with_lock("FOR UPDATE NOWAIT") { process_protobuf!(message) }
-      else
-        message = create_message_with_status!(RailwayIpc::ConsumedMessage::STATUSES[:processing])
-        message.with_lock("FOR UPDATE NOWAIT") { process_protobuf!(message) }
-      end
+    def queue_name
+      queue.name
     end
 
-    def process_unknown_message_type
-      if RailwayIpc::ConsumedMessage.exists?(uuid: protobuf_message.uuid)
-        return
-      else
-        create_message_with_status!(RailwayIpc::ConsumedMessage::STATUSES[:unknown_message_type])
-      end
+    def exchange_name
+      queue.opts[:exchange]
     end
 
-    def create_message_with_status!(status)
-      RailwayIpc::ConsumedMessage.create!(
-        uuid: protobuf_message.uuid,
-        status: status,
-        message_type: decoded_payload.type,
-        user_uuid: protobuf_message.user_uuid,
-        correlation_id: protobuf_message.correlation_id,
-        queue: delivery_info.consumer.queue.name,
-        exchange: delivery_info.exchange,
-        encoded_message: encoded_message
+    def work(payload)
+      message = RailwayIpc::IncomingMessage.new(payload)
+      RailwayIpc::ProcessIncomingMessage.call(self, message)
+      ack!
+    rescue StandardError => e
+      RailwayIpc.logger.error(
+        e.message,
+        feature: 'railway_ipc_consumer',
+        exchange: exchange_name,
+        queue: queue_name,
+        error: e.class,
+        payload: payload
       )
+      raise e
     end
 
-    def message_handler_for(decoded_payload)
-      ConsumerResponseHandlers.instance.get(decoded_payload.type).message
-    end
-
-    def handler_for(decoded_payload)
-      ConsumerResponseHandlers.instance.get(decoded_payload.type).handler.new
+    def get_handler(type)
+      manifest = handlers.get(type)
+      manifest ? manifest.handler.new : nil
     end
   end
 end

data/lib/railway_ipc/consumer/process_incoming_message.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module RailwayIpc
+  class ProcessIncomingMessage
+    class UnknownMessageJob
+      attr_reader :incoming_message, :logger
+
+      def initialize(incoming_message, logger)
+        @incoming_message = incoming_message
+        @logger = logger
+      end
+
+      def status
+        'unknown_message_type'
+      end
+
+      def run
+        logger.warn(
+          "Ignoring unknown message of type '#{incoming_message.type}'",
+          feature: 'railway_ipc_consumer',
+          protobuf: { type: incoming_message.type, data: incoming_message.decoded }
+        )
+      end
+    end
+
+    class IgnoredMessageJob
+      attr_reader :incoming_message, :logger
+
+      def initialize(incoming_message, logger)
+        @incoming_message = incoming_message
+        @logger = logger
+      end
+
+      def status
+        'ignored'
+      end
+
+      def run
+        logger.warn(
+          "Ignoring message, no registered handler for '#{incoming_message.type}'",
+          feature: 'railway_ipc_consumer',
+          protobuf: { type: incoming_message.type, data: incoming_message.decoded }
+        )
+      end
+    end
+
+    class NormalMessageJob
+      attr_reader :incoming_message, :handler, :status
+
+      def initialize(incoming_message, handler)
+        @incoming_message = incoming_message
+        @handler = handler
+        @status = 'not_processed'
+      end
+
+      def run
+        result = handler.handle(incoming_message.decoded)
+        @status = result.success? ? RailwayIpc::ConsumedMessage::STATUS_SUCCESS : RailwayIpc::ConsumedMessage::STATUS_FAILED_TO_PROCESS
+      end
+    end
+
+    attr_reader :consumer, :incoming_message, :logger
+
+    def self.call(consumer, incoming_message)
+      new(consumer, incoming_message).call
+    end
+
+    def initialize(consumer, incoming_message, logger: RailwayIpc.logger)
+      @consumer = consumer
+      @incoming_message = incoming_message
+      @logger = logger
+    end
+
+    def call
+      raise_message_invalid_error unless incoming_message.valid?
+      message = find_or_create_consumed_message
+      return if message.processed?
+
+      message.update_with_lock(classify_message)
+    end
+
+    private
+
+    def raise_message_invalid_error
+      error = "Message is invalid: #{incoming_message.stringify_errors}."
+      logger.error(
+        error,
+        feature: 'railway_ipc_consumer',
+        exchange: consumer.exchange_name,
+        queue: consumer.queue_name,
+        protobuf: { type: incoming_message.class, data: incoming_message.decoded }
+      )
+      raise RailwayIpc::IncomingMessage::InvalidMessage.new(error)
+    end
+
+    def find_or_create_consumed_message
+      RailwayIpc::ConsumedMessage.find_by(uuid: incoming_message.uuid, queue: consumer.queue_name) ||
+        RailwayIpc::ConsumedMessage.create_processing(consumer, incoming_message)
+    end
+
+    def classify_message
+      if incoming_message.decoded.is_a?(RailwayIpc::Messages::Unknown)
+        UnknownMessageJob.new(incoming_message, logger)
+      elsif (handler = consumer.get_handler(incoming_message.type))
+        NormalMessageJob.new(incoming_message, handler)
+      else
+        IgnoredMessageJob.new(incoming_message, logger)
+      end
+    end
+  end
+end