railway-ipc 0.1.7 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d54b5628ec775d68214605d423ffaea0cd56296efce288d9f6d225a3fe5b68a
-  data.tar.gz: b8fce8f52051eb667daa678b931ec24c3fa5de54783918849f618407e5b81863
+  metadata.gz: 11bbf2da076cd07dca74c10a4012cf6e737c7608db72cae9caee418e9188c105
+  data.tar.gz: b38f426df0aca6a34e18ea7de764d74eaccc183c10fe6af3a04681d7bb658af2
 SHA512:
-  metadata.gz: 68e7d82e728b4c3ace10b7c1926672c301bcd5090aa0bfc80b80485936c476fbf59c97493fe7d1eaaf7874620cf967b9b6ee2f95923cbeefff498248732c47e1
-  data.tar.gz: 2780cc3cf0d29b4abfda4fecc2d28a119e08d5592a87b6cba3ebf3c868bdea208c9f3c00ddf4708f35aa6f6b9e21defbd25baf2708006c725c6aec164038563d
+  metadata.gz: 3ff4e1f023ad41059c6094f1e6546fac6a4fdc1f4037a4fe9d482327b7b66c23a6db051624ac22338800d52d2c636ec272d01dc853fed22f86d539d7b6fb2333
+  data.tar.gz: 72f11fcae1cff787ebd8f0a361ff67b81afb1003a8e265d59726a0ee701a35f22b53cd7bddabfb725b6dba217d8fab8da19932594ee9ae1583a508c27083a5d2

data/.gitignore

@@ -6,14 +6,15 @@
 /pkg/
 /spec/reports/
 /tmp/
+Gemfile.lock
 
 # rspec failure tracking
 .rspec_status
 
 # rails support app files
-/spec/support/rails_app/.bundle
-/spec/support/rails_app/log/*
-/spec/support/rails_app/tmp/*
 /spec/support/rails_app!/log/.keep
 /spec/support/rails_app!/tmp/.keep
 /spec/support/rails_app.byebug_history
+/spec/support/rails_app/.bundle
+/spec/support/rails_app/log/*
+/spec/support/rails_app/tmp/*

data/CHANGELOG.md

@@ -0,0 +1,67 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+### Added
+### Changed
+### Removed
+### Fixed
+
+## [2.0.1] - 2020-08-24
+### Fixed
+* `RailwayIpc::Logger` now handles block syntax (i.e. `logger.info { 'my message' }`) correctly.
+
+## [2.0.0] - 2020-08-20
+### Added
+* Several additions to internal logging:
+  - Log messages now include a `feature` key. This can be used by logging aggregator tools to group log messages across different systems that use the gem. If one isn't provided a default value of `railway_ipc` is used.
+  - Protobufs are logged as a sub-hash which contains both the protobuf type and payload.
+  - Exchange and queue names are logged where applicable.
+  - The internal Bunny connection now uses the `RailwayIpc::Logger` instead of a generic `Logger`.
+
+### Changed
+* *Breaking Change* `RailwayIpc.configure` now takes `device`, `level`, and `formatter` instead of a complete `Logger` instance. The instance is now managed internally by Railway. This is a breaking change to the `RailwayIpc.configure` API, clients will need to be updated to use the new syntax.
+
+## [1.1.0] - 2020-08-07
+### Changed
+* allow multiple consumers to handle the same message
+* consumed messages table requires its own primary key due to ActiveRecord not having support for composite primary keys
+
+## [1.0.1] - 2020-07-23
+### Fixed
+* Fix publisher connection by using default connection if one isn't provided
+
+## [1.0.0] - 2020-07-20
+### Added
+* CircleCI build that runs the specs
+* Rubocop (also ran by CircleCI)
+* New error types for incoming messages
+* RailwayIpc::Messages::Unknown protobuf
+
+### Changed
+* Refactored worker to use ProcessIncomingMessage and IncomingMessage abstractions
+* Moved decoding logic from ConsumedMessage to IncomingMessage
+* Removed STATUSES constant from ConsumedMessage
+* Publisher is no longer a Singleton; kept a Singleton version of the Publisher for backwards compatibility that gives a "deprecated" warning
+
+### Removed
+* Removed `BaseMessage` protobuf
+* NullMessage and NullHandler were removed
+
+### Fixed
+* Fixed all Rubocop warnings and errors
+
+## [0.1.7] - 2020-06-29
+### Added
+- Correlation ID and message UUID are auto generated for messages for IDs are not passed in [#23](https://github.com/learn-co/railway_ipc_gem/pull/23)
+
+[Unreleased]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.1...HEAD
+[2.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v2.0.0...v2.0.1
+[2.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v1.1.0...v2.0.0
+[1.1.0]: https://github.com/learn-co/railway_ipc_gem/compare/v1.0.1...v1.1.0
+[1.0.1]: https://github.com/learn-co/railway_ipc_gem/compare/v1.0.0...v1.0.1
+[1.0.0]: https://github.com/learn-co/railway_ipc_gem/compare/v0.1.7...v1.0.0
+[0.1.7]: https://github.com/learn-co/railway_ipc_gem/releases/tag/v0.1.7

data/Gemfile

@@ -1,5 +1,5 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 gemspec

data/README.md

@@ -19,7 +19,7 @@ And then execute:
 ```ruby
 # config/initializers/railway_ipc.rb
 
-RailwayIpc.configure(logger: Rails.logger)
+RailwayIpc.configure(STDOUT, Logger::INFO, MyFormatter)
 ```
 
 * Load the rake tasks in your Rakefile

data/Rakefile

@@ -1,7 +1,13 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
-task :test => :spec
+task default: %i[spec lint]
+
+desc 'Lint code with Rubocop'
+task :lint do
+  exec('./bin/rubocop .')
+end

data/bin/console

@@ -1,10 +1,10 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "railway_ipc"
+require 'bundler/setup'
+require 'railway_ipc'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-require "pry"
+require 'pry'
 Pry.start

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/rubocop

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rubocop' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rubocop", "rubocop")

data/lib/railway_ipc.rb

@@ -1,42 +1,40 @@
+# frozen_string_literal: true
+
 require 'railway_ipc/version'
 require 'sneakers'
 require 'bunny'
 require 'active_record'
 require 'railway_ipc/version'
-require 'railway_ipc/errors'
 require 'railway_ipc/logger'
 require 'railway_ipc/unhandled_message_error'
 require 'railway_ipc/response'
 require 'railway_ipc/rabbitmq/payload'
-require 'railway_ipc/null_message'
-require 'railway_ipc/base_message.pb'
+require 'railway_ipc/unknown_message.pb'
 require 'railway_ipc/rabbitmq/adapter'
 require 'railway_ipc/handler'
 require 'railway_ipc/handler_store'
+require 'railway_ipc/incoming_message'
 require 'railway_ipc/publisher'
-require 'railway_ipc/null_handler'
 require 'railway_ipc/responder'
 require 'railway_ipc/rpc/rpc'
 require 'railway_ipc/consumer/consumer'
+require 'railway_ipc/consumer/process_incoming_message'
 require 'railway_ipc/models/published_message'
 require 'railway_ipc/models/consumed_message'
 require 'railway_ipc/railtie' if defined?(Rails)
+require 'railway_ipc/errors'
 
 module RailwayIpc
   def self.start
     Rake::Task['sneakers:run'].invoke
   end
 
-  def self.configure(logger: ::Logger.new(STDOUT))
-    @logger = RailwayIpc::Logger.new(logger)
+  def self.configure(log_device=STDOUT, level=::Logger::INFO, log_formatter=nil)
+    @logger = RailwayIpc::Logger.new(log_device, level, log_formatter)
   end
 
   def self.logger
-    @logger || RailwayIpc::Logger.new(::Logger.new(STDOUT))
-  end
-
-  def self.bunny_logger
-    logger.logger
+    @logger || RailwayIpc::Logger.new(STDOUT)
   end
 
   def self.bunny_connection

data/lib/railway_ipc/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'railway_ipc'
 
 path = File.expand_path(__dir__)

data/lib/railway_ipc/consumer/consumer.rb

@@ -1,11 +1,15 @@
-require "json"
-require "base64"
-require "railway_ipc/consumer/consumer_response_handlers"
+# frozen_string_literal: true
 
 module RailwayIpc
   class Consumer
     include Sneakers::Worker
-    attr_reader :message, :handler, :protobuf_message, :delivery_info, :decoded_payload, :encoded_message
+    def self.inherited(base)
+      base.instance_eval do
+        def handlers
+          @handlers ||= RailwayIpc::HandlerStore.new
+        end
+      end
+    end
 
     def self.listen_to(queue:, exchange:)
       from_queue queue,
@@ -16,88 +20,44 @@ module RailwayIpc
     end
 
     def self.handle(message_type, with:)
-      ConsumerResponseHandlers.instance.register(message: message_type, handler: with)
+      handlers.register(message: message_type, handler: with)
     end
 
-    def registered_handlers
-      ConsumerResponseHandlers.instance.registered
+    def handlers
+      self.class.handlers
     end
 
-    def work_with_params(payload, delivery_info, _metadata)
-      @delivery_info = delivery_info
-      @decoded_payload = RailwayIpc::Rabbitmq::Payload.decode(payload)
-      @encoded_message = payload
-
-      case decoded_payload.type
-      when *registered_handlers
-        @handler = handler_for(decoded_payload)
-        message_klass = message_handler_for(decoded_payload)
-        @protobuf_message = message_klass.decode(decoded_payload.message)
-        process_known_message_type
-      else
-        @protobuf_message = RailwayIpc::BaseMessage.decode(decoded_payload.message)
-        process_unknown_message_type
-      end
-      ack!
-    rescue StandardError => e
-      RailwayIpc.logger.log_exception(
-        feature: "railway_consumer",
-        error: e.class,
-        error_message: e.message,
-        payload: payload,
-      )
-      raise e
-    end
-
-    private
-
-    def process_protobuf!(message)
-      response = handler.handle(protobuf_message)
-      message.status = RailwayIpc::ConsumedMessage.response_to_status(response)
-
-      message.save!
+    def registered_handlers
+      handlers.registered
     end
 
-    def process_known_message_type
-      message = RailwayIpc::ConsumedMessage.find_by(uuid: protobuf_message.uuid)
-
-      return if message && message.processed?
-
-      if message && !message.processed?
-        message.with_lock("FOR UPDATE NOWAIT") { process_protobuf!(message) }
-      else
-        message = create_message_with_status!(RailwayIpc::ConsumedMessage::STATUSES[:processing])
-        message.with_lock("FOR UPDATE NOWAIT") { process_protobuf!(message) }
-      end
+    def queue_name
+      queue.name
     end
 
-    def process_unknown_message_type
-      if RailwayIpc::ConsumedMessage.exists?(uuid: protobuf_message.uuid)
-        return
-      else
-        create_message_with_status!(RailwayIpc::ConsumedMessage::STATUSES[:unknown_message_type])
-      end
+    def exchange_name
+      queue.opts[:exchange]
     end
 
-    def create_message_with_status!(status)
-      RailwayIpc::ConsumedMessage.create!(
-        uuid: protobuf_message.uuid,
-        status: status,
-        message_type: decoded_payload.type,
-        user_uuid: protobuf_message.user_uuid,
-        correlation_id: protobuf_message.correlation_id,
-        queue: delivery_info.consumer.queue.name,
-        exchange: delivery_info.exchange,
-        encoded_message: encoded_message
+    def work(payload)
+      message = RailwayIpc::IncomingMessage.new(payload)
+      RailwayIpc::ProcessIncomingMessage.call(self, message)
+      ack!
+    rescue StandardError => e
+      RailwayIpc.logger.error(
+        e.message,
+        feature: 'railway_ipc_consumer',
+        exchange: exchange_name,
+        queue: queue_name,
+        error: e.class,
+        payload: payload
       )
+      raise e
     end
 
-    def message_handler_for(decoded_payload)
-      ConsumerResponseHandlers.instance.get(decoded_payload.type).message
-    end
-
-    def handler_for(decoded_payload)
-      ConsumerResponseHandlers.instance.get(decoded_payload.type).handler.new
+    def get_handler(type)
+      manifest = handlers.get(type)
+      manifest ? manifest.handler.new : nil
     end
   end
 end

data/lib/railway_ipc/consumer/process_incoming_message.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module RailwayIpc
+  class ProcessIncomingMessage
+    class UnknownMessageJob
+      attr_reader :incoming_message, :logger
+
+      def initialize(incoming_message, logger)
+        @incoming_message = incoming_message
+        @logger = logger
+      end
+
+      def status
+        'unknown_message_type'
+      end
+
+      def run
+        logger.warn(
+          "Ignoring unknown message of type '#{incoming_message.type}'",
+          feature: 'railway_ipc_consumer',
+          protobuf: { type: incoming_message.type, data: incoming_message.decoded }
+        )
+      end
+    end
+
+    class IgnoredMessageJob
+      attr_reader :incoming_message, :logger
+
+      def initialize(incoming_message, logger)
+        @incoming_message = incoming_message
+        @logger = logger
+      end
+
+      def status
+        'ignored'
+      end
+
+      def run
+        logger.warn(
+          "Ignoring message, no registered handler for '#{incoming_message.type}'",
+          feature: 'railway_ipc_consumer',
+          protobuf: { type: incoming_message.type, data: incoming_message.decoded }
+        )
+      end
+    end
+
+    class NormalMessageJob
+      attr_reader :incoming_message, :handler, :status
+
+      def initialize(incoming_message, handler)
+        @incoming_message = incoming_message
+        @handler = handler
+        @status = 'not_processed'
+      end
+
+      def run
+        result = handler.handle(incoming_message.decoded)
+        @status = result.success? ? RailwayIpc::ConsumedMessage::STATUS_SUCCESS : RailwayIpc::ConsumedMessage::STATUS_FAILED_TO_PROCESS
+      end
+    end
+
+    attr_reader :consumer, :incoming_message, :logger
+
+    def self.call(consumer, incoming_message)
+      new(consumer, incoming_message).call
+    end
+
+    def initialize(consumer, incoming_message, logger: RailwayIpc.logger)
+      @consumer = consumer
+      @incoming_message = incoming_message
+      @logger = logger
+    end
+
+    def call
+      raise_message_invalid_error unless incoming_message.valid?
+      message = find_or_create_consumed_message
+      return if message.processed?
+
+      message.update_with_lock(classify_message)
+    end
+
+    private
+
+    def raise_message_invalid_error
+      error = "Message is invalid: #{incoming_message.stringify_errors}."
+      logger.error(
+        error,
+        feature: 'railway_ipc_consumer',
+        exchange: consumer.exchange_name,
+        queue: consumer.queue_name,
+        protobuf: { type: incoming_message.class, data: incoming_message.decoded }
+      )
+      raise RailwayIpc::IncomingMessage::InvalidMessage.new(error)
+    end
+
+    def find_or_create_consumed_message
+      RailwayIpc::ConsumedMessage.find_by(uuid: incoming_message.uuid, queue: consumer.queue_name) ||
+        RailwayIpc::ConsumedMessage.create_processing(consumer, incoming_message)
+    end
+
+    def classify_message
+      if incoming_message.decoded.is_a?(RailwayIpc::Messages::Unknown)
+        UnknownMessageJob.new(incoming_message, logger)
+      elsif (handler = consumer.get_handler(incoming_message.type))
+        NormalMessageJob.new(incoming_message, handler)
+      else
+        IgnoredMessageJob.new(incoming_message, logger)
+      end
+    end
+  end
+end