messaging 0.0.2 → 3.4.1

data/lib/messaging/consumer_supervisor.rb

@@ -0,0 +1,63 @@
+module Messaging
+  # @private
+  class ConsumerSupervisor
+    include Messaging::Instrumentation
+
+    def initialize
+      @threads = Concurrent::Array.new
+      Messaging.routes.define_consumers!
+    end
+
+    def start
+      Concurrent.use_simple_logger
+      Messaging.logger.info 'Consumers starting'
+      @signal_to_stop = false
+      @threads.clear
+      @thread_pool = Concurrent::FixedThreadPool.new(consumers.size, auto_terminate: false)
+
+      consumers.each do |consumer|
+        @thread_pool.post do
+          thread = Thread.current
+          thread.abort_on_exception = true
+          @threads << thread
+          run_consumer(consumer)
+        end
+      end
+
+      true
+    end
+
+    def stop
+      return if @signal_to_stop
+
+      instrument('consumer_supervisor.stop') do
+        @signal_to_stop = true
+        consumers.map { |consumer| Thread.new { consumer.stop } }.join
+        @threads.select(&:alive?).each { |thread| thread&.wakeup }
+        @thread_pool&.shutdown
+        @thread_pool&.wait_for_termination(60)
+        Messaging.logger.info 'Consumers stopped'
+      end
+    end
+
+    def status
+      consumers.map(&:log_current_status)
+    end
+
+    def consumers
+      Messaging.routes.consumers
+    end
+
+    private
+
+    def run_consumer(consumer)
+      consumer.start
+    rescue StandardError => e
+      return if @signal_to_stop
+
+      ExceptionHandler.call(e)
+      sleep 2
+      retry
+    end
+  end
+end

data/lib/messaging/exception_handler.rb

@@ -0,0 +1,15 @@
+module Messaging
+  class ExceptionHandler
+    def self.call(exception, context = {})
+      Config.error_handlers.each do |handler|
+        begin
+          handler.call(exception, context)
+        rescue => e
+          Messaging.logger.error '!!! ERROR HANDLER THREW AN ERROR !!!'
+          Messaging.logger.error e
+          Messaging.logger.error e.backtrace.join("\n") unless e.backtrace.nil?
+        end
+      end
+    end
+  end
+end

data/lib/messaging/expected_version.rb

@@ -0,0 +1,31 @@
+module Messaging
+  class ExpectedVersion
+    Error = Class.new(StandardError)
+
+    attr_reader :version
+
+    def initialize(version)
+      @version = version
+    end
+
+    def any?
+      version == :any
+    end
+
+    def none?
+      version == :none || version == -1
+    end
+
+    def matches?(other_version)
+      return true if any?
+      return true if none? && other_version == -1
+      return true if version == other_version
+      false
+    end
+
+    def match!(other_version)
+      return true if matches?(other_version)
+      raise Error, "expected: #{version} actual: #{other_version}"
+    end
+  end
+end

data/lib/messaging/instrumentation.rb

@@ -0,0 +1,21 @@
+module Messaging
+  module Instrumentation
+    NAMESPACE = 'messaging'
+
+    def instrument(event, extra = {})
+      ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do |extra|
+        yield(extra) if block_given?
+      end
+    end
+
+    def self.subscribe(event)
+      ActiveSupport::Notifications.subscribe("#{NAMESPACE}.#{event}") do |*args|
+        yield ActiveSupport::Notifications::Event.new(*args) if block_given?
+      end
+    end
+
+    def self.unsubscribe(subscriber)
+      ActiveSupport::Notifications.unsubscribe(subscriber)
+    end
+  end
+end

data/lib/messaging/message.rb

@@ -0,0 +1,103 @@
+require 'active_support'
+require 'active_support/concern'
+require 'active_support/core_ext/class'
+require 'active_support/core_ext/class/attribute'
+require 'active_support/core_ext/time'
+require 'dry-equalizer'
+
+module Messaging
+  module Message
+    extend ActiveSupport::Concern
+
+    included do
+      include Virtus.model
+      include Dry::Equalizer(:attributes)
+
+      attr_accessor :stream_position
+      attr_accessor :expected_version
+
+      key_attribute :uuid
+
+      attribute :message_name,  String, default: ->(message, _) { message.class.message_name }
+
+      attribute :timestamp,     Time,   default: ->(*) { Time.current.utc }
+      attribute :uuid,          String, default: ->(*) { SecureRandom.uuid }
+    end
+
+    module ClassMethods
+      # By default the topic is the same as the name of the message.
+      # We change the / that would be set for a namespaced message as "/" isn't valid in a topic
+      # To change the topic for a message just set it to whatever you want in your class definition.
+      def topic(topic_name = nil)
+        @topic ||= topic_name&.to_s || default_topic_name
+      end
+
+      # The attribute which value should be used as the key of the message.
+      # Must specify an attribute if ordering is important.
+      def key_attribute(attribute = nil)
+        @key_attribute = attribute if attribute
+        @key_attribute
+      end
+
+      def default_topic_name
+        return superclass.topic if superclass.respond_to?(:topic)
+
+        message_name.gsub('/', '-')
+      end
+
+      # Shorcut for creating a new message and publishing it
+      #
+      # @param attributes [Hash] The attributes of the message
+      # @option attributes [:any, Integer] :expected_version  Concurrency control
+      # @return [Message] the message just published
+      # @raise [ExpectedVersion::Error] If the expected_version does not match
+      def publish(attributes)
+        new(attributes).publish
+      end
+
+      def message_name
+        name.underscore
+      end
+
+      def message_type
+        to_s
+      end
+    end
+
+    # We do our own conversion for datetimes as by default they will only
+    # be stored with microseconds, which makes us loose precision when we
+    # serialize to json and back to message objects
+    def attributes_as_json(*_args)
+      attributes.transform_values { |v| v.respond_to?(:nsec) ? v.iso8601(9) : v }
+    end
+
+    def as_json(*_args)
+      attributes_as_json.merge(stream_position: stream_position)
+    end
+
+    def message_key
+      attributes[key_attribute&.to_sym]
+    end
+
+    def key_attribute
+      self.class.key_attribute
+    end
+
+    def topic
+      self.class.topic
+    end
+
+    def stream_name
+      # define stream_name in your message class to override
+      nil
+    end
+
+    def message_type
+      self.class.to_s
+    end
+
+    def publish
+      Messaging::Publish.(message: self)
+    end
+  end
+end

data/lib/messaging/message/from_json.rb

@@ -0,0 +1,31 @@
+module Messaging
+  module Message
+    class FromJson
+
+      def self.call(json)
+        new(json).to_message
+      end
+
+      def initialize(json)
+        @attributes = JSON.parse(json, symbolize_names: true)
+        @klass = message_class_from_name(@attributes[:message_name])
+      end
+
+      def empty_message
+        Hash[@klass.attribute_set.map { |a| [a.name, nil] }]
+      end
+
+      def new_without_defaults(attributes)
+        @klass.new(empty_message.merge(attributes))
+      end
+
+      def to_message
+        new_without_defaults(@attributes)
+      end
+
+      def message_class_from_name(message_name)
+        "::#{message_name.camelize}".constantize
+      end
+    end
+  end
+end

data/lib/messaging/meter.rb

@@ -0,0 +1,113 @@
+ActiveSupport::Notifications.subscribe /request.connection.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  if event.payload.key?(:exception)
+    tags = {
+      api: event.payload.fetch(:api, 'unknown'),
+      broker: event.payload.fetch(:broker_host)
+    }
+    Meter.increment('messaging.api.errors', tags: tags)
+  end
+end
+
+ActiveSupport::Notifications.subscribe /process_message.consumer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+
+  tags = {
+    group_id: event.payload.fetch(:group_id),
+    topic: event.payload.fetch(:topic),
+    partition: event.payload.fetch(:partition),
+  }
+
+  if event.payload.key?(:exception)
+    Meter.increment('messaging.consumer.process_message.errors', tags: tags)
+  else
+    Meter.increment('messaging.consumer.messages', tags: tags)
+  end
+end
+
+ActiveSupport::Notifications.subscribe /process_batch.consumer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  message_count = event.payload.fetch(:message_count)
+
+  tags = {
+    group_id: event.payload.fetch(:group_id),
+    topic: event.payload.fetch(:topic),
+    partition: event.payload.fetch(:partition),
+  }
+
+  if event.payload.key?(:exception)
+    Meter.increment('messaging.consumer.process_batch.errors', tags: tags)
+  else
+    Meter.increment('messaging.consumer.messages', value: message_count, tags: tags)
+  end
+end
+
+ActiveSupport::Notifications.subscribe /produce_message.producer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  topic = event.payload.fetch(:topic)
+  buffer_size = event.payload.fetch(:buffer_size)
+  max_buffer_size = event.payload.fetch(:max_buffer_size)
+  buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
+
+  # This gets us the write rate.
+  Meter.increment('messaging.producer.produce.messages', tags: { topic: topic })
+
+  # This gets us the avg/max buffer size per producer.
+  Meter.histogram('messaging.producer.buffer.size', buffer_size)
+
+  # This gets us the avg/max buffer fill ratio per producer.
+  Meter.histogram('messaging.producer.buffer.fill_ratio', buffer_fill_ratio)
+end
+
+ActiveSupport::Notifications.subscribe /buffer_overflow.producer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  tags = {
+    topic: event.payload.fetch(:topic)
+  }
+
+  Meter.increment('messaging.producer.produce.errors', tags: tags)
+end
+
+ActiveSupport::Notifications.subscribe /deliver_messages.producer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  message_count = event.payload.fetch(:delivered_message_count)
+  attempts = event.payload.fetch(:attempts)
+
+  Meter.increment('messaging.producer.deliver.errors') if event.payload.key?(:exception)
+
+  # Messages delivered to Kafka:
+  Meter.increment('messaging.producer.deliver.messages', value: message_count)
+
+  # Number of attempts to deliver messages:
+  Meter.histogram('messaging.producer.deliver.attempts', attempts)
+end
+
+ActiveSupport::Notifications.subscribe /topic_error.producer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  tags = {
+    topic: event.payload.fetch(:topic)
+  }
+
+  Meter.increment('messaging.producer.ack.errors', tags: tags)
+end
+
+ActiveSupport::Notifications.subscribe /enqueue_message.async_producer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  queue_size = event.payload.fetch(:queue_size)
+  max_queue_size = event.payload.fetch(:max_queue_size)
+  queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
+
+  # This gets us the avg/max queue size per producer.
+  Meter.histogram('messaging.async_producer.queue.size', queue_size)
+
+  # This gets us the avg/max queue fill ratio per producer.
+  Meter.histogram('messaging.async_producer.queue.fill_ratio', queue_fill_ratio)
+end
+
+ActiveSupport::Notifications.subscribe /buffer_overflow.async_producer.kafka/ do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  tags = {
+    topic: event.payload.fetch(:topic)
+  }
+  Meter.increment('messaging.async_producer.produce.errors', tags: tags)
+end

data/lib/messaging/middleware.rb

@@ -0,0 +1,14 @@
+module Messaging
+  module Middleware
+    def self.run(middlewares, *args)
+      chain = Array(middlewares.dup)
+
+      call_stack = lambda do
+        return yield if chain.empty?
+
+        chain.shift.call(*args, &call_stack)
+      end
+      call_stack.call
+    end
+  end
+end

data/lib/messaging/middleware/after_active_record_transaction.rb

@@ -0,0 +1,13 @@
+require 'after_transaction'
+
+module Messaging
+  module Middleware
+    class AfterActiveRecordTransaction
+      def self.call(*_args)
+        AfterTransaction.call do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/messaging/middleware/rails_wrapper.rb

@@ -0,0 +1,22 @@
+module Messaging
+  module Middleware
+    # Used for hot code reloading (in development) and connection handling.
+    # This makes Rails automatically return AR connections to the pool after we
+    # have used them when processing a messages.
+    #
+    # See https://guides.rubyonrails.org/threading_and_code_execution.html for more information.
+    class RailsWrapper
+      attr_accessor :app
+
+      def initialize(app)
+        self.app = app
+      end
+
+      def call(*_args)
+        app.reloader.wrap do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/messaging/publish.rb

@@ -0,0 +1,33 @@
+module Messaging
+  class Publish
+    include MethodObject
+
+    option :message
+
+    option :message_store, default: -> { Messaging.message_store }
+    option :dispatcher, default: -> { Messaging.dispatcher }
+    option :middlewares, default: -> { Config.dispatcher.middlewares }
+
+    class << self
+      # @!method call(message:, stream:)
+      #   Publishes the message
+      #
+      #   This will persist the message first (if given a stream),
+      #   if that succeeds it will be dispatched to the configured
+      #   dispatcher and all handlers configured in the routes.
+      #   @param message [Message] The message to publish
+      #   @return [Message] the message just published
+    end
+
+    # @see .call
+    # @!visibility private
+    def call
+      persisted_message = message_store.call(message)
+      Middleware.run(middlewares, persisted_message) do
+        dispatcher.call(persisted_message)
+        Messaging.routes.handle persisted_message
+      end
+      persisted_message
+    end
+  end
+end