messaging 0.0.2 → 3.4.1

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/messaging

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+STDOUT.sync = true
+
+# Load Rails environment if it exists
+env_path = File.expand_path('config/environment.rb')
+require env_path if File.exist? env_path
+
+require_relative '../lib/messaging/cli'
+
+cli = Messaging::CLI.new
+cli.run

data/lib/messaging.rb

@@ -1,17 +1,68 @@
+require 'concurrent'
+require 'dry-configurable'
+require 'virtus'
+require 'active_support'
+require 'active_support/notifications'
+require 'active_support/core_ext'
+
+require 'messaging/config'
+require 'messaging/expected_version'
+require 'messaging/instrumentation'
+require 'messaging/meter'
+require 'messaging/exception_handler'
+require 'messaging/message'
+require 'messaging/message/from_json'
+require 'messaging/routing'
+require 'messaging/routes'
+require 'messaging/base_handler'
+require 'messaging/adapters'
+require 'messaging/middleware'
+require 'messaging/publish'
+require 'messaging/rails/railtie' if defined? ::Rails
+
 module Messaging
-  module Models
-    autoload :Messageable, 'messaging/models/messageable'
+  def self.setup(&block)
+    Config.setup(&block)
+  end
+
+  def self.config
+    Config.config
+  end
+
+  def self.logger
+    Config.logger
+  end
+
+  def self.consumer_adapter
+    Adapters[Config.consumer.adapter]
+  end
+
+  def self.dispatcher
+    Adapters::Dispatcher.default
   end
 
-  mattr_accessor :user_class
-  mattr_accessor :attachment_storage
-  mattr_accessor :allowed_attachment_filetypes
+  def self.message_store
+    Adapters::Store.default
+  end
+
+  def self.[](adapter)
+    Adapters[adapter]
+  end
+
+  def self.routes
+    @routes ||= Routes.new
+  end
+
+  def self.stream(name)
+    name = name.stream_name if name.respond_to?(:stream_name)
+    message_store.stream(name)
+  end
+
+  def self.defined_messages
+    ObjectSpace.each_object(Class).select { |c| c.included_modules.include? Messaging::Message }
+  end
 
-  require 'messaging/engine'
-  require 'messaging/concerns/configurable_mailer'
-  class << self
-    def setup
-      yield self
-    end
+  def self.in_consumer_mode?
+    File.basename($PROGRAM_NAME) == 'messaging'
   end
 end

data/lib/messaging/adapters.rb

@@ -0,0 +1,35 @@
+require 'dry/container'
+
+module Messaging
+  module Adapters
+    extend Dry::Container::Mixin
+
+    class Consumer
+      extend Dry::Container::Mixin
+
+      def self.default
+        resolve(Config.consumer.adapter)
+      end
+    end
+
+    class Dispatcher
+      extend Dry::Container::Mixin
+
+      def self.default
+        resolve(Config.dispatcher.adapter)
+      end
+    end
+
+    class Store
+      extend Dry::Container::Mixin
+
+      def self.default
+        resolve(Config.message_store.adapter)
+      end
+    end
+  end
+end
+
+require 'messaging/adapters/kafka'
+require 'messaging/adapters/postgres'
+require 'messaging/adapters/test'

data/lib/messaging/adapters/kafka.rb

@@ -0,0 +1,55 @@
+require 'kafka'
+
+require_relative 'kafka/consumer'
+require_relative 'kafka/producer'
+
+module Messaging
+  module Adapters
+    # Internal: Adapter for producing and consuming messages with Kafka
+    class Kafka
+      # Internal: Ruby-Kafka client for the current thread.
+      #
+      # We keep one per thread as the client itself is not meant to be shared.
+      # See https://github.com/zendesk/ruby-kafka#thread-safety for more information.
+      def client
+        Thread.current[:messaging_kafka_client] ||= create_kafka_client
+      end
+
+      # The producer doesn't need to be per thread as it
+      # uses a background thread to deliver messages.
+      def dispatcher
+        @dispatcher ||= Producer.new(self)
+      end
+
+      def create_consumer(name, **options)
+        Consumer.new(name: name, kafka_adapter: self, **options)
+      end
+
+      # Internal: Setup a Ruby-Kafka consumer
+      def create_kafka_consumer(group_id)
+        client.consumer({ group_id: group_id }.merge(Config.kafka.consumer.to_hash))
+      end
+
+      private
+
+      # Internal: Setup a Ruby-Kafka client
+      def create_kafka_client
+        # We have a specific logger for Ruby-Kafka as it
+        # is a bit more noisy than we would like.
+        kafka_logger = Messaging.logger.dup.tap { |logger| logger.level = Config.kafka.log_level }
+       ::Kafka.new(Config.kafka.client.to_hash.merge(client_id: Config.app_name, logger: kafka_logger))
+      end
+
+      def self.register!
+        return if Adapters.key? :kafka
+
+        Adapters.register(:kafka, memoize: true) { Kafka.new }
+        Adapters::Consumer.register(:kafka, memoize: true) { Adapters[:kafka] }
+        Adapters::Dispatcher.register(:kafka, memoize: true) { Adapters[:kafka].dispatcher }
+      end
+      private_class_method :register!
+
+      register!
+    end
+  end
+end

data/lib/messaging/adapters/kafka/consumer.rb

@@ -0,0 +1,101 @@
+require 'dry-initializer'
+module Messaging
+  module Adapters
+    class Kafka
+      # Internal: Wraps a Ruby-Kafka consumer
+      #
+      # Subscribes to topics and dispatches messages
+      # to a handler for doing the actual work.
+      class Consumer
+        include Messaging::Instrumentation
+        include Messaging::Routing
+        extend Dry::Initializer
+
+        option :kafka_adapter
+        option :name
+
+        option :group_id, default: -> { Config.app_name + '-' + name.to_s.underscore }
+        option :logger,   default: -> { Messaging.logger }
+        option :consumer, default: -> { kafka_adapter.create_kafka_consumer(group_id) }
+
+        def start
+          logger.info "Consumer #{name} started"
+          subscribe_kafka_consumer_to_topics
+          @running = true
+          process_messages
+        ensure
+          stop if @running
+        end
+
+        def stop
+          logger.info "Consumer #{name} stopping"
+          @running = false
+          consumer&.stop
+        end
+
+        def log_current_status
+          logger.info "Current offsets for #{name}:"
+          logger.info '|- no messages yet' if current_offsets.empty?
+          current_offsets.each { |partition, offset| logger.info "|- #{partition}: #{offset}" }
+        end
+
+        private
+
+        def subscribe_kafka_consumer_to_topics
+          topics.each { |topic| consumer.subscribe(topic, start_from_beginning: false) }
+        end
+
+        def process_messages
+          consumer.each_message do |message|
+            process_message(message)
+            current_offsets["#{message.topic}#{message.partition}"] = message.offset
+          end
+        rescue ::Kafka::ProcessingError => e
+          report_processing_exception(e)
+          pause topic: e.topic, partition: e.partition
+
+          retry if @running
+        end
+
+        def current_offsets
+          @current_offsets ||= {}
+        end
+
+        def report_processing_exception(e)
+          logger.error "Error processing partition #{e.topic}/#{e.partition} at offset #{e.offset}"
+          ExceptionHandler.call(
+            e.cause,
+            kafka: {
+              consumer:  name,
+              topic:     e.topic,
+              partition: e.partition,
+              offset:    e.offset
+            }
+          )
+        end
+
+        def pause(topic:, partition:, timeout: Config.kafka.pause_timeout)
+          return unless timeout.positive?
+
+          logger.warn "Pausing partition #{topic}/#{partition} for #{timeout} s"
+          consumer.pause(topic, partition, timeout: timeout)
+        end
+
+        def process_message(raw_message)
+          metadata = {
+            group_id:  group_id,
+            key:       raw_message.key,
+            topic:     raw_message.topic,
+            partition: raw_message.partition,
+            offset:    raw_message.offset
+          }
+
+          message = Message::FromJson.(raw_message.value)
+          instrument('consumer.process_message', metadata) do
+            Middleware.run(Config.consumer.middlewares, message) { handle message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/kafka/producer.rb

@@ -0,0 +1,52 @@
+module Messaging
+  module Adapters
+    class Kafka
+      class Producer
+        attr_reader :producer
+        attr_reader :pid
+        attr_reader :kafka
+
+        def initialize(kafka_adapter)
+          @kafka = kafka_adapter
+          connect
+
+          at_exit { shutdown }
+        end
+
+        # Delivers a message to Kafka asynchronously in a background thread.
+        # This method will return immediately.
+        #
+        # @param message
+        def call(message)
+          reconnect if forked?
+          producer.produce(message.to_json, key: message.message_key, topic: message.topic)
+        rescue ::Kafka::BufferOverflow => exception
+          Trouble.notify(exception, pid: Process.pid, message: message.to_json)
+        end
+
+        def shutdown
+          return unless producer
+
+          producer.deliver_messages
+          producer.shutdown
+        end
+
+        def connect
+          @producer = create_producer
+          @pid = Process.pid
+        end
+        alias reconnect connect
+
+        private
+
+        def forked?
+          Process.pid != pid
+        end
+
+        def create_producer
+          kafka.client.async_producer(Config.kafka.producer.to_hash)
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres.rb

@@ -0,0 +1,23 @@
+require 'active_record'
+require_relative 'postgres/serialized_message'
+require_relative 'postgres/advisory_transaction_lock'
+
+require_relative 'postgres/store'
+module Messaging
+  module Adapters
+    # Adapter for using Postgres and Active Record as a message store.
+    # @see Messaging::Adapters::Postgres::Store Store - for more information on how to use the message store
+    # capabilities provided by this adapter.
+    class Postgres
+      def self.register!
+        return if Adapters.key? :postgres
+
+        Adapters.register(:postgres, memoize: true) { Postgres.new }
+        Adapters::Store.register(:postgres, memoize: true) { Store.new }
+      end
+      private_class_method :register!
+
+      register!
+    end
+  end
+end

data/lib/messaging/adapters/postgres/advisory_transaction_lock.rb

@@ -0,0 +1,28 @@
+require 'method_object'
+
+module Messaging
+  module Adapters
+    class Postgres
+      # Creates an advisory lock that is held during the rest of the transaction.
+      # If another session is trying to aquire the same lock it will wait until the
+      # lock is released.
+      # @api private
+      class AdvisoryTransactionLock
+        include MethodObject
+
+        option :key
+        option :connection, default: -> { SerializedMessage.connection }
+
+        def call
+          connection.execute "SELECT pg_advisory_xact_lock(#{lock_key});"
+        end
+
+        private
+
+        def lock_key
+          Zlib.crc32(key)
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/serialized_message.rb

@@ -0,0 +1,74 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class SerializedMessage < ActiveRecord::Base
+        self.table_name = :messaging_messages
+
+        attr_accessor :expected_version
+
+        # We override this AR method to make records retreived
+        # from scopes etc. be message objects of the corresponding
+        # message class instead of AR objects
+        #
+        # See https://api.rubyonrails.org/classes/ActiveRecord/Persistence/ClassMethods.html#method-i-instantiate
+        def self.instantiate(attributes, *_args)
+          attributes['message_type'].constantize.new(
+            JSON.parse(attributes['data']).merge(stream_position: attributes['stream_position'])
+          )
+        end
+
+        # Virtual setter for message so we can create a serialized message from a message
+        def message=(message)
+          self.data = message.attributes_as_json
+          self.expected_version = message.expected_version
+          self.message_type = message.message_type
+          self.stream = message.stream_name
+          self.uuid = message.uuid
+        end
+
+        def to_message
+          message_type.constantize.new(data.merge(stream_position: stream_position))
+        end
+
+        # You should never update a message after creating it
+        def readonly?
+          true unless new_record?
+        end
+
+        def stream
+          @stream ||= Stream.new(attributes['stream'])
+        end
+
+        def create_or_update(*args)
+          with_locked_stream do
+            set_stream_position
+            validate_expected_version!
+            super
+          end
+        end
+
+        private
+
+        # Memoize the current version to not make multiple SQL queries on create
+        def current_stream_position
+          @current_stream_position ||= stream.current_position
+        end
+
+        def set_stream_position
+          self.stream_position = current_stream_position + 1
+        end
+
+        def validate_expected_version!
+          ExpectedVersion.new(expected_version || :any).match!(current_stream_position)
+        end
+
+        def with_locked_stream
+          transaction do
+            AdvisoryTransactionLock.call key: stream.name
+            yield
+          end
+        end
+      end
+    end
+  end
+end