messaging 0.0.2 → 3.4.1

data/lib/messaging/adapters/postgres/store.rb

@@ -0,0 +1,66 @@
+require_relative 'stream'
+require_relative 'streams'
+
+module Messaging
+  module Adapters
+    class Postgres
+      # Message store adapter using Postgres and ActiveRecord.
+      # Prefer accessing the message store through Messaging.message_store
+      # instead of using it directly.
+      #
+      # @example Using Postgres as the default message store adapter:
+      #   # Put this in an initializer
+      #   Messaging.setup do |config|
+      #     config.message_store.adapter = :postgres
+      #   end
+      #
+      # @see Messaging.message_store
+      class Store
+        # @return [Streams] all the streams in the store
+        # @see Streams
+        attr_reader :streams
+
+        # Should not be used directly. Access the store though
+        # Messaging.message_store or Messaging::Adapters::Store[:postgres]
+        # @api private
+        def initialize
+          @streams = Streams.new
+        end
+
+        # Get a specific stream by name
+        # @return [Stream]
+        # @see Messaging.stream
+        def stream(name)
+          streams[name]
+        end
+
+        # Access to all messages.
+        # Use with caution in production as there are probably
+        # a lot of messages so queries could take a long time or timeout.
+        #
+        # @example Check that a message has been added to the store with Rspec
+        #   expect do
+        #     # Your code that should add a message to the store
+        #   end.to change { Messaging.message_store.messages.count }.from(0).to(1)
+        #
+        # @return [SerializedMessage]
+        def messages
+          SerializedMessage
+        end
+
+        # Writes the message to Postgres
+        # Skips messages that hasn't defined a stream name
+        # We do this to begin with so PG is opt-in per message
+        #
+        # @param message [Messaging::Message] The message to persist
+        # @return [Messaging::Message] A new copy of the message with
+        #   the stream position added (if persisted)
+        def call(message)
+          return message unless message.stream_name
+
+          SerializedMessage.create!(message: message).to_message
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/stream.rb

@@ -0,0 +1,37 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class Stream
+        # @return [String] the name of the stream
+        attr_reader :name
+
+        # Should not be used directly.
+        # Use {Messaging.stream} or {Store#stream}
+        # @api private
+        # @see Messaging.stream
+        # @see Store.stream
+        def initialize(name)
+          @name = name
+        end
+
+        # Access to all messages in the stream sorted by stream_position
+        # @return [ActiveRecord::Relation]
+        def messages
+          SerializedMessage.where(stream: name).order(:stream_position)
+        end
+
+        # The current position of the last message in the stream
+        # @return [-1] if the stream is empty -1 will be returned
+        # @return [Integer] the stream position of the last message in the stream
+        def current_position
+          messages.maximum(:stream_position) || -1
+        end
+
+        def inspect
+          info = "current_position: #{current_position}"
+          "#<Stream:#{name}> #{info}>"
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/streams.rb

@@ -0,0 +1,27 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class Streams
+        include Enumerable
+
+        def each
+          return enum_for(:each) unless block_given?
+
+          all_streams.each do |name|
+            yield Stream.new(name)
+          end
+        end
+
+        def [](name)
+          Stream.new(name)
+        end
+
+        private
+
+        def all_streams
+          SerializedMessage.distinct.pluck(:stream).lazy
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/test.rb

@@ -0,0 +1,59 @@
+require_relative 'test/consumer'
+require_relative 'test/store'
+module Messaging
+  module Adapters
+    # Public: An adapter useful for test.
+    #
+    # Do NOT use this outside tests.
+    # All messages will be saved in memory until cleared.
+    class Test
+      # Public: Used to inspect what have been published
+      #
+      # It's a hash keyed on the topic name.
+      # Each value is an array of the messages that has been published to the topic
+      attr_reader :topics
+
+      def initialize
+        # See https://www.rubytapas.com/2013/01/11/episode-045-hash-default-value/
+        # for why this is needed for default values in a Hash
+        @topics = Hash.new { |h, k| h[k] = [] }
+      end
+
+      def call(message)
+        topics[message.topic] << message
+        Messaging.routes.consumers.each do |c|
+          Messaging.logger.info "Sending #{message} to #{c.queue} with length #{c.queue.length}"
+          c.queue << message
+        end
+      end
+
+      def create_consumer(name, **options)
+        Consumer.new(name: name, **options)
+      end
+
+      # Resests the topics to a blank state.
+      #
+      # Useful to run before each example in specs to not share state.
+      def clear_topics
+        topics.clear
+      end
+
+      # @api private
+      def store
+        @store ||= Store.new
+      end
+
+      def self.register!
+        return if Adapters.key? :test
+
+        Adapters.register(:test, memoize: true) { Test.new }
+        Adapters::Consumer.register(:test, memoize: true) { Adapters[:test] }
+        Adapters::Dispatcher.register(:test, memoize: true) { Adapters[:test] }
+        Adapters::Store.register(:test, memoize: true) { Adapters[:test].store }
+      end
+      private_class_method :register!
+
+      register!
+    end
+  end
+end

data/lib/messaging/adapters/test/consumer.rb

@@ -0,0 +1,46 @@
+require 'dry-initializer'
+module Messaging
+  module Adapters
+    class Test
+      class Consumer
+        include Messaging::Routing
+        extend Dry::Initializer
+
+        option :name
+
+        option :logger,   default: -> { Config.logger }
+        option :queue,    default: -> { Queue.new }
+
+        def start
+          logger.info "Consumer #{name} started"
+          @running = true
+          process_messages
+        ensure
+          shutdown
+        end
+
+        def stop
+          @running = false
+          queue << :stop
+          logger.info "Consumer #{name} stopping"
+        end
+
+        def shutdown
+          logger.info "Consumer #{name} stopped"
+        end
+
+        private
+
+        def process_messages
+          while @running
+            message = queue.pop
+            logger.info "Consumer #{name} got #{message}"
+            break if message == :stop
+
+            RunMiddlewares.call(Config.consumer.middlewares, message) { handle message }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/test/store.rb

@@ -0,0 +1,67 @@
+require_relative 'stream'
+
+module Messaging
+  module Adapters
+    class Test
+      # Message store adapter used for testing.
+      # It stores all messages in memory.
+      # Prefer accessing the message store through Messaging.message_store
+      # instead of using it directly.
+      #
+      # @example Using Test as the default message store adapter during tests:
+      #   # Put this in an initializer
+      #   Messaging.setup do |config|
+      #     config.message_store.adapter = Rails.env.test? ? :test : :postgres
+      #   end
+      #
+      # @see Messaging.message_store
+      class Store
+        # Access to all messages.
+        #
+        # @example Check that a message has been added to the store with Rspec
+        #   expect do
+        #     # Your code that should add a message to the store
+        #   end.to change { Messaging.message_store.messages.count }.from(0).to(1)
+        #
+        # @return [Array<Messaging::Message>]
+        attr_reader :messages
+        # @return [Streams] all the streams in the store
+        # @see Stream
+        attr_reader :streams
+
+        def initialize
+          clear!
+        end
+
+        # Get a specific stream by name
+        # @return [Stream]
+        # @see Messaging.strea
+        def stream(name)
+          streams[name]
+        end
+
+        def clear!
+          @streams = Hash.new { |h, k| h[k] = Stream.new(k) }
+          @messages = []
+        end
+
+        # Writes the message to the store
+        # Skips messages that hasn't defined a stream name
+        #
+        # @param message [Messaging::Message] The message to persist
+        # @return [Messaging::Message] A new copy of the message with
+        #   the stream position added (if persisted)
+        def call(message)
+          return message unless message.stream_name
+
+          stream = stream(message.stream_name)
+          ExpectedVersion.new(message.expected_version || :any).match!(stream.current_position)
+          persisted_message = message.class.new(message.attributes.merge(stream_position: stream.current_position + 1))
+          @messages << persisted_message
+          stream.messages << persisted_message
+          persisted_message
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/test/stream.rb

@@ -0,0 +1,21 @@
+module Messaging
+  module Adapters
+    class Test
+      class Stream
+        attr_accessor :name
+
+        def initialize(name)
+          self.name = name
+        end
+
+        def messages
+          @messages ||= []
+        end
+
+        def current_position
+          messages.length - 1
+        end
+      end
+    end
+  end
+end

data/lib/messaging/base_handler.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Messaging
+  # @private
+  # @deprecated
+  class BaseHandler
+    class << self
+      def only_messages(messages = nil)
+        @only_messages = Array(messages) if messages
+        @only_messages || []
+      end
+      alias only_message only_messages
+
+      def group_id(group_id = nil)
+        @group_id = group_id if group_id
+        @group_id || default_group_id
+      end
+
+      def default_group_id
+        Config.app_name + '-' + name.underscore
+      end
+
+      def call(message)
+        return unless allowed_message?(message.class)
+
+        new.on_message(message, nil)
+      end
+
+      def listen_on(topic:)
+        topics = Array(topic).map(&:to_s)
+        Messaging.routes.consumer(name, group_id: group_id) do |c|
+          topics.each { |t| c.on(/.*/, topic: t, call: self) }
+        end
+      end
+
+      def allowed_message?(message_class)
+        return true if only_messages.empty?
+
+        only_messages.include? message_class
+      end
+    end
+
+    def on_message(message, _metadata); end
+  end
+end

data/lib/messaging/cli.rb

@@ -0,0 +1,62 @@
+require 'messaging'
+require_relative 'consumer_supervisor'
+
+module Messaging
+  # @private
+  # Starts up the consumer supervisor and handles signals
+  class CLI
+
+    SIGNALS = %i[INT TERM QUIT].freeze
+
+    def initialize
+      @signal_queue = []
+      @reader, @writer = IO.pipe
+      @consumer_supervisor = Messaging::ConsumerSupervisor.new
+    end
+
+    def run
+      setup_signals
+      consumer_supervisor.start
+      wait_for_signal
+    end
+
+    # See https://www.sitepoint.com/the-self-pipe-trick-explained/ for background on this
+    def wait_for_signal
+      loop do
+        case signal_queue.pop
+        when *SIGNALS
+          consumer_supervisor.stop
+          break
+        when "USR1"
+          consumer_supervisor.status
+        else
+          ready = IO.select([reader, writer])
+
+          # drain the self-pipe so it won't be returned again next time
+          reader.read_nonblock(1) if ready[0].include?(reader)
+        end
+      end
+    end
+
+    private
+
+    attr_reader :reader, :writer, :signal_queue, :consumer_supervisor
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def setup_signals
+      SIGNALS.each do |signal|
+        Signal.trap(signal) { unblock(signal) }
+      end
+      Signal.trap("USR1") { unblock("USR1") }
+    end
+
+    def unblock(signal)
+      writer.write_nonblock('.')
+      signal_queue << signal
+    end
+
+  end
+end

data/lib/messaging/config.rb

@@ -0,0 +1,66 @@
+module Messaging
+  class Config
+    extend ::Dry::Configurable
+
+    setting :app_name, 'messaging', reader: true
+
+    setting(:error_handlers, [], reader: true) { |value| Array(value) }
+
+    setting :background_job_handler, ->(message, handler_name) { handler_name.constantize.call(message) }, reader: true
+
+    setting :logger, Logger.new(STDOUT), reader: true
+
+    setting :consumer, reader: true do
+      setting :adapter, :kafka
+      setting(:middlewares, []) { |value| Array(value) }
+    end
+
+    setting :dispatcher, reader: true do
+      setting :adapter, :kafka
+      setting(:middlewares, []) { |value| Array(value) }
+    end
+
+    setting :message_store, reader: true do
+      setting :adapter, :postgres
+      setting(:middlewares, []) { |value| Array(value) }
+    end
+
+    setting :kafka, reader: true do
+      setting :log_level, :warn
+      setting :pause_timeout, 10
+
+      setting :client do
+        setting :seed_brokers, ['localhost:9092']
+        setting :connect_timeout
+        setting :socket_timeout
+        setting :ssl_ca_certs_from_system
+        setting :sasl_plain_username
+        setting :sasl_plain_password
+        setting :ssl_ca_cert, ENV['KAFKA_SSL_CA']
+        setting :ssl_client_cert, ENV['KAFKA_SSL_CERTIFICATE']
+        setting :ssl_client_cert_key, ENV['KAFKA_SSL_KEY']
+      end
+
+      setting :consumer do
+        setting :session_timeout, 30
+        setting :offset_commit_interval, 10
+        setting :offset_commit_threshold, 0
+        setting :heartbeat_interval, 10
+      end
+
+      setting :producer do
+        setting :max_queue_size, 5_000
+        setting :delivery_threshold, 10
+        setting :delivery_interval, 0.05
+      end
+    end
+
+    class << self
+      def setup
+        configure do |config|
+          yield(config)
+        end
+      end
+    end
+  end
+end