rimless 0.1.0

data/lib/rimless/configuration.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Rimless
+  # The configuration for the rimless gem.
+  class Configuration
+    include ActiveSupport::Configurable
+
+    # Used to identity this client on the user agent header
+    config_accessor(:app_name) { Rimless.local_app_name }
+
+    # Environment to use
+    config_accessor(:env) do
+      next(ENV.fetch('KAFKA_ENV', Rails.env).to_sym) if defined? Rails
+
+      ENV.fetch('KAFKA_ENV', 'development').to_sym
+    end
+
+    # The Apache Kafka client id (consumer group name)
+    config_accessor(:client_id) do
+      ENV.fetch('KAFKA_CLIENT_ID', Rimless.local_app_name)
+    end
+
+    # The logger instance to use (when available we use the +Rails.logger+)
+    config_accessor(:logger) do
+      next(Rails.logger) if defined? Rails
+
+      Logger.new($stdout)
+    end
+
+    # At least one broker of the Apache Kafka cluster
+    config_accessor(:kafka_brokers) do
+      ENV.fetch('KAFKA_BROKERS', 'kafka://message-bus.local:9092').split(',')
+    end
+
+    # The source Apache Avro schema files location (templates)
+    config_accessor(:avro_schema_path) do
+      path = %w[config avro_schemas]
+      next(Rails.root.join(*path)) if defined? Rails
+
+      Pathname.new(Dir.pwd).join(*path)
+    end
+
+    # The compiled Apache Avro schema files location (usable with Avro gem)
+    config_accessor(:compiled_avro_schema_path) do
+      path = %w[config avro_schemas compiled]
+      next(Rails.root.join(*path)) if defined? Rails
+
+      Pathname.new(Dir.pwd).join(*path)
+    end
+
+    # The Confluent Schema Registry API URL to use
+    config_accessor(:schema_registry_url) do
+      ENV.fetch('KAFKA_SCHEMA_REGISTRY_URL',
+                'http://schema-registry.message-bus.local')
+    end
+  end
+end

data/lib/rimless/configuration_handling.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Rimless
+  # The top-level configuration handling.
+  #
+  # rubocop:disable Style/ClassVars because we split module code
+  module ConfigurationHandling
+    extend ActiveSupport::Concern
+
+    class_methods do
+      # Retrieve the current configuration object.
+      #
+      # @return [Configuration]
+      def configuration
+        @@configuration ||= Configuration.new
+      end
+
+      # Configure the concern by providing a block which takes
+      # care of this task. Example:
+      #
+      #   FactoryBot::Instrumentation.configure do |conf|
+      #     # conf.xyz = [..]
+      #   end
+      def configure
+        yield(configuration)
+        configure_dependencies
+      end
+
+      # Reset the current configuration with the default one.
+      def reset_configuration!
+        @@configuration = Configuration.new
+      end
+
+      # Retrieve the current configured environment. You can use it like
+      # +Rails.env+ to query it. E.g. +Rimless.env.production?+.
+      #
+      # @return [ActiveSupport::StringInquirer] the environment
+      def env
+        @@env = ActiveSupport::StringInquirer.new(configuration.env.to_s) \
+          if @env.to_s != configuration.env.to_s
+        @@env
+      end
+
+      # A simple convention helper to setup Apache Kafka topic names.
+      #
+      # @param app [String] the application namespace
+      # @return [String] the Apache Kafka topic name prefix
+      def topic_prefix(app = Rimless.configuration.app_name)
+        "#{Rimless.env}.#{app}."
+      end
+
+      # Pass back the local application name. When we are loaded together with
+      # a Rails application we use the application class name. This
+      # application name is URI/GID compatible. When no local application is
+      # available, we just pass back +nil+.
+      #
+      # @return [String, nil] the Rails application name, or +nil+
+      def local_app_name
+        # Check for non-Rails integration
+        return unless defined? Rails
+        # Check if a application is defined
+        return if Rails.application.nil?
+
+        # Pass back the URI compatible application name
+        Rails.application.class.parent_name.underscore.dasherize
+      end
+
+      # Retrieve the current configured logger instance.
+      #
+      # @return [Logger] the logger instance
+      delegate :logger, to: :configuration
+    end
+  end
+  # rubocop:enable Style/ClassVars
+end

data/lib/rimless/dependencies.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Rimless
+  # The top-level dependencies helpers.
+  module Dependencies
+    extend ActiveSupport::Concern
+
+    class_methods do
+      # (Re)configure our gem dependencies. We take care of setting up
+      # +WaterDrop+, our Apache Kafka driver and +AvroTurf+, our Confluent
+      # Schema Registry driver.
+      def configure_dependencies
+        configure_waterdrop
+        configure_avro_turf
+      end
+
+      # Set sensible defaults for the +WaterDrop+ gem.
+      def configure_waterdrop
+        WaterDrop.setup do |config|
+          # Activate message delivery and use the default logger
+          config.deliver = true
+          config.logger = Rimless.logger
+          # An optional identifier of a Kafka consumer (in a consumer group)
+          # that is passed to a Kafka broker with every request. A logical
+          # application name to be included in Kafka logs and monitoring
+          # aggregates.
+          config.client_id = Rimless.configuration.client_id
+          # All the known brokers, at least one. The ruby-kafka driver will
+          # discover the whole cluster structure once and when issues occur to
+          # dynamically adjust scaling operations.
+          config.kafka.seed_brokers = Rimless.configuration.kafka_brokers
+          # All brokers MUST acknowledge a new message
+          config.kafka.required_acks = -1
+        end
+      end
+
+      # Set sensible defaults for the +AvroTurf+ gem and (re)compile the Apache
+      # Avro schema templates (ERB), so the gem can handle them properly.
+      def configure_avro_turf
+        # Setup a global available Apache Avro decoder/encoder with support for
+        # the Confluent Schema Registry, but first create a helper instance
+        avro_utils = Rimless::AvroUtils.new
+        # Compile our Avro schema templates to ready-to-consume Avro schemas
+        avro_utils.recompile_schemas
+        # Register a global Avro messaging instance
+        Rimless.avro = AvroTurf::Messaging.new(
+          logger: Rimless.logger,
+          namespace: avro_utils.namespace,
+          schemas_path: avro_utils.output_path,
+          registry_url: Rimless.configuration.schema_registry_url
+        )
+      end
+    end
+  end
+end

data/lib/rimless/kafka_helpers.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Rimless
+  # The top-level Apache Kafka helpers.
+  module KafkaHelpers
+    extend ActiveSupport::Concern
+
+    # rubocop:disable Metrics/BlockLength because its an Active Support concern
+    class_methods do
+      # Generate a common topic name for Apache Kafka while taking care of
+      # configured prefixes.
+      #
+      # @param name [String, Symbol] the topic name
+      # @param app [String, Symbol] a different application name, by default
+      #   the local app
+      # @return [String] the complete topic name
+      #
+      # @example Name only
+      #   Rimless.topic(:users)
+      # @example Name with app
+      #   Rimless.topic(:users, app: 'test-api')
+      # @example Mix and match
+      #   Rimless.topic(name: 'test', app: :fancy_app)
+      #
+      # rubocop:disable Metrics/AbcSize because of the usage flexibility
+      def topic(*args)
+        opts = args.last
+        name = args.first if [String, Symbol].member?(args.first.class)
+
+        if opts.is_a?(Hash)
+          name = opts[:name] if opts.key?(:name)
+          app = opts[:app] if opts.key?(:app)
+        end
+
+        name ||= nil
+        app ||= Rimless.configuration.app_name
+
+        raise ArgumentError, 'No name given' if name.nil?
+
+        "#{Rimless.topic_prefix(app)}#{name}"
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      # Send a single message to Apache Kafka. The data is encoded according to
+      # the given Apache Avro schema. The destination Kafka topic may be a
+      # relative name, or a hash which is passed to the +.topic+ method to
+      # manipulate the application details. The message is send is a
+      # synchronous, blocking way.
+      #
+      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
+      # @param schema [String, Symbol] the Apache Avro schema to use
+      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
+      #   Apache Kafka topic
+      def sync_message(data:, schema:, topic:, **args)
+        encoded = Rimless.avro.encode(data, schema_name: schema.to_s)
+        sync_raw_message(data: encoded, topic: topic, **args)
+      end
+      alias_method :message, :sync_message
+
+      # Send a single message to Apache Kafka. The data is encoded according to
+      # the given Apache Avro schema. The destination Kafka topic may be a
+      # relative name, or a hash which is passed to the +.topic+ method to
+      # manipulate the application details. The message is send is an
+      # asynchronous, non-blocking way.
+      #
+      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
+      # @param schema [String, Symbol] the Apache Avro schema to use
+      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
+      #   Apache Kafka topic
+      def async_message(data:, schema:, topic:, **args)
+        encoded = Rimless.avro.encode(data, schema_name: schema.to_s)
+        async_raw_message(data: encoded, topic: topic, **args)
+      end
+
+      # Send a single message to Apache Kafka. The data is not touched, so you
+      # need to encode it yourself before you pass it in. The destination Kafka
+      # topic may be a relative name, or a hash which is passed to the +.topic+
+      # method to manipulate the application details. The message is send is a
+      # synchronous, blocking way.
+      #
+      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
+      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
+      #   Apache Kafka topic
+      def sync_raw_message(data:, topic:, **args)
+        args = args.merge(topic: topic(topic))
+        WaterDrop::SyncProducer.call(data, **args)
+      end
+      alias_method :raw_message, :sync_raw_message
+
+      # Send a single message to Apache Kafka. The data is not touched, so you
+      # need to encode it yourself before you pass it in. The destination Kafka
+      # topic may be a relative name, or a hash which is passed to the +.topic+
+      # method to manipulate the application details. The message is send is an
+      # asynchronous, non-blocking way.
+      #
+      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
+      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
+      #   Apache Kafka topic
+      def async_raw_message(data:, topic:, **args)
+        args = args.merge(topic: topic(topic))
+        WaterDrop::AsyncProducer.call(data, **args)
+      end
+    end
+    # rubocop:enable Metrics/BlockLength
+  end
+end

data/lib/rimless/railtie.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Rimless
+  # Rails-specific initializations.
+  class Railtie < Rails::Railtie
+    # Run before all Rails initializers, but after the application is defined
+    config.before_initialize do
+      conf = Rimless.configuration
+      app_name = Rimless.local_app_name
+
+      # Reset the default application name (which is +nil+), because the Rails
+      # application was not defined when the rimless gem was loaded
+      conf.app_name = app_name
+
+      # Set the app name as default client id, when not already set
+      conf.client_id ||= app_name
+    end
+
+    # Run after all configuration is set via Rails initializers
+    config.after_initialize do
+      # Reconfigure our dependencies
+      Rimless.configure_dependencies
+    end
+  end
+end

data/lib/rimless/rspec.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'webmock'
+require 'webmock/rspec'
+require 'avro_turf/test/fake_confluent_schema_registry_server'
+require 'rimless'
+require 'rimless/rspec/helpers'
+require 'rimless/rspec/matchers'
+
+# RSpec 1.x and 2.x compatibility
+#
+# @see http://bit.ly/2GbAYsU
+raise 'No RSPEC_CONFIGURER is defined, webmock is missing?' \
+  unless defined?(RSPEC_CONFIGURER)
+
+RSPEC_CONFIGURER.configure do |config|
+  config.include Rimless::RSpec::Helpers
+  config.include Rimless::RSpec::Matchers
+
+  # Stub all Confluent Schema Registry requests and handle them gracefully with
+  # the help of the faked (inlined) Schema Registry server. This allows us to
+  # perform the actual Apache Avro message encoding/decoding without the need
+  # to have a Schema Registry up and running.
+  config.before do
+    # Get the Excon connection from the AvroTurf instance
+    connection = Rimless.avro.instance_variable_get(:@registry)
+                        .instance_variable_get(:@upstream)
+                        .instance_variable_get(:@connection)
+                        .instance_variable_get(:@data)
+    # Enable WebMock on the already instantiated
+    # Confluent Schema Registry Excon connection
+    connection[:mock] = true
+    # Grab all Confluent Schema Registry requests and send
+    # them to the faked (inlined) Schema Registry
+    stub_request(:any, %r{^http://#{connection[:hostname]}})
+      .to_rack(FakeConfluentSchemaRegistryServer)
+    # Clear any cached data
+    FakeConfluentSchemaRegistryServer.clear
+  end
+end

data/lib/rimless/rspec/helpers.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Rimless
+  # Some general RSpec testing stuff.
+  module RSpec
+    # A collection of Rimless/RSpec helpers.
+    module Helpers
+      # A simple helper to parse a blob of Apache Avro data.
+      #
+      # @param data [String] the Apache Avro blob
+      # @return [Hash{String => Mixed}] the parsed payload
+      def avro_parse(data)
+        Rimless.avro.decode(data)
+      end
+    end
+  end
+end

data/lib/rimless/rspec/matchers.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+module Rimless
+  # Some general RSpec testing stuff.
+  module RSpec
+    # A set of Rimless/RSpec matchers.
+    module Matchers
+      # The Apache Kafka message expectation.
+      #
+      # rubocop:disable Metrics/ClassLength because its almost RSpec API code
+      class HaveSentKafkaMessage < ::RSpec::Matchers::BuiltIn::BaseMatcher
+        include ::RSpec::Mocks::ExampleMethods
+
+        # Instantiate a new expectation object.
+        #
+        # @param schema [String, Symbol, nil] the expected message schema
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def initialize(schema)
+          @schema = schema
+          @args = {}
+          @data = {}
+          @messages = []
+          set_expected_number(:exactly, 1)
+        end
+
+        # Collect the expectation arguments for the Kafka message passing. (eg.
+        # topic)
+        #
+        # @param args [Hash{Symbol => Mixed}] the expected arguments
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def with(**args)
+          @args = args
+          self
+        end
+
+        # Collect the expectations for the encoded message. The passed message
+        # will be decoded accordingly for the check.
+        #
+        # @param args [Hash{Symbol => Mixed}] the expected arguments
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def with_data(**args)
+          @data = args
+          self
+        end
+
+        # Set the expected amount of message (exactly).
+        #
+        # @param count [Integer] the expected amount
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def exactly(count)
+          set_expected_number(:exactly, count)
+          self
+        end
+
+        # Set the expected amount of message (at least).
+        #
+        # @param count [Integer] the expected amount
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def at_least(count)
+          set_expected_number(:at_least, count)
+          self
+        end
+
+        # Set the expected amount of message (at most).
+        #
+        # @param count [Integer] the expected amount
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def at_most(count)
+          set_expected_number(:at_most, count)
+          self
+        end
+
+        # Just syntactic sugar.
+        #
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def times
+          self
+        end
+
+        # Just syntactic sugar for a regular +exactly(:once)+ call.
+        #
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def once
+          exactly(:once)
+        end
+
+        # Just syntactic sugar for a regular +exactly(:twice)+ call.
+        #
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def twice
+          exactly(:twice)
+        end
+
+        # Just syntactic sugar for a regular +exactly(:thrice)+ call.
+        #
+        # @return [HaveSentKafkaMessage] the expectation instance
+        def thrice
+          exactly(:thrice)
+        end
+
+        # Serve the RSpec matcher API and signalize we support block evaluation.
+        #
+        # @return [Boolean] the answer
+        def supports_block_expectations?
+          true
+        end
+
+        # The actual RSpec API check for the expectation.
+        #
+        # @param proc [Proc] the block to evaluate
+        # @return [Boolean] whenever the check was successful or not
+        def matches?(proc)
+          unless proc.is_a? Proc
+            raise ArgumentError, 'have_sent_kafka_message and ' \
+                                 'sent_kafka_message only support block ' \
+                                 'expectations'
+          end
+
+          listen_to_messages
+          proc.call
+          check
+        end
+
+        # The actual RSpec API check for the expectation (negative).
+        #
+        # @param proc [Proc] the block to evaluate
+        # @return [Boolean] whenever the check was unsuccessful or not
+        def does_not_match?(proc)
+          set_expected_number(:at_least, 1)
+
+          !matches?(proc)
+        end
+
+        private
+
+        # Set the expectation type and count for the checking.
+        #
+        # @param relativity [Symbol] the amount expectation type
+        # @param count [Integer] the expected amount
+        def set_expected_number(relativity, count)
+          @expectation_type = relativity
+          @expected_number = case count
+                             when :once then 1
+                             when :twice then 2
+                             when :thrice then 3
+                             else Integer(count)
+                             end
+        end
+
+        # Perform the result set checking of recorded message which were sent.
+        #
+        # @return [Boolean] the answer
+        def check
+          @matching, @unmatching = @messages.partition do |message|
+            schema_match?(message) && arguments_match?(message) &&
+              data_match?(message)
+          end
+
+          @matching_count = @matching.size
+
+          case @expectation_type
+          when :exactly then @expected_number == @matching_count
+          when :at_most then @expected_number >= @matching_count
+          when :at_least then @expected_number <= @matching_count
+          end
+        end
+
+        # Check for the expected schema on the given message.
+        #
+        # @param message [Hash{Symbol => Mixed}] the message under inspection
+        # @return [Boolean] the check result
+        def schema_match?(message)
+          return true unless @schema
+
+          begin
+            Rimless.avro.decode(message[:data], schema_name: @schema.to_s)
+            return true
+          rescue Avro::IO::SchemaMatchException
+            false
+          end
+        end
+
+        # Check for the expected arguments on the Kafka message producer call.
+        #
+        # @param message [Hash{Symbol => Mixed}] the message under inspection
+        # @return [Boolean] the check result
+        def arguments_match?(message)
+          return true unless @args.any?
+
+          ::RSpec::Mocks::ArgumentListMatcher.new(*@args)
+                                             .args_match?(*message[:args])
+        end
+
+        # Check for the expected data on the encoded Apache Avro message.
+        # (deep include)
+        #
+        # @param message [Hash{Symbol => Mixed}] the message under inspection
+        # @return [Boolean] the check result
+        def data_match?(message)
+          return true unless @data.any?
+
+          actual_data = Rimless.avro.decode(message[:data])
+          expected_data = @data.deep_stringify_keys
+
+          actual_data.merge(expected_data) == actual_data
+        end
+
+        # Setup the +WaterDrop+ spies and record each sent message.
+        def listen_to_messages
+          allow(WaterDrop::SyncProducer).to receive(:call) do |data, **args|
+            @messages << { data: data, args: args, type: :sync }
+            nil
+          end
+
+          allow(WaterDrop::AsyncProducer).to receive(:call) do |data, **args|
+            @messages << { data: data, args: args, type: :async }
+            nil
+          end
+        end
+
+        # Serve the RSpec API and return the positive failure message.
+        #
+        # @return [String] the message to display
+        def failure_message
+          result = ["expected to send #{base_message}"]
+
+          if @unmatching.any?
+            result << "\nSent messages:"
+            @unmatching.each do |message|
+              result << "\n  #{base_message_detail(message)}"
+            end
+          end
+
+          result.join
+        end
+
+        # Serve the RSpec API and return the negative failure message.
+        #
+        # @return [String] the message to display
+        def failure_message_when_negated
+          "expected not to send #{base_message}"
+        end
+
+        # The base error message with all the expectation details included.
+        #
+        # @return [String] the expectation details message
+        def base_message
+          expectation_mod = @expectation_type.to_s.humanize.downcase
+          result = ["#{expectation_mod} #{@expected_number} messages,"]
+
+          result << " with schema #{@schema}," if @schema
+          result << " with #{@args}," if @args.any?
+          result << " with data #{@data}," if @data.any?
+          result << " but sent #{@matching_count}"
+
+          result.join
+        end
+
+        # The expectation details of a single message when unmatching messages
+        # were found.
+        #
+        # @return [String] the expectation details of a single message
+        def base_message_detail(message)
+          result = ['message']
+
+          result << " with #{message[:args]}" if message[:args].any?
+          result << " with data: #{Rimless.avro.decode(message[:data])}"
+
+          result.join
+        end
+      end
+      # rubocop:enable Metrics/ClassLength
+
+      # Check for messages which were sent to Apache Kafka by the given block.
+      #
+      # @param schema [String, Symbol, nil] the Apache Avro schema to check
+      #
+      # rubocop:disable Naming/PredicateName because its a RSpec matcher
+      def have_sent_kafka_message(schema = nil)
+        HaveSentKafkaMessage.new(schema)
+      end
+      alias sent_kafka_message have_sent_kafka_message
+      # rubocop:enable Naming/PredicateName
+    end
+  end
+end