rimless 0.3.0 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d97aaf1dd758a38f0537d4d2419052241412cba8f7ff45c30d40d5a5254a6e3
-  data.tar.gz: 483c104c176c020d79bec4347789be3d6558f21777571a36ffe5956813a75dff
+  metadata.gz: aa04c0230d9defc00a64eaebabbad4d06dd22c6432316d5fd7baab0e64527cdd
+  data.tar.gz: 311d8c3b112b6f628d66f6d59eb3b8f9298bd56b5c8f7d973e456668dab34dbc
 SHA512:
-  metadata.gz: deb7b05ed932b8dee372ecf9812cff38bd851e8d82cad80d31615278d182c0d4fdc258dc1e974f55d3330a2696ac803faf95156c54f7c5c8948eecd9ef15c50a
-  data.tar.gz: 8c3191a6e4e15240a72d36511143b775dce8602c375727defca0c2e11d6c56b6df7471a3ba2d70f95a5dc23e2eefbddfe74414c651904e7fd1fcf5b9dc2d9baa
+  metadata.gz: 9a93590a36bdb88097de37ce3d1dc0cd93d8f763a860f7cdac5d796764ecf8a74aebdc1db5a6e0642cde7cc108796226f9fd0e7ba02be1d768b30632829eec26
+  data.tar.gz: d8893ee4ce93e8682d55bec99645f401953b3fe92daeb5a7d607e3ca76ce1f295b7af469029ec5818a0c71987fcd32b151563e1bbc389a29ef4d9c3cda65ee97

data/.rubocop.yml

@@ -8,12 +8,12 @@ Documentation:
 
 AllCops:
   DisplayCopNames: true
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.5
   Exclude:
     - bin/**/*
     - vendor/**/*
     - build/**/*
-    - gemfiles/vendor/**/*
+    - gemfiles/**/*
 
 Metrics/BlockLength:
   Exclude:

data/.travis.yml

@@ -6,18 +6,26 @@ sudo: false
 language: ruby
 cache: bundler
 rvm:
+  - 2.7
   - 2.6
   - 2.5
-  - 2.4
-  - 2.3
 
 gemfile:
   - gemfiles/rails_4.2.gemfile
   - gemfiles/rails_5.0.gemfile
   - gemfiles/rails_5.1.gemfile
   - gemfiles/rails_5.2.gemfile
+  - gemfiles/rails_6.0.gemfile
 
 before_install: gem install bundler
+install:
+  # Rails 4 is not Ruby 2.7 compatible, so we skip this build
+  - |
+    [[ "${BUNDLE_GEMFILE}" =~ rails_4 && \
+      "${TRAVIS_RUBY_VERSION}" =~ 2.7 ]] && exit || true
+  # Regular build
+  - bundle install --jobs=3 --retry=3
+
 before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter

data/Appraisals

@@ -19,3 +19,8 @@ appraise 'rails-5.2' do
   gem 'activesupport', '~> 5.2.2'
   gem 'railties', '~> 5.2.2'
 end
+
+appraise 'rails-6.0' do
+  gem 'activesupport', '~> 6.0.0'
+  gem 'railties', '~> 6.0.0'
+end

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+### 1.0.4
+
+* Mocked WaterDrop producers in the rimless rspec helper so that tests
+  won't actually talk to Kafka (#9)
+
+### 1.0.3
+
+* Corrected broken stats when no consumer is yet defined (#8)
+
+### 1.0.2
+
+* Only load the statistics rake task when Rails is available and the
+  environment is development (instead of not production, this may cause issues
+  for +canary+ or +stage+ Rails environments)
+
+### 1.0.1
+
+* Added the missing +Karafka::Testing::RSpec::Helpers+ include to the
+  RSpec configuration
+
+### 1.0.0
+
+* Dropped support for Ruby 2.3/2.4 and added support for Rails 6.0 (#6)
+* Implemented a simple opinionated Kafka consumer setup (#7)
+
 ### 0.3.0
 
 * Upgraded the avro_turf gem (`~> 0.11.0`) (#5)

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM hausgold/ruby:2.3
+FROM hausgold/ruby:2.5
 MAINTAINER Hermann Mayer <hermann.mayer@hausgold.de>
 
 # Update system gem

data/Makefile

@@ -26,12 +26,13 @@ RM ?= rm
 XARGS ?= xargs
 
 # Container binaries
-BUNDLE ?= bundle
 APPRAISAL ?= appraisal
+BUNDLE ?= bundle
+GEM ?= gem
 RAKE ?= rake
-YARD ?= yard
 RAKE ?= rake
 RUBOCOP ?= rubocop
+YARD ?= yard
 
 # Files
 GEMFILES ?= $(subst _,-,$(patsubst $(GEMFILES_DIR)/%.gemfile,%,\
@@ -75,6 +76,8 @@ install:
 	# Install the dependencies
 	@$(MKDIR) -p $(VENDOR_DIR)
 	@$(call run-shell,$(BUNDLE) check || $(BUNDLE) install --path $(VENDOR_DIR))
+	@$(call run-shell,GEM_HOME=vendor/bundle/ruby/$${RUBY_MAJOR}.0 \
+		$(GEM) install bundler -v "~> 1.0")
 	@$(call run-shell,$(BUNDLE) exec $(APPRAISAL) install)
 
 update: install

data/README.md

@@ -23,6 +23,11 @@ opinionated framework which sets up solid conventions for producing messages.
     - [Confluent Schema Registry Subject](#confluent-schema-registry-subject)
   - [Organize and write schema definitions](#organize-and-write-schema-definitions)
   - [Producing messages](#producing-messages)
+  - [Consuming messages](#consuming-messages)
+    - [Routing messages to consumers](#routing-messages-to-consumers)
+    - [Consuming event messages](#consuming-event-messages)
+    - [Listing consumer routes](#listing-consumer-routes)
+    - [Starting the consumer process(es)](#starting-the-consumer-processes)
   - [Encoding/Decoding messages](#encodingdecoding-messages)
     - [Handling of schemaless deep blobs](#handling-of-schemaless-deep-blobs)
   - [Writing tests for your messages](#writing-tests-for-your-messages)
@@ -80,6 +85,9 @@ Rimless.configure do |conf|
   # The Confluent Schema Registry API URL,
   # set to HAUSGOLD defaults when not set
   conf.schema_registry_url = 'http://your.schema-registry.local'
+
+  # The Sidekiq job queue to use for consuming jobs
+  config.consumer_job_queue = 'default'
 end
 ```
 
@@ -98,6 +106,7 @@ available configuration options:
 * **KAFKA_CLIENT_ID**: The Apache Kafka client identifier, falls back the the local application name.
 * **KAFKA_BROKERS**: A comma separated list of Apache Kafka brokers for cluster discovery (Plaintext, no-auth/no-SSL only for now) (eg. `kafka://your.domain:9092,kafka..`)
 * **KAFKA_SCHEMA_REGISTRY_URL**: The Confluent Schema Registry API URL to use for schema registrations.
+* **KAFKA_SIDEKIQ_JOB_QUEUE**: The Sidekiq job queue to use for consuming jobs. Falls back to `default`.
 
 ### Conventions
 
@@ -279,6 +288,138 @@ Rimless.message(data: user, schema: :user_v1,
 Rimless.async_raw_message(data: encoded, topic: :users)
 ```
 
+### Consuming messages
+
+The rimless gem makes it super easy to build consumer logic right into your
+(Rails, standalone) application by utilizing the [Karafka
+framework](https://github.com/karafka/karafka) under the hood. When you have
+the rimless gem already installed you are ready to rumble to setup your
+application to consume Apache Kafka messages. Just run the `$ rake
+rimless:install` and all the consuming setup is done for you.
+
+Afterwards you find the `karafka.rb` file at the root of your project together
+with an example consumer (including specs). The default configuration follows
+the base conventions and ships some opinions on the architecture. The
+architecture looks like this:
+
+```
+              +----[Apache Kafka]
+              |
+     fetch message batches
+              |
+              v
+  +-----------------------------+
+  | Karafka/Rimless Consumer    |    +--------------------------------------+
+  |   Shares a single consumer  |--->| Sidekiq                              |
+  |   group, multiple processes |    |   Runs the consumer class (children  |
+  +-----------------------------+    |   of Rimless::BaseConsumer) for each |
+                                     |   message (Rimless::ConsumerJob),    |
+                                     |   one message per job                |
+                                     +--------------------------------------+
+```
+
+This architecture allows the consumer process to run mostly non-blocking and
+the messages can be processed concurrently via Sidekiq. (including the error
+handling and retrying)
+
+#### Routing messages to consumers
+
+The `karafka.rb` file at the root of your project is dedicated to configure the
+consumer process, including the routing table. The routing is as easy as it
+gets by following this pattern: `topic => consumer`. Here comes a the full
+examples:
+
+```ruby
+# Setup the topic-consumer routing table and boot the consumer application
+Rimless.consumer.topics(
+  { app: :your_app, name: :your_topic } => CustomConsumer
+).boot!
+```
+
+The key side of the hash is anything which is understood by the `Rimless.topic`
+method. With one addition: you can change `:name` to `:names` and pass an array
+of strings or symbols to listen to multiple application topics with a single
+configuration line.
+
+```ruby
+Rimless.consumer.topics(
+  { app: :your_app, names: %i[a b c] } => CustomConsumer
+).boot!
+
+# is identical to:
+
+Rimless.consumer.topics(
+  { app: :your_app, name: :a } => CustomConsumer,
+  { app: :your_app, name: :b } => CustomConsumer,
+  { app: :your_app, name: :c } => CustomConsumer
+).boot!
+```
+
+#### Consuming event messages
+
+By convention it makes sense to produce messages with various event types on a
+single Apache Kafka topic. This is fine, they just must follow a single
+constrain: each message must contain an `event`-named field at the Apache Avro
+schema with a dedicated name. This allow to structure data at Kafka like this:
+
+```
+Topic: production.users-api.users
+Events: user_created, user_updated, user_deleted
+```
+
+While respecting this convention your consumer classes will be super clean. See
+the following example: (we keep the users api example)
+
+```ruby
+class UserApiConsumer < ApplicationConsumer
+  def user_created(schema_field1:, optional_schema_field2: nil)
+    # Do whatever you need when a user was created
+  end
+end
+```
+
+Just name a method like the name of the event and specify all Apache Avro
+schema fields of it, except the event field. The messages will be automatically
+decoded with the help of the schema registry. All hashes/arrays ship deeply
+symbolized keys for easy access.
+
+**Heads up!** All messages with events which are not reflected by a method will
+just be ignored.
+
+See the automatically generated spec (`spec/consumers/custom_consumer_spec.rb`)
+for an example on how to test this.
+
+#### Listing consumer routes
+
+The rimless gem ships a simple tool to view all your consumer routes and the
+event messages it reacts on. Just run:
+
+```shell
+# Print all Apache Kafka consumer routes
+$ rake rimless:routes
+
+#    Topic: users-api.users
+# Consumer: UserApiConsumer
+#   Events: user_created
+```
+
+#### Starting the consumer process(es)
+
+From system integration perspective you just need to start the consumer
+processes and Sidekiq to get the thing going. Rimless allows you to start the
+consumer with `$ rake rimless:consumer` or you can just use the [Karafka
+binary](https://github.com/karafka/karafka/wiki/Fetching-messages) to start the
+consumer (`$ bundle exec karafka server`). Both work identically.
+
+When running inside a Rails application the consumer application initialization
+is automatically done for Sidekiq. Otherwise you need to initialize the
+consumer application manually with:
+
+```ruby
+# Manual consumer application initialization
+Sidekiq.configure_server { Rimless.consumer.initialize! }
+```
+
 ### Encoding/Decoding messages
 
 By convention we focus on the [Apache Avro](https://avro.apache.org/) data

data/gemfiles/rails_4.2.gemfile

@@ -1,10 +1,8 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activesupport', '~> 4.2.11'
-gem 'railties', '~> 4.2.11'
+gem "activesupport", "~> 4.2.11"
+gem "railties", "~> 4.2.11"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5.0.gemfile

@@ -1,10 +1,8 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activesupport', '~> 5.0.7'
-gem 'railties', '~> 5.0.7'
+gem "activesupport", "~> 5.0.7"
+gem "railties", "~> 5.0.7"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5.1.gemfile

@@ -1,10 +1,8 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activesupport', '~> 5.1.6'
-gem 'railties', '~> 5.1.6'
+gem "activesupport", "~> 5.1.6"
+gem "railties", "~> 5.1.6"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -1,10 +1,8 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'activesupport', '~> 5.2.2'
-gem 'railties', '~> 5.2.2'
+gem "activesupport", "~> 5.2.2"
+gem "railties", "~> 5.2.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_6.0.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 6.0.0"
+gem "railties", "~> 6.0.0"
+
+gemspec path: "../"

data/lib/rimless.rb

@@ -11,6 +11,8 @@ require 'active_support/core_ext/hash'
 require 'active_support/core_ext/string'
 require 'waterdrop'
 require 'avro_turf/messaging'
+require 'karafka'
+require 'karafka-sidekiq-backend'
 require 'sparsify'
 require 'erb'
 require 'pp'
@@ -24,6 +26,16 @@ module Rimless
   autoload :AvroUtils, 'rimless/avro_utils'
   autoload :KafkaHelpers, 'rimless/kafka_helpers'
   autoload :Dependencies, 'rimless/dependencies'
+  autoload :BaseConsumer, 'rimless/base_consumer'
+  autoload :Consumer, 'rimless/consumer'
+  autoload :ConsumerJob, 'rimless/consumer_job'
+
+  # All Karafka-framework related components
+  module Karafka
+    autoload :Base64Interchanger, 'rimless/karafka/base64_interchanger'
+    autoload :PassthroughMapper, 'rimless/karafka/passthrough_mapper'
+    autoload :AvroDeserializer, 'rimless/karafka/avro_deserializer'
+  end
 
   # Load standalone code
   require 'rimless/version'
@@ -34,4 +46,5 @@ module Rimless
   include Rimless::AvroHelpers
   include Rimless::KafkaHelpers
   include Rimless::Dependencies
+  include Rimless::Consumer
 end

data/lib/rimless/base_consumer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Rimless
+  # The base consumer where all Apache Kafka messages will be processed. It
+  # comes with some simple conventions to keep the actual application code
+  # simple to use.
+  class BaseConsumer < ::Karafka::BaseConsumer
+    # A generic message consuming handler which resolves the message event name
+    # to an actual method. All message data (top-level keys) is passed down to
+    # the event method as symbol arguments.
+    def consume
+      # We ignore events we do not handle by definition
+      send(event, **arguments) if respond_to? event
+    end
+
+    # Prepare the message payload as event method arguments.
+    #
+    # @return [Hash{Symbol => Mixed}] the event method arguments
+    def arguments
+      params.payload.except(:event)
+    end
+
+    # A shortcut to fetch the event name from the Kafka message.
+    #
+    # @return [Symbol] the event name of the current message
+    def event
+      params.payload[:event].to_sym
+    end
+  end
+end

data/lib/rimless/configuration.rb

@@ -53,5 +53,21 @@ module Rimless
       ENV.fetch('KAFKA_SCHEMA_REGISTRY_URL',
                 'http://schema-registry.message-bus.local')
     end
+
+    # The Sidekiq job queue to use for consuming jobs
+    config_accessor(:consumer_job_queue) do
+      ENV.fetch('KAFKA_SIDEKIQ_JOB_QUEUE', 'default').to_sym
+    end
+
+    # A custom writer for the consumer job queue name.
+    #
+    # @param val [String, Symbol] the new job queue name
+    def consumer_job_queue=(val)
+      config.consumer_job_queue = val.to_sym
+      # Refresh the consumer job queue
+      Rimless::ConsumerJob.sidekiq_options(
+        queue: Rimless.configuration.consumer_job_queue
+      )
+    end
   end
 end

data/lib/rimless/consumer.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Rimless
+  # The global rimless Apache Kafka consumer application based on
+  # the Karafka framework.
+  #
+  # rubocop:disable Style/ClassVars because we just work as a singleton
+  class ConsumerApp < ::Karafka::App
+    # We track our own initialization with this class variable
+    @@rimless_initialized = false
+
+    class << self
+      # Initialize the Karafka framework and our global consumer application
+      # with all our conventions/opinions.
+      #
+      # @return [Rimless::ConsumerApp] our self for chaining
+      def initialize!
+        # When already initialized, skip it
+        return self if @@rimless_initialized
+
+        # Initialize all the parts one by one
+        initialize_rails!
+        initialize_monitors!
+        initialize_karafka!
+        initialize_logger!
+        initialize_code_reload!
+
+        # Load the custom Karafka boot file when it exists, it contains
+        # custom configurations and the topic/consumer routing table
+        require ::Karafka.boot_file if ::Karafka.boot_file.exist?
+
+        # Set our custom initialization process as completed to
+        # skip subsequent calls
+        @@rimless_initialized = true
+        self
+      end
+
+      # Check if Rails is available and not already initialized, then
+      # initialize it.
+      def initialize_rails!
+        rails_env = ::Karafka.root.join('config', 'environment.rb')
+
+        # Stop, when Rails is already initialized
+        return if defined? Rails
+
+        # Stop, when there is no Rails at all
+        return unless rails_env.exist?
+
+        ENV['RAILS_ENV'] ||= 'development'
+        ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
+        require rails_env
+        Rails.application.eager_load!
+      end
+
+      # We like to listen to instrumentation and logging events to allow our
+      # users to handle them like they need.
+      def initialize_monitors!
+        [
+          WaterDrop::Instrumentation::StdoutListener.new,
+          ::Karafka::Instrumentation::StdoutListener.new,
+          ::Karafka::Instrumentation::ProctitleListener.new
+        ].each do |listener|
+          ::Karafka.monitor.subscribe(listener)
+        end
+      end
+
+      # Configure the pure basics on the Karafka application.
+      #
+      # rubocop:disable Metrics/MethodLength because of the various settings
+      def initialize_karafka!
+        setup do |config|
+          mapper = Rimless::Karafka::PassthroughMapper.new
+          config.consumer_mapper = config.topic_mapper = mapper
+          config.deserializer = Rimless::Karafka::AvroDeserializer.new
+          config.kafka.seed_brokers = Rimless.configuration.kafka_brokers
+          config.client_id = Rimless.configuration.client_id
+          config.logger = Rimless.logger
+          config.backend = :sidekiq
+          config.batch_fetching = true
+          config.batch_consuming = false
+          config.shutdown_timeout = 10
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # When we run in development mode, we want to write the logs
+      # to file and to stdout.
+      def initialize_logger!
+        return unless Rimless.env.development? && server?
+
+        STDOUT.sync = true
+        Rimless.logger.extend(ActiveSupport::Logger.broadcast(
+                                ActiveSupport::Logger.new($stdout)
+                              ))
+      end
+
+      # Perform code hot-reloading when we are in Rails and in development
+      # mode.
+      def initialize_code_reload!
+        return unless defined?(Rails) && Rails.env.development?
+
+        ::Karafka.monitor.subscribe(::Karafka::CodeReloader.new(
+                                      *Rails.application.reloaders
+                                    ))
+      end
+
+      # Allows the user to re-configure the Karafka application if this is
+      # needed. (eg. to set some ruby-kafka driver default settings, etc)
+      #
+      # @return [Rimless::ConsumerApp] our self for chaining
+      def configure(&block)
+        setup(&block)
+        self
+      end
+
+      # Configure the topics-consumer routing table in a lean way.
+      #
+      # Examples:
+      #
+      #   topics({ app: :test_app, name: :admins } => YourConsumer)
+      #   topics({ app: :test_app, names: %i[users admins] } => YourConsumer)
+      #
+      # @param topics [Hash{Hash => Class}] the topic to consumer mapping
+      #
+      # rubocop:disable Metrics/MethodLength because of the Karafka DSL
+      def topics(topics)
+        consumer_groups.draw do
+          consumer_group Rimless.configuration.client_id do
+            topics.each do |topic_parts, dest_consumer|
+              Rimless.consumer.topic_names(topic_parts).each do |topic_name|
+                topic(topic_name) do
+                  consumer dest_consumer
+                  worker Rimless::ConsumerJob
+                  interchanger Rimless::Karafka::Base64Interchanger
+                end
+              end
+            end
+          end
+        end
+
+        self
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # Build the conventional Apache Kafka topic names from the given parts.
+      # This allows various forms like single strings/symbols and a hash in the
+      # form of +{ app: [String, Symbol], name: [String, Symbol], names:
+      # [Array<String, Symbol>] }+. This allows the maximum of flexibility.
+      #
+      # @param parts [String, Symbol, Hash{Symbol => Mixed}] the topic
+      #   name parts
+      # @return [Array<String>] the full topic names
+      def topic_names(parts)
+        # We have a single app, but multiple names so we handle them
+        if parts.is_a?(Hash) && parts.key?(:names)
+          return parts[:names].map do |name|
+            Rimless.topic(parts.merge(name: name))
+          end
+        end
+
+        # We cannot handle the given input
+        [Rimless.topic(parts)]
+      end
+
+      # Check if we run as the Karafka server (consumer) process or not.
+      #
+      # @return [Boolean] whenever we run as the Karafka server or not
+      def server?
+        $PROGRAM_NAME.end_with?('karafka') && ARGV.include?('server')
+      end
+    end
+  end
+  # rubocop:enable Style/ClassVars
+
+  # A rimless top-level concern which adds lean access to
+  # the consumer application.
+  module Consumer
+    extend ActiveSupport::Concern
+
+    class_methods do
+      # A simple shortcut to fetch the Karafka consumer application.
+      #
+      # @return [Rimless::ConsumerApp] the Karafka consumer application class
+      def consumer
+        ConsumerApp.initialize!
+      end
+    end
+  end
+end

data/lib/rimless/consumer_job.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Rimless
+  # The base consumer job where each message is processed asynchronous via
+  # Sidekiq. We need to inherit the Karafka base worker class into a custom
+  # one, otherwise it fails.
+  class ConsumerJob < ::Karafka::BaseWorker
+    sidekiq_options queue: Rimless.configuration.consumer_job_queue
+  end
+end

data/lib/rimless/kafka_helpers.rb

@@ -35,7 +35,7 @@ module Rimless
 
         raise ArgumentError, 'No name given' if name.nil?
 
-        "#{Rimless.topic_prefix(app)}#{name}"
+        "#{Rimless.topic_prefix(app)}#{name}".tr('_', '-')
       end
       # rubocop:enable Metrics/AbcSize
 

data/lib/rimless/karafka/avro_deserializer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Karafka
+    # A custom Apache Avro compatible message deserializer.
+    class AvroDeserializer
+      # Deserialize an Apache Avro encoded Apache Kafka message.
+      #
+      # @param message [String] the binary blob to deserialize
+      # @return [Hash{Symbol => Mixed}] the deserialized Apache Avro message
+      def call(message)
+        # We use sparsed hashes inside of Apache Avro messages for schema-less
+        # blobs of data, such as loosely structured metadata blobs.  Thats a
+        # somewhat bad idea on strictly typed and defined messages, but their
+        # occurence should be rare.
+        Rimless
+          .decode(message.payload)
+          .yield_self { |data| Sparsify(data, sparse_array: true) }
+          .yield_self { |data| data.transform_keys { |key| key.delete('\\') } }
+          .yield_self { |data| Unsparsify(data, sparse_array: true) }
+          .deep_symbolize_keys
+      end
+    end
+  end
+end

data/lib/rimless/karafka/base64_interchanger.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Karafka
+    # Allow the +karafka-sidekiq-backend+ gem to transfer binary Apache Kafka
+    # messages to the actual Sidekiq job.
+    #
+    # rubocop:disable Security/MarshalLoad because we encode/decode the
+    #   messages in our own controlled context
+    class Base64Interchanger
+      # Encode a binary Apache Kafka message(s) so they can be passed to the
+      # Sidekiq +Rimless::ConsumerJob+.
+      #
+      # @param params_batch [Mixed] the raw message(s) to encode
+      # @return [String] the marshaled+base64 encoded data
+      def self.encode(params_batch)
+        Base64.encode64(Marshal.dump(params_batch.to_a))
+      end
+
+      # Decode the binary Apache Kafka message(s) so they can be processed by
+      # the Sidekiq +Rimless::ConsumerJob+.
+      #
+      # @param params_string [String] the marshaled+base64 encoded data
+      # @return [Mixed] the unmarshaled+base64 decoded data
+      def self.decode(params_string)
+        Marshal.load(Base64.decode64(params_string))
+      end
+    end
+    # rubocop:enable Security/MarshalLoad
+  end
+end

data/lib/rimless/karafka/passthrough_mapper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Karafka
+    # The Karafka framework makes some assumptions about the consumer group and
+    # topic names. We have our own opinions/conventions, so we just pass them
+    # through unmodified.
+    class PassthroughMapper
+      # We do not want to modify the given consumer group name, so we
+      # pass it through.
+      #
+      # @param raw_consumer_group_name [String, Symbol] the original
+      #   consumer group name
+      # @return [String, Symbol] the original consumer group name
+      def call(raw_consumer_group_name)
+        raw_consumer_group_name
+      end
+
+      # We do not want to modify the given topic name, so we pass it through.
+      #
+      # @param topic [String, Symbol] the original topic name
+      # @return [String, Symbol] the original topic name
+      def incoming(topic)
+        topic
+      end
+      alias outgoing incoming
+    end
+  end
+end

data/lib/rimless/railtie.rb

@@ -20,6 +20,18 @@ module Rimless
     config.after_initialize do
       # Reconfigure our dependencies
       Rimless.configure_dependencies
+
+      # Load the Karafka application inside the Sidekiq server application
+      if defined? Sidekiq
+        Sidekiq.configure_server do
+          Rimless.consumer.initialize!
+        end
+      end
+    end
+
+    # Load all our Rake tasks if we're supposed to do
+    rake_tasks do
+      Dir[File.join(__dir__, 'tasks', '*.rake')].each { |file| load file }
     end
   end
 end

data/lib/rimless/rspec.rb

@@ -6,6 +6,7 @@ require 'avro_turf/test/fake_confluent_schema_registry_server'
 require 'rimless'
 require 'rimless/rspec/helpers'
 require 'rimless/rspec/matchers'
+require 'karafka/testing/rspec/helpers'
 
 # RSpec 1.x and 2.x compatibility
 #
@@ -13,9 +14,17 @@ require 'rimless/rspec/matchers'
 raise 'No RSPEC_CONFIGURER is defined, webmock is missing?' \
   unless defined?(RSPEC_CONFIGURER)
 
+# rubocop:disable Metrics/BlockLength because we have to configure
+#   RSpec properly
 RSPEC_CONFIGURER.configure do |config|
   config.include Rimless::RSpec::Helpers
   config.include Rimless::RSpec::Matchers
+  config.include Karafka::Testing::RSpec::Helpers
+
+  # Set the custom +consumer+ type for consumer spec files
+  config.define_derived_metadata(file_path: %r{/spec/consumers/}) do |meta|
+    meta[:type] = :consumer
+  end
 
   # Take care of the initial test configuration.
   config.before(:suite) do
@@ -37,7 +46,7 @@ RSPEC_CONFIGURER.configure do |config|
   # the help of the faked (inlined) Schema Registry server. This allows us to
   # perform the actual Apache Avro message encoding/decoding without the need
   # to have a Schema Registry up and running.
-  config.before(:each) do
+  config.before(:each) do |example|
     # Get the Excon connection from the AvroTurf instance
     connection = Rimless.avro.instance_variable_get(:@registry)
                         .instance_variable_get(:@upstream)
@@ -53,7 +62,16 @@ RSPEC_CONFIGURER.configure do |config|
     # Clear any cached data
     FakeConfluentSchemaRegistryServer.clear
 
+    # Do not interact with Apache Kafka itself on tests
+    allow(WaterDrop::AsyncProducer).to receive(:call)
+    allow(WaterDrop::SyncProducer).to receive(:call)
+
     # Reconfigure the Rimless AvroTurf instance
     Rimless.configure_avro_turf
+
+    # When the example type is a Kafka consumer, we must initialize
+    # the Karafka framework first.
+    Rimless.consumer.initialize! if example.metadata[:type] == :consumer
   end
 end
+# rubocop:enable Metrics/BlockLength

data/lib/rimless/rspec/helpers.rb

@@ -13,6 +13,31 @@ module Rimless
       def avro_parse(data, **opts)
         Rimless.avro_decode(data, **opts)
       end
+
+      # A simple helper to fake a deserialized Apache Kafka message for
+      # consuming.
+      #
+      # @param payload [Hash{Symbol => Mixed}] the message payload
+      # @param topic [String, Hash{Symbol => Mixed}] the actual message
+      #   topic (full as string, or parts via hash)
+      # @return [OpenStruct] the fake deserialized Kafka message
+      #
+      # rubocop:disable Metrics/MethodLength because of the various properties
+      def kafka_message(topic: nil, headers: {}, **payload)
+        OpenStruct.new(
+          topic: Rimless.topic(topic),
+          headers: headers,
+          payload: payload,
+          is_control_record: false,
+          key: nil,
+          offset: 206,
+          partition: 0,
+          create_time: Time.current,
+          receive_time: Time.current,
+          deserialized: true
+        )
+      end
+      # rubocop:enable Metrics/MethodLength
     end
   end
 end

data/lib/rimless/tasks/consumer.rake

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+namespace :rimless do
+  desc 'Start the Apache Kafka consumer'
+  task :consumer do
+    system 'bundle exec karafka server'
+  end
+
+  desc 'Print all the consumer routes'
+  task routes: :environment do
+    require 'rimless'
+
+    Rimless.consumer.consumer_groups.each do |consumer_group|
+      consumer_group.topics.each do |topic|
+        name = topic.name.split('.')[1..-1].join('.')
+
+        puts "#    Topic: #{name}"
+        puts "# Consumer: #{topic.consumer}"
+
+        base = topic.consumer.superclass.new(topic).methods
+        events = topic.consumer.new(topic).methods - base
+
+        puts "#   Events: #{events.join(', ')}"
+        puts
+      end
+    end
+  end
+end

data/lib/rimless/tasks/generator.rake

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+namespace :rimless do
+  require 'fileutils'
+
+  # Install a template file to the project.
+  #
+  # @param src [String] the template source file name
+  # @param dest [Array<String>] the relative destination parts
+  def install_template(src, *dest)
+    src = File.join(__dir__, 'templates', src)
+    dest = File.join(Dir.pwd, *dest, File.basename(src))
+
+    return puts "#    [Skip] #{dest}" if File.exist? dest
+
+    puts "# [Install] #{dest}"
+    FileUtils.mkdir_p(File.dirname(dest))
+    FileUtils.copy(src, dest)
+  end
+
+  desc 'Install the Rimless consumer components'
+  task :install do
+    install_template('karafka.rb')
+    install_template('application_consumer.rb', 'app', 'consumers')
+    install_template('custom_consumer.rb', 'app', 'consumers')
+    install_template('custom_consumer_spec.rb', 'spec', 'consumers')
+
+    puts <<~OUTPUT
+      #
+      # Installation done.
+      #
+      # You can now configure your routes at the +karafka.rb+ file at
+      # your project root. And list all routes with +rake rimless:routes+.
+    OUTPUT
+  end
+end

data/lib/rimless/tasks/stats.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+if defined?(Rails) && Rails.env.development?
+  require 'rspec/core/rake_task'
+
+  task :stats do
+    require 'rails/code_statistics'
+
+    [
+      [:unshift, 'Consumer', 'app/consumers']
+    ].each do |method, type, dir|
+      next unless File.directory? dir
+
+      ::STATS_DIRECTORIES.send(method, [type, dir])
+      ::CodeStatistics::TEST_TYPES << type if type.include? 'specs'
+    end
+  end
+end

data/lib/rimless/tasks/templates/application_consumer.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# The overall shared base consumer for Apache Kafka messages. Just write your
+# own specific consumer and inherit this one to share logic.
+class ApplicationConsumer < Rimless::BaseConsumer
+end

data/lib/rimless/tasks/templates/custom_consumer.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# A dedicated consumer to handle event-messages from your producing application.
+# Just write a method with the name of an event and it is called directly with
+# all the event data as parameters.
+class CustomConsumer < ApplicationConsumer
+  # Handle +custom_event+ event messages.
+  def custom_event(property1:, property2: nil)
+    # Do whatever you need to do
+    [property1, property2]
+  end
+end

data/lib/rimless/tasks/templates/custom_consumer_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe CustomConsumer do
+  let(:topic) { Rimless.topic(app: :your_app, name: :your_topic) }
+  let(:instance) { karafka_consumer_for(topic) }
+  let(:action) { instance.consume }
+  let(:params) { kafka_message(topic: topic, **payload) }
+
+  before { allow(instance).to receive(:params).and_return(params) }
+
+  context 'with custom_event message' do
+    let(:payload) do
+      { event: 'custom_event', property1: 'test', property2: nil }
+    end
+
+    it 'returns the payload properties' do
+      expect(action).to be_eql(['test', nil])
+    end
+  end
+end

data/lib/rimless/tasks/templates/karafka.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'rimless'
+
+# Setup the topic-consumer routing table and boot the consumer application
+Rimless.consumer.topics(
+  { app: :your_app, name: :your_topic } => CustomConsumer
+).boot!
+
+# Configure Karafka/ruby-kafka settings
+# Rimless.consumer.configure do |config|
+#   # See https://github.com/karafka/karafka/wiki/Configuration
+#   # config.kafka.start_from_beginning = false
+# end
+
+# We want a less verbose logging on development
+# Rimless.logger.level = Logger::INFO if Rails.env.development?
+
+# Use a different Sidekiq queue for the consumer jobs
+# Rimless.configuration.consumer_job_queue = :messages

data/lib/rimless/version.rb

@@ -2,5 +2,5 @@
 
 module Rimless
   # The version of the +rimless+ gem
-  VERSION = '0.3.0'
+  VERSION = '1.0.4'
 end

data/rimless.gemspec

@@ -25,6 +25,9 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'activesupport', '>= 4.2.0'
   spec.add_runtime_dependency 'avro_turf', '~> 0.11.0'
+  spec.add_runtime_dependency 'karafka', '~> 1.3'
+  spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.3'
+  spec.add_runtime_dependency 'karafka-testing', '~> 1.3'
   spec.add_runtime_dependency 'sinatra'
   spec.add_runtime_dependency 'sparsify', '~> 1.1'
   spec.add_runtime_dependency 'waterdrop', '~> 1.2'
@@ -34,7 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '>= 1.16', '< 3'
   spec.add_development_dependency 'factory_bot', '~> 4.11'
   spec.add_development_dependency 'railties', '>= 4.2.0'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rdoc', '~> 6.1'
   spec.add_development_dependency 'redcarpet', '~> 3.4'
   spec.add_development_dependency 'rspec', '~> 3.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rimless
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.4
 platform: ruby
 authors:
 - Hermann Mayer
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-11-08 00:00:00.000000000 Z
+date: 2020-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.11.0
+- !ruby/object:Gem::Dependency
+  name: karafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: karafka-sidekiq-backend
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: karafka-testing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
@@ -162,14 +204,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -345,23 +387,38 @@ files:
 - gemfiles/rails_5.0.gemfile
 - gemfiles/rails_5.1.gemfile
 - gemfiles/rails_5.2.gemfile
+- gemfiles/rails_6.0.gemfile
 - lib/rimless.rb
 - lib/rimless/avro_helpers.rb
 - lib/rimless/avro_utils.rb
+- lib/rimless/base_consumer.rb
 - lib/rimless/configuration.rb
 - lib/rimless/configuration_handling.rb
+- lib/rimless/consumer.rb
+- lib/rimless/consumer_job.rb
 - lib/rimless/dependencies.rb
 - lib/rimless/kafka_helpers.rb
+- lib/rimless/karafka/avro_deserializer.rb
+- lib/rimless/karafka/base64_interchanger.rb
+- lib/rimless/karafka/passthrough_mapper.rb
 - lib/rimless/railtie.rb
 - lib/rimless/rspec.rb
 - lib/rimless/rspec/helpers.rb
 - lib/rimless/rspec/matchers.rb
+- lib/rimless/tasks/consumer.rake
+- lib/rimless/tasks/generator.rake
+- lib/rimless/tasks/stats.rake
+- lib/rimless/tasks/templates/application_consumer.rb
+- lib/rimless/tasks/templates/custom_consumer.rb
+- lib/rimless/tasks/templates/custom_consumer_spec.rb
+- lib/rimless/tasks/templates/karafka.rb
 - lib/rimless/version.rb
+- log/development.log
 - rimless.gemspec
 homepage: https://github.com/hausgold/rimless
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -376,8 +433,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.1.3
+signing_key:
 specification_version: 4
 summary: A bundle of opinionated Apache Kafka / Confluent Schema Registry helpers.
 test_files: []