rimless 1.0.3 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9cedf797bd0b5ec9de392d00177dd7332eb96815b4bcb65429165a0cf3d5d10
-  data.tar.gz: 2eaa2e476785ab6e360c0ad68990c9dd3c7bde5ad0043c37cfcd08fea2e655a9
+  metadata.gz: 5763fa79c04ca9d1db4e4c75d98bb76da2ff843acdd6d4be001a85bdfb37bfc2
+  data.tar.gz: ede22c5243eab5f54ded67dd3f7c5d9b5d8aafb02b0c0d63280341e5ca96fc29
 SHA512:
-  metadata.gz: e827a4aa9f672d84456d4fd78e5a7d2b3d10565d83cb519493bac7fd61062d2e465d71663b46c5a48f8b9ebf16a7be18d40590769f9d37ae0afb816e021a44aa
-  data.tar.gz: 5d1031d3b658925b03ddd5069ca942b45b68c3d4eb15cdab33a22377881cb33f9ef54775f6543027ea7b226745d0d11c7740014370ebb495b77ae5143f3e468e
+  metadata.gz: 5e9c10c5e6699c126aba2aafbbea3812bb429b2bbcc6337fc4b324a06ca2eaf3560fb51b59481763b37f6e8014ff66efee3be831049da3cf1179d3966a0029a5
+  data.tar.gz: 996be359cf6113ae58dc7847351b438987af6436510c7f8da5031eaf1e4bd957737c9f0feff8acab3b336b9c7b232348cf2f9b1978e9a911da2b0f46eebca5ce

data/.github/workflows/documentation.yml

@@ -0,0 +1,38 @@
+name: Build Documentation
+on:
+  repository_dispatch:
+    types: [documentation]
+
+concurrency:
+  group: 'docs'
+
+jobs:
+  docs:
+    name: Build gem documentation
+    runs-on: ubuntu-20.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install the correct Ruby version
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.5
+          bundler-cache: true
+
+      - name: Prepare the virtual environment
+        uses: hausgold/actions/ci@master
+        with:
+          clone_token: '${{ secrets.CLONE_TOKEN }}'
+          settings: '${{ github.repository }}'
+          target: ci/gem-test
+
+      - name: Build gem documentation
+        run: make docs
+
+      - name: Upload the code coverage report
+        run: coverage
+
+      - name: Add this job to the commit status
+        run: commit-status '${{ job.status }}'
+        if: always()

data/.github/workflows/test.yml

@@ -0,0 +1,66 @@
+name: Test
+on:
+  push:
+    branches:
+      - '**'
+  schedule:
+    - cron: '0 0 * * MON'
+
+concurrency:
+  group: '${{ github.ref }}'
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: 'Test the gem (Ruby ${{ matrix.ruby }}, Rails ${{ matrix.rails }})'
+    runs-on: ubuntu-20.04
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.5, 2.6, 2.7]
+        rails: [4.2, '5.0', 5.1, 5.2, '6.0']
+        exclude:
+          # Rails 4.2 is not compatible with Ruby 2.7
+          - ruby: 2.7
+            rails: 4.2
+    env:
+      BUNDLE_GEMFILE: 'gemfiles/rails_${{ matrix.rails }}.gemfile'
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install the correct Ruby version
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Prepare the virtual environment
+        uses: hausgold/actions/ci@master
+        with:
+          clone_token: '${{ secrets.CLONE_TOKEN }}'
+          settings: '${{ github.repository }}'
+          target: ci/gem-test
+
+      - name: Run the gem tests
+        run: make test
+
+      - name: Upload the code coverage report
+        run: coverage
+
+  trigger-docs:
+    name: Trigger the documentation upload
+    runs-on: ubuntu-20.04
+    timeout-minutes: 2
+    needs: test
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - name: Prepare the virtual environment
+        uses: hausgold/actions/ci@master
+        with:
+          clone_token: '${{ secrets.CLONE_TOKEN }}'
+          settings: '${{ github.repository }}'
+          target: ci/noop
+
+      - name: Trigger the documentation upload
+        run: workflow documentation

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+### 1.2.0
+
+* Added a `capture_kafka_messages` helper for RSpec (#12)
+
+### 1.1.1
+
+* Corrected the GNU Make release target
+
+### 1.1.0
+
+* Added support for Karafka `~> 1.4.0` and set is as minimum dependency version
+  (#10)
+
+### 1.0.4
+
+* Mocked WaterDrop producers in the rimless rspec helper so that tests
+  won't actually talk to Kafka (#9)
+
 ### 1.0.3
 
 * Corrected broken stats when no consumer is yet defined (#8)

data/Makefile

@@ -30,7 +30,6 @@ APPRAISAL ?= appraisal
 BUNDLE ?= bundle
 GEM ?= gem
 RAKE ?= rake
-RAKE ?= rake
 RUBOCOP ?= rubocop
 YARD ?= yard
 
@@ -80,14 +79,18 @@ install:
 		$(GEM) install bundler -v "~> 1.0")
 	@$(call run-shell,$(BUNDLE) exec $(APPRAISAL) install)
 
-update: install
+update:
 	# Install the dependencies
 	@$(MKDIR) -p $(VENDOR_DIR)
 	@$(call run-shell,$(BUNDLE) exec $(APPRAISAL) update)
 
-test: #install
+test: \
+	test-specs \
+	test-style
+
+test-specs:
 	# Run the whole test suite
-	@$(call run-shell,$(BUNDLE) exec $(RAKE))
+	@$(call run-shell,$(BUNDLE) exec $(RAKE) stats spec)
 
 $(TEST_GEMFILES): GEMFILE=$(@:test-%=%)
 $(TEST_GEMFILES):
@@ -126,27 +129,27 @@ endif
 
 distclean: clean clean-containers clean-images
 
-shell: install
+shell:
 	# Run an interactive shell on the container
 	@$(call run-shell,$(BASH) -i)
 
-shell-irb: install
+shell-irb:
 	# Run an interactive IRB shell on the container
 	@$(call run-shell,bin/console)
 
-docs: install
+docs:
 	# Build the API documentation
 	@$(call run-shell,$(BUNDLE) exec $(YARD) -q && \
 		$(BUNDLE) exec $(YARD) stats --list-undoc --compact)
 
-notes: install
+notes:
 	# Print the code statistics (library and test suite)
 	@$(call run-shell,$(BUNDLE) exec $(RAKE) notes)
 
-stats: install
+stats:
 	# Print all the notes from the code
 	@$(call run-shell,$(BUNDLE) exec $(RAKE) stats)
 
 release:
 	# Release a new gem version
-	@$(RAKE) release
+	@$(BUNDLE) exec $(RAKE) release

data/README.md

@@ -1,10 +1,10 @@
 ![rimless](doc/assets/project.svg)
 
-[![Build Status](https://travis-ci.com/hausgold/rimless.svg?token=4XcyqxxmkyBSSV3wWRt7&branch=master)](https://travis-ci.com/hausgold/rimless)
+[![Continuous Integration](https://github.com/hausgold/rimless/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/hausgold/rimless/actions/workflows/test.yml)
 [![Gem Version](https://badge.fury.io/rb/rimless.svg)](https://badge.fury.io/rb/rimless)
-[![Maintainability](https://api.codeclimate.com/v1/badges/0d51996b52def6cf0262/maintainability)](https://codeclimate.com/repos/5cb06f700f7b09026e00a896/maintainability)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/0d51996b52def6cf0262/test_coverage)](https://codeclimate.com/repos/5cb06f700f7b09026e00a896/test_coverage)
-[![API docs](https://img.shields.io/badge/docs-API-blue.svg)](https://www.rubydoc.info/gems/rimless)
+[![Test Coverage](https://automate-api.hausgold.de/v1/coverage_reports/rimless/coverage.svg)](https://knowledge.hausgold.de/coverage)
+[![Test Ratio](https://automate-api.hausgold.de/v1/coverage_reports/rimless/ratio.svg)](https://knowledge.hausgold.de/coverage)
+[![API docs](https://automate-api.hausgold.de/v1/coverage_reports/rimless/documentation.svg)](https://www.rubydoc.info/gems/rimless)
 
 This project is dedicated to ship a ready to use [Apache
 Kafka](https://kafka.apache.org/) / [Confluent Schema
@@ -575,6 +575,13 @@ describe 'message producer job' do
         .with(key: user.id, topic: 'test.identity-api.users').twice
         .with_data(firstname: 'John', lastname: 'Doe').twice
   end
+
+  it 'allows to capture messages to check them in detail' do
+    (capture_kafka_messages { action }).tap do |messages|
+      expect(messages.first[:data]).to \
+        match(a_hash_including('entity' => a_hash_including('items' => items)))
+    end
+  end
 end
 ```
 

data/Rakefile

@@ -54,12 +54,15 @@ end
 
 # Configure all code statistics directories
 vendors = [
+  [:unshift, 'Top-levels', 'lib', %r{lib(/rimless)?/[^/]+\.rb$}],
+  [:unshift, 'Top-levels specs', 'spec',
+   %r{spec/rimless(_spec\.rb|/[^/]+\.rb$)}],
+
   [:unshift, 'RSpec matchers', 'lib/rimless/rspec'],
   [:unshift, 'RSpec matchers specs', 'spec/rimless/rspec'],
 
-  [:unshift, 'Top-levels', 'lib', %r{lib(/rimless)?/[^/]+\.rb$}],
-  [:unshift, 'Top-levels specs', 'spec',
-   %r{spec/rimless(_spec\.rb|/[^/]+\.rb$)}]
+  [:unshift, 'Rake Tasks', 'lib/rimless/tasks'],
+  [:unshift, 'Karafka Extensions', 'lib/rimless/karafka']
 ].reverse
 
 vendors.each do |method, type, dir, pattern|

data/lib/rimless/consumer.rb

@@ -131,7 +131,7 @@ module Rimless
                 topic(topic_name) do
                   consumer dest_consumer
                   worker Rimless::ConsumerJob
-                  interchanger Rimless::Karafka::Base64Interchanger
+                  interchanger Rimless::Karafka::Base64Interchanger.new
                 end
               end
             end

data/lib/rimless/karafka/avro_deserializer.rb

@@ -6,15 +6,19 @@ module Rimless
     class AvroDeserializer
       # Deserialize an Apache Avro encoded Apache Kafka message.
       #
-      # @param message [String] the binary blob to deserialize
+      # @param params [Karafka::Params::Params] the Karafka message parameters
       # @return [Hash{Symbol => Mixed}] the deserialized Apache Avro message
-      def call(message)
+      def call(params)
+        # When the Kafka message does not have a payload, we won't fail.
+        # This is for Kafka users which use log compaction with a nil payload.
+        return if params.raw_payload.nil?
+
         # We use sparsed hashes inside of Apache Avro messages for schema-less
         # blobs of data, such as loosely structured metadata blobs.  Thats a
         # somewhat bad idea on strictly typed and defined messages, but their
         # occurence should be rare.
         Rimless
-          .decode(message.payload)
+          .decode(params.raw_payload)
           .yield_self { |data| Sparsify(data, sparse_array: true) }
           .yield_self { |data| data.transform_keys { |key| key.delete('\\') } }
           .yield_self { |data| Unsparsify(data, sparse_array: true) }

data/lib/rimless/karafka/base64_interchanger.rb

@@ -7,14 +7,14 @@ module Rimless
     #
     # rubocop:disable Security/MarshalLoad because we encode/decode the
     #   messages in our own controlled context
-    class Base64Interchanger
+    class Base64Interchanger < ::Karafka::Interchanger
       # Encode a binary Apache Kafka message(s) so they can be passed to the
       # Sidekiq +Rimless::ConsumerJob+.
       #
       # @param params_batch [Mixed] the raw message(s) to encode
       # @return [String] the marshaled+base64 encoded data
-      def self.encode(params_batch)
-        Base64.encode64(Marshal.dump(params_batch.to_a))
+      def encode(params_batch)
+        Base64.encode64(Marshal.dump(super))
       end
 
       # Decode the binary Apache Kafka message(s) so they can be processed by
@@ -22,8 +22,8 @@ module Rimless
       #
       # @param params_string [String] the marshaled+base64 encoded data
       # @return [Mixed] the unmarshaled+base64 decoded data
-      def self.decode(params_string)
-        Marshal.load(Base64.decode64(params_string))
+      def decode(params_string)
+        Marshal.load(Base64.decode64(super)).map(&:stringify_keys)
       end
     end
     # rubocop:enable Security/MarshalLoad

data/lib/rimless/rspec/helpers.rb

@@ -38,6 +38,14 @@ module Rimless
         )
       end
       # rubocop:enable Metrics/MethodLength
+
+      # Capture all Apache Kafka messages of the given block.
+      #
+      # @yield the given block to capture the messages
+      # @return [Array<Hash{Symbol => Mixed}>] the captured messages
+      def capture_kafka_messages(&block)
+        Rimless::RSpec::Matchers::HaveSentKafkaMessage.new(nil).capture(&block)
+      end
     end
   end
 end

data/lib/rimless/rspec/matchers.rb

@@ -23,6 +23,15 @@ module Rimless
           set_expected_number(:exactly, 1)
         end
 
+        # Capture all Apache Kafka messages of the given block.
+        #
+        # @yield the given block to capture the messages
+        # @return [Array<Hash{Symbol => Mixed}>] the captured messages
+        def capture(&block)
+          matches?(block)
+          @messages
+        end
+
         # Collect the expectation arguments for the Kafka message passing. (eg.
         # topic)
         #
@@ -173,7 +182,8 @@ module Rimless
           return true unless @schema
 
           begin
-            Rimless.avro.decode(message[:data], schema_name: @schema.to_s)
+            Rimless.avro.decode(message[:encoded_data],
+                                schema_name: @schema.to_s)
             return true
           rescue Avro::IO::SchemaMatchException
             false
@@ -199,24 +209,30 @@ module Rimless
         def data_match?(message)
           return true unless @data.any?
 
-          actual_data = Rimless.avro.decode(message[:data])
-          expected_data = @data.deep_stringify_keys
-
-          actual_data.merge(expected_data) == actual_data
+          message[:data].merge(@data.deep_stringify_keys) == message[:data]
         end
 
         # Setup the +WaterDrop+ spies and record each sent message.
+        #
+        # rubocop:disable Metrics/AbcSize because of the message decoding
+        # rubocop:disable Metrics/MethodLength dito
         def listen_to_messages
+          decode = proc do |encoded|
+            { encoded_data: encoded, data: Rimless.avro.decode(encoded) }
+          end
+
           allow(WaterDrop::SyncProducer).to receive(:call) do |data, **args|
-            @messages << { data: data, args: args, type: :sync }
+            @messages << { args: args, type: :sync }.merge(decode[data])
             nil
           end
 
           allow(WaterDrop::AsyncProducer).to receive(:call) do |data, **args|
-            @messages << { data: data, args: args, type: :async }
+            @messages << { args: args, type: :async }.merge(decode[data])
             nil
           end
         end
+        # rubocop:enable Metrics/AbcSize
+        # rubocop:enable Metrics/MethodLength
 
         # Serve the RSpec API and return the positive failure message.
         #
@@ -264,7 +280,7 @@ module Rimless
           result = ['message']
 
           result << " with #{message[:args]}" if message[:args].any?
-          result << " with data: #{Rimless.avro.decode(message[:data])}"
+          result << " with data: #{message[:data]}"
 
           result.join
         end

data/lib/rimless/rspec.rb

@@ -62,6 +62,10 @@ RSPEC_CONFIGURER.configure do |config|
     # Clear any cached data
     FakeConfluentSchemaRegistryServer.clear
 
+    # Do not interact with Apache Kafka itself on tests
+    allow(WaterDrop::AsyncProducer).to receive(:call)
+    allow(WaterDrop::SyncProducer).to receive(:call)
+
     # Reconfigure the Rimless AvroTurf instance
     Rimless.configure_avro_turf
 

data/lib/rimless/version.rb

@@ -2,5 +2,5 @@
 
 module Rimless
   # The version of the +rimless+ gem
-  VERSION = '1.0.3'
+  VERSION = '1.2.0'
 end

data/rimless.gemspec

@@ -25,9 +25,9 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'activesupport', '>= 4.2.0'
   spec.add_runtime_dependency 'avro_turf', '~> 0.11.0'
-  spec.add_runtime_dependency 'karafka', '~> 1.3'
-  spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.3'
-  spec.add_runtime_dependency 'karafka-testing', '~> 1.3'
+  spec.add_runtime_dependency 'karafka', '~> 1.4'
+  spec.add_runtime_dependency 'karafka-sidekiq-backend', '~> 1.4'
+  spec.add_runtime_dependency 'karafka-testing', '~> 1.4'
   spec.add_runtime_dependency 'sinatra'
   spec.add_runtime_dependency 'sparsify', '~> 1.1'
   spec.add_runtime_dependency 'waterdrop', '~> 1.2'
@@ -36,14 +36,14 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '>= 1.16', '< 3'
   spec.add_development_dependency 'factory_bot', '~> 4.11'
-  spec.add_development_dependency 'railties', '>= 4.2.0'
+  spec.add_development_dependency 'railties', '>= 4.2.0', '< 6.1'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rdoc', '~> 6.1'
   spec.add_development_dependency 'redcarpet', '~> 3.4'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '~> 0.63.1'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.31'
-  spec.add_development_dependency 'simplecov', '~> 0.15'
+  spec.add_development_dependency 'simplecov', '< 0.18'
   spec.add_development_dependency 'timecop', '~> 0.9.1'
   spec.add_development_dependency 'vcr', '~> 3.0'
   spec.add_development_dependency 'yard', '~> 0.9.18'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rimless
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.2.0
 platform: ruby
 authors:
 - Hermann Mayer
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-14 00:00:00.000000000 Z
+date: 2021-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -44,42 +44,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: karafka-sidekiq-backend
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: karafka-testing
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
@@ -191,6 +191,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -198,6 +201,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -286,16 +292,16 @@ dependencies:
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.18'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.18'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -360,11 +366,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/workflows/documentation.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".simplecov"
-- ".travis.yml"
 - ".yardopts"
 - Appraisals
 - CHANGELOG.md
@@ -418,7 +425,7 @@ files:
 homepage: https://github.com/hausgold/rimless
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -433,8 +440,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.16
+signing_key:
 specification_version: 4
 summary: A bundle of opinionated Apache Kafka / Confluent Schema Registry helpers.
 test_files: []

data/.travis.yml

@@ -1,35 +0,0 @@
-env:
-  global:
-    - CC_TEST_REPORTER_ID=f926dbf2ed89c7918e7a47f4f14f7d8386cc102d4cfa0a4e84051c6c976975ea
-
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.7
-  - 2.6
-  - 2.5
-
-gemfile:
-  - gemfiles/rails_4.2.gemfile
-  - gemfiles/rails_5.0.gemfile
-  - gemfiles/rails_5.1.gemfile
-  - gemfiles/rails_5.2.gemfile
-  - gemfiles/rails_6.0.gemfile
-
-before_install: gem install bundler
-install:
-  # Rails 4 is not Ruby 2.7 compatible, so we skip this build
-  - |
-    [[ "${BUNDLE_GEMFILE}" =~ rails_4 && \
-      "${TRAVIS_RUBY_VERSION}" =~ 2.7 ]] && exit || true
-  # Regular build
-  - bundle install --jobs=3 --retry=3
-
-before_script:
-  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-  - chmod +x ./cc-test-reporter
-  - ./cc-test-reporter before-build
-script: bundle exec rake
-after_script:
-  - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT