karafka-sidekiq-backend 1.2.0 → 1.3.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fd28798ad1b7204f30fac7e291bb9fe40f09158b5707f0e1b2d78f11d936036
-  data.tar.gz: 6e82e1395e08289f40584c756ef39abd756bac9505cc2f82c18731cb85b83e18
+  metadata.gz: a5fc5bf5aa80d215a0fa572fbe6853e046b4c5e9d6e39c1aac7a5178614e45f4
+  data.tar.gz: 9fcbdb5dd8b193c48669656bcdad9b20b894cf982ae400f0a11de57610922373
 SHA512:
-  metadata.gz: 84fdd1fb717042b3316ac761489e4ced64bffc69799d3fa0a00333209c58c3d3a5c58329d435386218c4c15edd33c5e4074bfa51eef87800f3418f1a80a7dbd4
-  data.tar.gz: e35e3578b2478aeeea23b891b19b5e507b38ba91329572dfce748127c05d5d019e692d87b0da54495d5833082b626a4181e9b82f1e643bccce99e72e50eefac2
+  metadata.gz: 37405ff9a618e5f21fe0bc38488f6ee6beef120f148e73cba28c78b91409656ef72d7dc2d28f0cf265af025e1b63c368dc645c2f86dae713d73147a0ddcedf12
+  data.tar.gz: 406316953d9e34be8f173e72f4baf389b982c504019b9da557a853975ed14cfa49020146565d5de74672db62690cf1e5b44b18509b71e1b727e3c5049c88eac2

checksums.yaml.gz.sig

@@ -0,0 +1,2 @@
+>����St3z֒X�f��ɾ, ���v#.�����`���ǫÿ�*�?�� �I+�;�̽.t�m�v��:bd����M�	(�k�NM���q���y��9U��p�G"�Mr�[><$`��<�AX"�2$��v�4����L�ݻf��T�bw���t�,8�,�P)P
�]O���k��c}�Sg�}�b����0R��@8�2�7�iɸ:�[{@�G�g�*�+
+'�}c��P����/�#!�QcT�-읮�t+

data/.coditsu/ci.yml

@@ -0,0 +1,3 @@
+repository_id: '84213f08-9083-4d3d-99c6-b8af8dd0e609'
+api_key: <%= ENV['CODITSU_API_KEY'] %>
+api_secret: <%= ENV['CODITSU_API_SECRET'] %>

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+open_collective: karafka

data/.gitignore

@@ -13,6 +13,7 @@ db/*.sqlite3
 /tmp/*
 *.gem
 *.~
+.coditsu/local.yml
 
 # various artifacts
 **.war

data/.ruby-version

@@ -1 +1 @@
-2.5.0
+2.6.3

data/.travis.yml

@@ -1,18 +1,37 @@
-language: ruby
-sudo: false
-rvm:
-  - 2.3.0
-  - 2.3.1
-  - 2.3.2
-  - 2.3.3
-  - 2.3.4
-  - 2.4.0
-  - 2.4.1
-  - 2.4.2
-  - 2.5.0
-  - jruby-head
-script: bundle exec rspec spec/
-env:
-  global:
-    - JRUBY_OPTS='--debug'
-install: bundle install --jobs=3 --retry=3
+services:
+  - docker
+
+dist: trusty
+cache: bundler
+
+git:
+  depth: false
+
+test: &test
+  stage: Test
+  language: ruby
+  before_install:
+    - gem install bundler
+    - gem update --system
+  script: bundle exec rspec
+
+jobs:
+  include:
+    - <<: *test
+      rvm: 2.6.3
+    - <<: *test
+      rvm: 2.5.5
+    - <<: *test
+      rvm: 2.4.6
+
+    - stage: coditsu
+      language: ruby
+      rvm: 2.6.3
+      before_install:
+        - gem update --system
+        - gem install bundler
+      script: \curl -sSL https://api.coditsu.io/run/ci | bash
+
+stages:
+  - coditsu
+  - test

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka Sidekiq Backend
 
+## 1.3.0.rc1 (2019-07-31)
+- Ruby 2.6.3 support
+- Drop Ruby 2.3 support
+- Sync with Karafka: metadata support
+- Sync with Karafka #463 (Split parsers into serializers / deserializers)
+- #26 - Make listeners as instances
+- Zeitwerk integration
+- drop jruby support
+
 ## 1.2.0
 - ```#load``` and ```#parse``` are renamed to ```#encode``` and ```#decode``` in interchangers
 - #274 - Rename controllers to consumers

data/Gemfile

@@ -4,8 +4,10 @@ source 'https://rubygems.org'
 
 gemspec
 
-group :development, :test do
+gem 'karafka', '1.3.0.rc1'
+gem 'waterdrop', '1.3.0.rc1'
+
+group :test do
   gem 'rspec'
   gem 'simplecov'
-  gem 'timecop'
 end

data/Gemfile.lock

@@ -1,136 +1,135 @@
 PATH
   remote: .
   specs:
-    karafka-sidekiq-backend (1.2.0)
-      karafka (~> 1.2)
+    karafka-sidekiq-backend (1.3.0.rc1)
+      karafka (~> 1.3.0.rc1)
       sidekiq (>= 4.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.5)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    concurrent-ruby (1.0.5)
-    connection_pool (2.2.1)
-    delivery_boy (0.2.4)
-      king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.5.1)
+    concurrent-ruby (1.1.5)
+    connection_pool (2.2.2)
+    delivery_boy (0.2.8)
+      king_konf (~> 0.3)
+      ruby-kafka (~> 0.7.8)
     diff-lcs (1.3)
-    docile (1.3.0)
-    dry-configurable (0.7.0)
+    digest-crc (0.4.1)
+    docile (1.3.2)
+    dry-configurable (0.8.3)
       concurrent-ruby (~> 1.0)
-    dry-container (0.6.0)
+      dry-core (~> 0.4, >= 0.4.7)
+    dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.5)
+    dry-core (0.4.8)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.0)
-    dry-events (0.1.0)
+    dry-equalizer (0.2.2)
+    dry-events (0.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-    dry-inflector (0.1.1)
-    dry-logic (0.4.2)
-      dry-container (~> 0.2, >= 0.2.6)
+    dry-inflector (0.1.2)
+    dry-initializer (3.0.1)
+    dry-logic (1.0.2)
+      concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-monitor (0.1.2)
+    dry-monitor (0.3.1)
       dry-configurable (~> 0.5)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
       dry-events (~> 0.1)
-      rouge (~> 2.0, >= 2.2.1)
-    dry-types (0.12.2)
+    dry-schema (1.3.1)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1)
-      dry-container (~> 0.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-configurable (~> 0.8, >= 0.8.3)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.2)
-      inflecto (~> 0.0.0, >= 0.0.2)
-    dry-validation (0.11.1)
+      dry-initializer (~> 3.0)
+      dry-logic (~> 1.0)
+      dry-types (~> 1.0)
+    dry-types (1.1.1)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-container (~> 0.3)
+      dry-core (~> 0.4, >= 0.4.4)
+      dry-equalizer (~> 0.2, >= 0.2.2)
+      dry-inflector (~> 0.1, >= 0.1.2)
+      dry-logic (~> 1.0, >= 1.0.2)
+    dry-validation (1.2.1)
+      concurrent-ruby (~> 1.0)
+      dry-container (~> 0.7, >= 0.7.1)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.0)
-      dry-types (~> 0.12.0)
+      dry-initializer (~> 3.0)
+      dry-schema (~> 1.0, >= 1.3.1)
     envlogic (1.1.0)
       dry-inflector (~> 0.1)
-    i18n (0.9.5)
-      concurrent-ruby (~> 1.0)
-    inflecto (0.0.2)
-    json (2.1.0)
-    karafka (1.2.0)
-      activesupport (>= 4.0)
-      dry-configurable (~> 0.7)
-      dry-inflector (~> 0.1.1)
-      dry-monitor (~> 0.1)
-      dry-validation (~> 0.11)
+    irb (1.0.0)
+    json (2.2.0)
+    karafka (1.3.0.rc1)
+      dry-configurable (~> 0.8)
+      dry-inflector (~> 0.1)
+      dry-monitor (~> 0.3)
+      dry-validation (~> 1.2)
       envlogic (~> 1.0)
+      irb (~> 1.0)
       multi_json (>= 1.12)
       rake (>= 11.3)
-      require_all (>= 1.4)
-      ruby-kafka (>= 0.5.3)
-      thor (~> 0.19)
-      waterdrop (~> 1.2)
-    king_konf (0.1.10)
-    minitest (5.11.3)
+      ruby-kafka (>= 0.7.8)
+      thor (~> 0.20)
+      waterdrop (~> 1.3.0.rc1)
+      zeitwerk (~> 2.1)
+    king_konf (0.3.7)
     multi_json (1.13.1)
-    null-logger (0.1.5)
-    rack (2.0.4)
-    rack-protection (2.0.1)
+    rack (2.0.7)
+    rack-protection (2.0.5)
       rack
-    rake (12.3.1)
-    redis (4.0.1)
-    require_all (2.0.0)
-    rouge (2.2.1)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rake (12.3.3)
+    redis (4.1.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.2)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    ruby-kafka (0.5.4)
-    sidekiq (5.1.1)
-      concurrent-ruby (~> 1.0)
-      connection_pool (~> 2.2, >= 2.2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.2)
+    ruby-kafka (0.7.9)
+      digest-crc
+    sidekiq (5.2.7)
+      connection_pool (~> 2.2, >= 2.2.2)
+      rack (>= 1.5.0)
       rack-protection (>= 1.5.0)
       redis (>= 3.3.5, < 5)
-    simplecov (0.16.1)
+    simplecov (0.17.0)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    thor (0.20.0)
-    thread_safe (0.3.6)
-    timecop (0.9.1)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    waterdrop (1.2.0)
+    thor (0.20.3)
+    waterdrop (1.3.0.rc1)
       delivery_boy (~> 0.2)
-      dry-configurable (~> 0.7)
-      dry-monitor (~> 0.1)
-      dry-validation (~> 0.11)
-      null-logger
+      dry-configurable (~> 0.8)
+      dry-monitor (~> 0.3)
+      dry-validation (~> 1.2)
+      ruby-kafka (>= 0.7.8)
+      zeitwerk (~> 2.1)
+    zeitwerk (2.1.9)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  karafka (= 1.3.0.rc1)
   karafka-sidekiq-backend!
   rspec
   simplecov
-  timecop
+  waterdrop (= 1.3.0.rc1)
 
 BUNDLED WITH
-   1.16.1
+   2.0.2

data/README.md

@@ -1,6 +1,6 @@
 # Karafka Sidekiq Backend
 
-[![Build Status](https://travis-ci.org/karafka/sidekiq-backend.png)](https://travis-ci.org/karafka/karafka-sidekiq-backend)
+[![Build Status](https://travis-ci.org/karafka/sidekiq-backend.svg)](https://travis-ci.org/karafka/sidekiq-backend)
 [![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 [Karafka Sidekiq Backend](https://github.com/karafka/sidekiq-backend) provides support for consuming (processing) received Kafka messages inside of Sidekiq workers.
@@ -45,7 +45,7 @@ or on a per topic level:
 App.routes.draw do
   consumer_group :videos_consumer do
     topic :binary_video_details do
-      controller Videos::DetailsController
+      consumer Videos::DetailsConsumer
       worker Workers::DetailsWorker
       interchanger Interchangers::MyCustomInterchanger
     end
@@ -53,7 +53,7 @@ App.routes.draw do
 end
 ```
 
-You don't need to do anything beyond that. Karafka will know, that you want to run your controllers ```#perform``` method in a background job.
+You don't need to do anything beyond that. Karafka will know, that you want to run your consumer's ```#consume``` method in a background job.
 
 ## Configuration
 
@@ -67,7 +67,7 @@ There are two options you can set inside of the ```topic``` block:
 
 ### Workers
 
-Karafka by default will build a worker that will correspond to each of your controllers (so you will have a pair - controller and a worker). All of them will inherit from ```ApplicationWorker``` and will share all its settings.
+Karafka by default will build a worker that will correspond to each of your consumers (so you will have a pair - consumer and a worker). All of them will inherit from ```ApplicationWorker``` and will share all its settings.
 
 To run Sidekiq you should have sidekiq.yml file in *config* folder. The example of ```sidekiq.yml``` file will be generated to config/sidekiq.yml.example once you run ```bundle exec karafka install```.
 
@@ -75,17 +75,17 @@ However, if you want to use a raw Sidekiq worker (without any Karafka additional
 
 ```ruby
 topic :incoming_messages do
-  controller MessagesController
+  consumer MessagesConsumer
   worker MyCustomWorker
 end
 ```
 
-Note that even then, you need to specify a controller that will schedule a background task.
+Note that even then, you need to specify a consumer that will schedule a background task.
 
 Custom workers need to provide a ```#perform_async``` method. It needs to accept two arguments:
 
  - ```topic_id``` - first argument is a current topic id from which a given message comes
- - ```params_batch``` - all the params that came from Kafka + additional metadata. This data format might be changed if you use custom interchangers. Otherwise it will be an instance of Karafka::Params::ParamsBatch.
+ - ```params_batch``` - all the params that came from Kafka + additional metadata. This data format might be changed if you use custom interchangers. Otherwise, it will be an instance of Karafka::Params::ParamsBatch.
 
 **Note**: If you use custom interchangers, keep in mind, that params inside params batch might be in two states: parsed or unparsed when passed to #perform_async. This means, that if you use custom interchangers and/or custom workers, you might want to look into Karafka's sources to see exactly how it works.
 
@@ -93,6 +93,37 @@ Custom workers need to provide a ```#perform_async``` method. It needs to accept
 
 Custom interchangers target issues with non-standard (binary, etc.) data that we want to store when we do ```#perform_async```. This data might be corrupted when fetched in a worker (see [this](https://github.com/karafka/karafka/issues/30) issue). With custom interchangers, you can encode/compress data before it is being passed to scheduling and decode/decompress it when it gets into the worker.
 
+To specify the interchanger for a topic, specify the interchanger inside routes like this:
+
+```ruby
+App.routes.draw do
+  consumer_group :videos_consumer do
+    topic :binary_video_details do
+      consumer Videos::DetailsConsumer
+      interchanger Interchangers::MyCustomInterchanger
+    end
+  end
+end
+```
+Each custom interchanger should define `encode` to encode params before they get stored in Redis, and `decode` to convert the params to hash format, as shown below:
+
+```ruby
+class Base64Interchanger
+  class << self
+    def encode(params_batch)
+      # Note, that you need to cast the params_batch to an array in order to get it work
+      # in sidekiq later
+      Base64.encode64(Marshal.dump(params_batch.to_a))
+    end
+
+    def decode(params_string)
+      Marshal.load(Base64.decode64(params_string))
+    end
+  end
+end
+
+```
+
 **Warning**: if you decide to use slow interchangers, they might significantly slow down Karafka.
 
 ## References
@@ -105,7 +136,7 @@ Custom interchangers target issues with non-standard (binary, etc.) data that we
 
 First, thank you for considering contributing to Karafka! It's people like you that make the open source community such a great community!
 
-Each pull request must pass all the rspec specs and meet our quality requirements.
+Each pull request must pass all the RSpec specs and meet our quality requirements.
 
 To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
 

data/certs/mensfeld.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
+NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
+dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
+uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
+MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
+y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
+seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
+94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
+TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
+Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
+ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
+xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
+A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
+hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
+CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
+i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
+KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
+nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+-----END CERTIFICATE-----

data/karafka-sidekiq-backend.gemspec

@@ -15,10 +15,15 @@ Gem::Specification.new do |spec|
   spec.description = 'Karafka Sidekiq backend for background messages processing'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'karafka', '~> 1.2'
+  spec.add_dependency 'karafka', '~> 1.3.0.rc1'
   spec.add_dependency 'sidekiq', '>= 4.2'
-  spec.required_ruby_version = '>= 2.3.0'
+  spec.required_ruby_version = '>= 2.4.0'
 
+  if $PROGRAM_NAME.end_with?('gem')
+    spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
+
+  spec.cert_chain    = %w[certs/mensfeld.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]

data/lib/karafka-sidekiq-backend.rb

@@ -1,34 +1,3 @@
 # frozen_string_literal: true
 
-# @note Active Support is already included since Karafka is using it directly so no need
-#   to require it again in the gemspec, etc
-%w[
-  karafka
-  sidekiq
-  active_support/core_ext/class/subclasses
-].each(&method(:require))
-
-# Karafka framework namespace
-module Karafka
-  # Namespace for all the backends that process data
-  module Backends
-    # Sidekiq Karafka backend
-    module Sidekiq
-      class << self
-        # @return [String] path to Karafka gem root core
-        def core_root
-          Pathname.new(File.expand_path('karafka', __dir__))
-        end
-      end
-    end
-  end
-end
-
-# Uses Karafka loader to load all the sources that this backend needs
-Karafka::Loader.load!(Karafka::Backends::Sidekiq.core_root)
-Karafka::AttributesMap.prepend(Karafka::Extensions::SidekiqAttributesMap)
-# Register internal events for instrumentation
-%w[
-  backends.sidekiq.process
-  backends.sidekiq.base_worker.perform
-].each(&Karafka.monitor.method(:register_event))
+require 'karafka_sidekiq_backend'

data/lib/karafka/backends/sidekiq.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Namespace for alternative processing backends for Karafka framework
   module Backends
     # Sidekiq backend that schedules stuff to Sidekiq worker for delayed execution
     module Sidekiq
       # Karafka Sidekiq backend version
-      VERSION = '1.2.0'
+      VERSION = '1.3.0.rc1'
 
       # Enqueues the execution of perform method into a worker.
       # @note Each worker needs to have a class #perform_async method that will allow us to pass
@@ -16,7 +17,8 @@ module Karafka
         Karafka.monitor.instrument('backends.sidekiq.process', caller: self) do
           topic.worker.perform_async(
             topic.id,
-            topic.interchanger.encode(params_batch)
+            topic.interchanger.encode(params_batch),
+            respond_to?(:metadata) ? metadata : nil
           )
         end
       end

data/lib/karafka/base_worker.rb

@@ -5,11 +5,30 @@ module Karafka
   class BaseWorker
     include Sidekiq::Worker
 
+    class << self
+      # Returns the base worker class for application.
+      #
+      # @return [Class] first worker that inherited from Karafka::BaseWorker. Karafka
+      #   assumes that it is the base worker for an application.
+      # @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when application
+      #   base worker was not defined.
+      def base_worker
+        @inherited || raise(Errors::BaseWorkerDescentantMissing)
+      end
+
+      # @param subclass [Class] subclass of the worker
+      # @return [Class] subclass of the worker that was selected
+      def inherited(subclass)
+        @inherited ||= subclass
+      end
+    end
+
     # Executes the logic that lies in #perform Karafka consumer method
     # @param topic_id [String] Unique topic id that we will use to find a proper topic
-    # @param params_batch [Array] Array with messages batch
-    def perform(topic_id, params_batch)
-      consumer = consumer(topic_id, params_batch)
+    # @param params_batch [Array<Hash>] Array with messages batch
+    # @param metadata [Hash, nil] hash with all the metadata or nil if not present
+    def perform(topic_id, params_batch, metadata)
+      consumer = consumer(topic_id, params_batch, metadata)
 
       Karafka.monitor.instrument(
         'backends.sidekiq.base_worker.perform',
@@ -20,12 +39,27 @@ module Karafka
 
     private
 
+    # @see `#perform` for exact params descriptions
+    # @param topic_id [String]
+    # @param params_batch [Array<Hash>]
+    # @param metadata [Hash, nil]
     # @return [Karafka::Consumer] descendant of Karafka::BaseConsumer that matches the topic
     #   with params_batch assigned already (consumer is ready to use)
-    def consumer(topic_id, params_batch)
+    def consumer(topic_id, params_batch, metadata)
       topic = Karafka::Routing::Router.find(topic_id)
-      consumer = topic.consumer.new
-      consumer.params_batch = topic.interchanger.decode(params_batch)
+      consumer = topic.consumer.new(topic)
+      consumer.params_batch = Params::Builders::ParamsBatch.from_array(
+        topic.interchanger.decode(params_batch),
+        topic
+      )
+
+      if topic.batch_fetching
+        consumer.metadata = Params::Builders::Metadata.from_hash(
+          metadata,
+          topic
+        )
+      end
+
       consumer
     end
   end

data/lib/karafka/extensions/metadata_builder.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Extensions
+    # Extension for metadata builder to allow building metadata from a hash
+    module MetadataBuilder
+      # Builds metadata from hash
+      # @param hash [Hash] hash with metadata
+      # @param topic [Karafka::Routing::Topic] topic instance
+      # @return [Karafka::Params::Metadata] metadata instance
+      def from_hash(hash, topic)
+        # Parser needs to be merged as this is the only non-serializable object
+        # so it gets reconstructed from the topic
+        Karafka::Params::Metadata
+          .new
+          .merge!(hash)
+          .merge!('deserializer' => topic.deserializer)
+      end
+    end
+  end
+end

data/lib/karafka/extensions/params_batch_builder.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Extensions
+    # Extension for params batch builder for reconstruction of the batch from an array
+    module ParamsBatchBuilder
+      # Builds params batch from array of hashes
+      # @param array [Array<Hash>] array with hash messages
+      # @param topic [Karafka::Routing::Topic] topic for which we build the batch
+      # @return [Karafka::Params::ParamsBatch] built batch
+      # @note We rebuild the params batch from array after the serialization
+      def from_array(array, topic)
+        params_array = array.map! do |hash|
+          Karafka::Params::Builders::Params.from_hash(hash, topic)
+        end
+
+        Karafka::Params::ParamsBatch.new(params_array)
+      end
+    end
+  end
+end

data/lib/karafka/extensions/params_builder.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Extensions
+    # Extension for rebuilding params from a hash
+    module ParamsBuilder
+      # Builds params from a hash
+      # @param hash [Hash] hash with params details
+      # @param topic [Karafka::Routing::Topic] topic for which we build the params
+      # @return [Karafka::Params::Params] built params
+      def from_hash(hash, topic)
+        Karafka::Params::Params
+          .new
+          .merge!(hash)
+          .merge!('deserializer' => topic.deserializer)
+      end
+    end
+  end
+end

data/lib/karafka/extensions/sidekiq_attributes_map.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Extensions for Karafka components
   module Extensions
     # Additional Karafka::Attributes map topic attributes that can be used when worker
     # is active and we use sidekiq backend

data/lib/karafka/extensions/sidekiq_topic_attributes.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Namespace for additional extensions that we include into some Karafka components, to gain
-  # extra features that we require
   module Extensions
     # Additional Karafka::Routing::Topic methods that are required to work with Sidekiq backend
     module SidekiqTopicAttributes
@@ -16,7 +14,7 @@ module Karafka
       # @return [Class] Interchanger class (not an instance) that we want to use to interchange
       #   params between Karafka server and Karafka background job
       def interchanger
-        @interchanger ||= Karafka::Interchanger
+        @interchanger ||= Karafka::Interchanger.new
       end
 
       # Creates attributes writers for worker and interchanger, so they can be overwritten
@@ -28,5 +26,3 @@ module Karafka
     end
   end
 end
-
-Karafka::Routing::Topic.include Karafka::Extensions::SidekiqTopicAttributes

data/lib/karafka/extensions/stdout_listener.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Extensions
+    # Additional methods for listener that listen on instrumentation related to the Sidekiq
+    # backend of Karafka
+    module StdoutListener
+      # Logs info about scheduling of a certain dataset with a Sidekiq backend
+      # @param event [Dry::Events::Event] event details including payload
+      def on_backends_sidekiq_process(event)
+        count = event[:caller].send(:params_batch).to_a.size
+        topic = event[:caller].topic.name
+        time = event[:time]
+        info "Scheduling of #{count} messages to Sidekiq on topic #{topic} took #{time} ms"
+      end
+
+      # Logs ino about processing certain events with a given Sidekiq worker
+      # @param event [Dry::Events::Event] event details including payload
+      def on_backends_sidekiq_base_worker_perform(event)
+        count = event[:consumer].send(:params_batch).to_a.size
+        topic = event[:consumer].topic.name
+        time = event[:time]
+        info "Sidekiq processing of topic #{topic} with #{count} messages took #{time} ms"
+      end
+    end
+  end
+end

data/lib/karafka/interchanger.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Interchangers allow us to format/encode/pack data that is being send to perform_async
+  # Interchanger allows us to format/encode/pack data that is being send to perform_async
   # This is meant to target mostly issues with data encoding like this one:
   # https://github.com/mperham/sidekiq/issues/197
   # Each custom interchanger should implement following methods:
@@ -10,19 +10,17 @@ module Karafka
   #
   # This interchanger uses default Sidekiq options to exchange data
   class Interchanger
-    class << self
-      # @param params_batch [Karafka::Params::ParamsBatch] Karafka params batch object
-      # @return [Array<Karafka::Params::Params>] Array with hash/hashwithindiff values that will
-      #   be serialized using Sidekiq serialization engine
-      def encode(params_batch)
-        params_batch.to_a
-      end
+    # @param params_batch [Karafka::Params::ParamsBatch] Karafka params batch object
+    # @return [Array<Karafka::Params::Params>] Array with hash/hashwithindiff values that will
+    #   be serialized using Sidekiq serialization engine
+    def encode(params_batch)
+      params_batch.to_a
+    end
 
-      # @param params_batch [Array<Hash>] Sidekiq params that are now an array
-      # @return [Array<Hash>] exactly what we've fetched from Sidekiq
-      def decode(params_batch)
-        params_batch
-      end
+    # @param params_batch [Array<Hash>] Sidekiq params that are now an array
+    # @return [Array<Hash>] exactly what we've fetched from Sidekiq
+    def decode(params_batch)
+      params_batch
     end
   end
 end

data/lib/karafka/workers/builder.rb

@@ -23,20 +23,13 @@ module Karafka
       #   build #=> Videos::NewVideosWorker
       def build
         return matcher.match if matcher.match
-        klass = Class.new(base)
+
+        klass = Class.new(Karafka::BaseWorker.base_worker)
         matcher.scope.const_set(matcher.name, klass)
       end
 
       private
 
-      # @return [Class] descendant of Karafka::BaseWorker from which all other workers
-      #   should inherit
-      # @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when Karafka cannot detect
-      #   direct Karafka::BaseWorker descendant from which it could build workers
-      def base
-        Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
-      end
-
       # @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between consumer
       #   and appropriate worker
       def matcher

data/lib/karafka_sidekiq_backend.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+%w[
+  karafka
+  sidekiq
+].each(&method(:require))
+
+require_relative 'karafka/errors'
+
+Zeitwerk::Loader
+  .for_gem
+  .tap { |loader| loader.ignore("#{__dir__}/karafka_sidekiq_backend.rb") }
+  .tap { |loader| loader.ignore("#{__dir__}/karafka-sidekiq-backend.rb") }
+  .tap(&:setup)
+  .tap(&:eager_load)
+
+Karafka::Params::Builders::Params.extend(Karafka::Extensions::ParamsBuilder)
+Karafka::Params::Builders::ParamsBatch.extend(Karafka::Extensions::ParamsBatchBuilder)
+Karafka::Params::Builders::Metadata.extend(Karafka::Extensions::MetadataBuilder)
+Karafka::Routing::Topic.include(Karafka::Extensions::SidekiqTopicAttributes)
+Karafka::AttributesMap.prepend(Karafka::Extensions::SidekiqAttributesMap)
+Karafka::Instrumentation::StdoutListener.include(Karafka::Extensions::StdoutListener)
+
+# Register internal events for instrumentation
+%w[
+  backends.sidekiq.process
+  backends.sidekiq.base_worker.perform
+].each(&Karafka.monitor.method(:register_event))

metadata

@@ -1,14 +1,40 @@
 --- !ruby/object:Gem::Specification
 name: karafka-sidekiq-backend
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
 autorequire: 
 bindir: bin
-cert_chain: []
-date: 2018-03-23 00:00:00.000000000 Z
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
+  NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
+  dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
+  uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
+  MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
+  y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
+  seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
+  94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
+  TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
+  Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
+  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+  EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
+  ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
+  xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
+  A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
+  hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
+  CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
+  i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
+  KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
+  nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+  -----END CERTIFICATE-----
+date: 2019-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka
@@ -16,14 +42,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 1.3.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 1.3.0.rc1
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -45,6 +71,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coditsu/ci.yml"
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -56,17 +84,22 @@ files:
 - MIT-LICENCE
 - README.md
 - Rakefile
+- certs/mensfeld.pem
 - karafka-sidekiq-backend.gemspec
 - lib/karafka-sidekiq-backend.rb
 - lib/karafka/backends/sidekiq.rb
 - lib/karafka/base_worker.rb
 - lib/karafka/cli/worker.rb
 - lib/karafka/errors.rb
+- lib/karafka/extensions/metadata_builder.rb
+- lib/karafka/extensions/params_batch_builder.rb
+- lib/karafka/extensions/params_builder.rb
 - lib/karafka/extensions/sidekiq_attributes_map.rb
 - lib/karafka/extensions/sidekiq_topic_attributes.rb
-- lib/karafka/instrumentation/listener.rb
+- lib/karafka/extensions/stdout_listener.rb
 - lib/karafka/interchanger.rb
 - lib/karafka/workers/builder.rb
+- lib/karafka_sidekiq_backend.rb
 homepage: https://github.com/karafka/karafka-sidekiq-backend
 licenses:
 - MIT
@@ -79,15 +112,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Karafka Sidekiq backend for background messages processing

data/lib/karafka/instrumentation/listener.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for extensions of Karafka core framework instrumentation
-  module Instrumentation
-    # Additional methods for listener that listen on instrumentation related to the Sidekiq
-    # backend of Karafka
-    module Listener
-      class << self
-        # Logs info about scheduling of a certain dataset with a Sidekiq backend
-        # @param event [Dry::Events::Event] event details including payload
-        def on_backends_sidekiq_process(event)
-          count = event[:caller].send(:params_batch).to_a.size
-          topic = event[:caller].topic.name
-          time = event[:time]
-          info "Scheduling of #{count} messages to Sidekiq on topic #{topic} took #{time} ms"
-        end
-
-        # Logs ino about processing certain events with a given Sidekiq worker
-        # @param event [Dry::Events::Event] event details including payload
-        def on_backends_sidekiq_base_worker_perform(event)
-          count = event[:consumer].send(:params_batch).to_a.size
-          topic = event[:consumer].topic.name
-          time = event[:time]
-          info "Sidekiq processing of topic #{topic} with #{count} messages took #{time} ms"
-        end
-      end
-    end
-  end
-end