karafka-sidekiq-backend 1.1.0 → 1.2.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 553a28cdeb2098d283624a91466b7818d9714e72
-  data.tar.gz: c5ff002048db97b873050bcaf536b3a060106f7e
+SHA256:
+  metadata.gz: 16558b17b692066dafda116ac5305f97eb090223c525ef9f7d928421ff035778
+  data.tar.gz: 47a3e0b1dcc3553077c2144f2e175318ac1de7a2bff40f33501c54361bf369b6
 SHA512:
-  metadata.gz: 2aebcdd89c8584069f08f1e40940e92f2f431c97498612a1e28a08f8a1ef71b88451b60fdc1c6912829766fb18e89650560c5bc2c8c0bbd89def76a2a8f94b62
-  data.tar.gz: 181d1cfbd38631224bd2ece579e1bd149d6b3d0cc5dc52e5b6bd39f0cfbdcc8637fc6f4a902c4a4bb7b4f49c4b76512bdcdb4eae6940fef98db5bbe336bccbed
+  metadata.gz: 43ea83d99f762fa1d9d709773959fd5c94cba803be58f4f32d19fd1bf080c4ec25d195c72bc874a947af9b4220ea91bfee47e890d229319d8f7607e1da30e0fe
+  data.tar.gz: 74b9743f1df06af45ca2953e055bd3622693b326ce347f1bf887d8a299c59a1b8c3c3cedbaefb7b76453aee0f788571717a85484934d5cab2578ec0355906310

data/.ruby-version

@@ -1 +1 @@
-2.4.2
+2.5.0

data/.travis.yml

@@ -9,6 +9,7 @@ rvm:
   - 2.4.0
   - 2.4.1
   - 2.4.2
+  - 2.5.0
   - jruby-head
 script: bundle exec rspec spec/
 env:

data/CHANGELOG.md

@@ -1,9 +1,14 @@
 # Karafka Sidekiq Backend
 
+## 1.2.0.beta1
+- #274 - Rename controllers to consumers
+- Karafka 1.2 support
+
 ## 1.1
 - Ruby 2.4.2 support
 - #235 - Rename perform to consume
 - Karafka 1.1 support
+- Ruby 2.5.0 support
 
 ## 1.0
 

data/Gemfile

@@ -5,7 +5,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
-  gem 'timecop'
   gem 'rspec'
   gem 'simplecov'
+  gem 'timecop'
 end

data/Gemfile.lock

@@ -1,23 +1,23 @@
 PATH
   remote: .
   specs:
-    karafka-sidekiq-backend (1.1.0)
-      karafka (>= 1.1.0.alpha2)
+    karafka-sidekiq-backend (1.2.0.beta1)
+      karafka (>= 1.2.0.beta1)
       sidekiq (>= 4.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.1.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     concurrent-ruby (1.0.5)
     connection_pool (2.2.1)
-    delivery_boy (0.2.2)
+    delivery_boy (0.2.4)
       king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.4)
+      ruby-kafka (~> 0.5.1)
     diff-lcs (1.3)
     docile (1.1.5)
     dry-configurable (0.7.0)
@@ -25,14 +25,24 @@ GEM
     dry-container (0.6.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.0)
+    dry-core (0.4.4)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.2.0)
+    dry-events (0.1.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+    dry-inflector (0.1.1)
     dry-logic (0.4.2)
       dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-types (0.12.1)
+    dry-monitor (0.1.2)
+      dry-configurable (~> 0.5)
+      dry-equalizer (~> 0.2)
+      dry-events (~> 0.1)
+      rouge (~> 2.0, >= 2.2.1)
+    dry-types (0.12.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1)
       dry-container (~> 0.3)
@@ -49,36 +59,39 @@ GEM
       dry-types (~> 0.12.0)
     envlogic (1.0.4)
       activesupport
-    i18n (0.9.1)
+    i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     inflecto (0.0.2)
     json (2.1.0)
-    karafka (1.1.0.alpha2)
-      activesupport (>= 5.0)
+    karafka (1.2.0.beta1)
+      activesupport (>= 4.0)
       dry-configurable (~> 0.7)
+      dry-inflector (~> 0.1.1)
+      dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       envlogic (~> 1.0)
       multi_json (>= 1.12)
       rake (>= 11.3)
       require_all (>= 1.4)
-      ruby-kafka (>= 0.5)
+      ruby-kafka (>= 0.5.3)
       thor (~> 0.19)
-      waterdrop (>= 1.0.alpha2)
-    king_konf (0.1.8)
-    minitest (5.10.3)
-    multi_json (1.12.2)
+      waterdrop (>= 1.2.0.beta1)
+    king_konf (0.1.10)
+    minitest (5.11.3)
+    multi_json (1.13.1)
     null-logger (0.1.4)
-    rack (2.0.3)
+    rack (2.0.4)
     rack-protection (2.0.0)
       rack
-    rake (12.2.1)
+    rake (12.3.0)
     redis (4.0.1)
-    require_all (1.4.0)
+    require_all (1.5.0)
+    rouge (2.2.1)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
       rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.0)
+    rspec-core (3.7.1)
       rspec-support (~> 3.7.0)
     rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -86,13 +99,13 @@ GEM
     rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
-    ruby-kafka (0.5.0)
-    sidekiq (5.0.5)
+    rspec-support (3.7.1)
+    ruby-kafka (0.5.3)
+    sidekiq (5.1.1)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
-      redis (>= 3.3.4, < 5)
+      redis (>= 3.3.5, < 5)
     simplecov (0.15.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
@@ -101,14 +114,15 @@ GEM
     thor (0.20.0)
     thread_safe (0.3.6)
     timecop (0.9.1)
-    tzinfo (1.2.4)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    waterdrop (1.0.0)
-      delivery_boy (>= 0.2.2)
+    waterdrop (1.2.0.beta1)
+      delivery_boy (>= 0.2.3)
       dry-configurable (~> 0.7)
+      dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       null-logger
-      ruby-kafka (>= 0.5)
+      ruby-kafka (>= 0.5.3)
 
 PLATFORMS
   ruby
@@ -120,4 +134,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   1.15.4
+   1.16.1

data/README.md

@@ -120,23 +120,14 @@ end
 * [Karafka Sidekiq Backend Travis CI](https://travis-ci.org/karafka/karafka-sidekiq-backend)
 * [Karafka Sidekiq Backend Coditsu](https://app.coditsu.io/karafka/repositories/karafka-sidekiq-backend)
 
-## Note on Patches/Pull Requests
+## Note on contributions
 
-Fork the project.
-Make your feature addition or bug fix.
-Add tests for it. This is important so we don't break it in a future versions unintentionally.
-Commit, do not mess with Rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull). Send me a pull request. Bonus points for topic branches.
+First, thank you for considering contributing to Karafka! It's people like you that make the open source community such a great community!
 
-[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/repositories/karafka-sidekiq-backend)
+Each pull request must pass all the rspec specs and meet our quality requirements.
 
-Each pull request must pass our quality requirements. To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combinse multiple linters and code analyzers for both code and documentation.
+To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
 
-Unfortunately, it does not yet support independent forks, however you should be fine by looking at what we require.
+Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/repositories/karafka-sidekiq-backend/builds/commit_builds) of Karafka Sidekiq Backend repository.
 
-Please run:
-
-```bash
-bundle exec rake
-```
-
-to check if everything is in order. After that you can submit a pull request.
+[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/repositories/karafka-sidekiq-backend/builds/commit_builds)

data/karafka-sidekiq-backend.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
   spec.description = 'Karafka Sidekiq backend for background messages processing'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'karafka', '>= 1.1.0.alpha2'
+  spec.add_dependency 'karafka', '>= 1.2.0.beta1'
   spec.add_dependency 'sidekiq', '>= 4.2'
   spec.required_ruby_version = '>= 2.3.0'
 

data/lib/karafka/backends/sidekiq.rb

@@ -5,21 +5,20 @@ module Karafka
     # Sidekiq backend that schedules stuff to Sidekiq worker for delayed execution
     module Sidekiq
       # Karafka Sidekiq backend version
-      VERSION = '1.1.0'
-
-      private
+      VERSION = '1.2.0.beta1'
 
       # Enqueues the execution of perform method into a worker.
       # @note Each worker needs to have a class #perform_async method that will allow us to pass
       #   parameters into it. We always pass topic as a first argument and this request
-      #   params_batch as a second one (we pass topic to be able to build back the controller
+      #   params_batch as a second one (we pass topic to be able to build back the consumer
       #   in the worker)
-      def process
-        Karafka.monitor.notice(self.class, params_batch)
-        topic.worker.perform_async(
-          topic.id,
-          topic.interchanger.load(params_batch.to_a)
-        )
+      def call
+        Karafka.monitor.instrument('backends.sidekiq.process', caller: self) do
+          topic.worker.perform_async(
+            topic.id,
+            topic.interchanger.load(params_batch.to_a)
+          )
+        end
       end
     end
   end

data/lib/karafka/base_worker.rb

@@ -5,23 +5,28 @@ module Karafka
   class BaseWorker
     include Sidekiq::Worker
 
-    # Executes the logic that lies in #perform Karafka controller method
+    # Executes the logic that lies in #perform Karafka consumer method
     # @param topic_id [String] Unique topic id that we will use to find a proper topic
     # @param params_batch [Array] Array with messages batch
     def perform(topic_id, params_batch)
-      Karafka.monitor.notice(self.class, params_batch)
-      controller(topic_id, params_batch).consume
+      consumer = consumer(topic_id, params_batch)
+
+      Karafka.monitor.instrument(
+        'backends.sidekiq.base_worker.perform',
+        caller: self,
+        consumer: consumer
+      ) { consumer.consume }
     end
 
     private
 
-    # @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
-    #   with params_batch assigned already (controller is ready to use)
-    def controller(topic_id, params_batch)
+    # @return [Karafka::Consumer] descendant of Karafka::BaseConsumer that matches the topic
+    #   with params_batch assigned already (consumer is ready to use)
+    def consumer(topic_id, params_batch)
       topic = Karafka::Routing::Router.find(topic_id)
-      controller = topic.controller.new
-      controller.params_batch = topic.interchanger.parse(params_batch)
-      controller
+      consumer = topic.consumer.new
+      consumer.params_batch = topic.interchanger.parse(params_batch)
+      consumer
     end
   end
 end

data/lib/karafka/extensions/sidekiq_topic_attributes.rb

@@ -8,9 +8,9 @@ module Karafka
     module SidekiqTopicAttributes
       # @return [Class] Class (not an instance) of a worker that should be used to schedule the
       #   background job
-      # @note If not provided - will be built based on the provided controller
+      # @note If not provided - will be built based on the provided consumer
       def worker
-        @worker ||= backend == :sidekiq ? Karafka::Workers::Builder.new(controller).build : nil
+        @worker ||= backend == :sidekiq ? Karafka::Workers::Builder.new(consumer).build : nil
       end
 
       # @return [Class] Interchanger class (not an instance) that we want to use to interchange

data/lib/karafka/instrumentation/listener.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for extensions of Karafka core framework instrumentation
+  module Instrumentation
+    # Additional methods for listener that listen on instrumentation related to the Sidekiq
+    # backend of Karafka
+    module Listener
+      class << self
+        # Logs info about scheduling of a certain dataset with a Sidekiq backend
+        # @param event [Dry::Events::Event] event details including payload
+        def on_backends_sidekiq_process(event)
+          count = event[:caller].send(:params_batch).to_a.size
+          topic = event[:caller].topic.name
+          time = event[:time]
+          info "Scheduling of #{count} messages to Sidekiq on topic #{topic} took #{time} ms"
+        end
+
+        # Logs ino about processing certain events with a given Sidekiq worker
+        # @param event [Dry::Events::Event] event details including payload
+        def on_backends_sidekiq_base_worker_perform(event)
+          count = event[:consumer].send(:params_batch).to_a.size
+          topic = event[:consumer].topic.name
+          time = event[:time]
+          info "Sidekiq processing of topic #{topic} with #{count} messages took #{time} ms"
+        end
+      end
+    end
+  end
+end

data/lib/karafka/interchanger.rb

@@ -20,12 +20,20 @@ module Karafka
       end
 
       # @param params_batch [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
-      # @note Hash is what we need to build Karafka::Params::Params, so we do nothing
-      #   with it. If you implement your own interchanger logic, this method needs to return
-      #   a hash with appropriate data that will be used to build Karafka::Params::Params
-      # @return [Hash] We return exactly what we received. We rely on sidekiqs default
-      #   interchanging format
+      # @note Since Sidekiq does not like symbols, we restore symbolized keys for system keys, so
+      #   everything can work as expected. Keep in mind, that custom data will always be assigned
+      #   with string keys per design. To change it, pleasae change this interchanger and create
+      #   your own custom parser
       def parse(params_batch)
+        params_batch.map! do |params|
+          Karafka::Params::Params::SYSTEM_KEYS.each do |key|
+            stringified_key = key.to_s
+            params[key] = params.delete(stringified_key) if params.key?(stringified_key)
+          end
+
+          params
+        end
+
         params_batch
       end
     end

data/lib/karafka/workers/builder.rb

@@ -3,23 +3,23 @@
 module Karafka
   # Internal stuff related to workers
   module Workers
-    # Builder is used to check if there is a proper controller with the same name as
-    # a controller and if not, it will create a default one using Karafka::BaseWorker
-    # This is used as a building layer between controllers and workers. it will be only used
-    # when user does not provide his own worker that should perform controller stuff
+    # Builder is used to check if there is a proper consumer with the same name as
+    # a consumer and if not, it will create a default one using Karafka::BaseWorker
+    # This is used as a building layer between consumers and workers. it will be only used
+    # when user does not provide his own worker that should perform consumer stuff
     class Builder
-      # @param controller_class [Karafka::BaseController] descendant of Karafka::BaseController
+      # @param consumer_class [Karafka::BaseConsumer] descendant of Karafka::BaseConsumer
       # @example Create a worker builder
-      #   Karafka::Workers::Builder.new(SuperController)
-      def initialize(controller_class)
-        @controller_class = controller_class
+      #   Karafka::Workers::Builder.new(SuperConsumer)
+      def initialize(consumer_class)
+        @consumer_class = consumer_class
       end
 
       # @return [Class] Sidekiq worker class that already exists or new build based
-      #   on the provided controller_class name
-      # @example Controller: SuperController
+      #   on the provided consumer_class name
+      # @example Consumer: SuperConsumer
       #   build #=> SuperWorker
-      # @example Controller: Videos::NewVideosController
+      # @example Consumer: Videos::NewVideosConsumer
       #   build #=> Videos::NewVideosWorker
       def build
         return matcher.match if matcher.match
@@ -37,12 +37,12 @@ module Karafka
         Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
       end
 
-      # @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between controller
+      # @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between consumer
       #   and appropriate worker
       def matcher
         @matcher ||= Helpers::ClassMatcher.new(
-          @controller_class,
-          from: 'Controller',
+          @consumer_class,
+          from: 'Consumer',
           to: 'Worker'
         )
       end

data/lib/karafka-sidekiq-backend.rb

@@ -27,3 +27,8 @@ end
 # Uses Karafka loader to load all the sources that this backend needs
 Karafka::Loader.load!(Karafka::Backends::Sidekiq.core_root)
 Karafka::AttributesMap.prepend(Karafka::Extensions::SidekiqAttributesMap)
+# Register internal events for instrumentation
+%w[
+  backends.sidekiq.process
+  backends.sidekiq.base_worker.perform
+].each(&Karafka.monitor.method(:register_event))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: karafka-sidekiq-backend
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-03 00:00:00.000000000 Z
+date: 2018-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.0.alpha2
+        version: 1.2.0.beta1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.0.alpha2
+        version: 1.2.0.beta1
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -64,6 +64,7 @@ files:
 - lib/karafka/errors.rb
 - lib/karafka/extensions/sidekiq_attributes_map.rb
 - lib/karafka/extensions/sidekiq_topic_attributes.rb
+- lib/karafka/instrumentation/listener.rb
 - lib/karafka/interchanger.rb
 - lib/karafka/workers/builder.rb
 homepage: https://github.com/karafka/karafka-sidekiq-backend
@@ -81,12 +82,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Karafka Sidekiq backend for background messages processing