karafka-sidekiq-backend 1.2.0.beta2 → 1.2.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5cbf46f4edac28d13fad77fc120bf92456e37380bcee52718713ad94f4e9a5f2
-  data.tar.gz: 5da6e57d3f61b3d2f59410e78c7c94a1a26b04051714f2895f36ce00c2099e8c
+  metadata.gz: f1e9d5b7c2a522e2ff126eca4b34ffc37efe1d4e9117d6996daf6b9fd959f393
+  data.tar.gz: ecf5117a7726db83707021ffeeab152b51d0f1e044fae590c8f2018ed94d52c8
 SHA512:
-  metadata.gz: c98cfa129b4733d21fa6fc0491ba522c89c5187fc1d1113b8256e9a980f50dba131353b61e37ee3f9867d8ad6e6a665abb0d2980ec3cff9bcab91fc3dfa9b53f
-  data.tar.gz: 63fd8534461a01e418e365f13ad207cfad9d5f7dff24adc09b8b15301e22749b3ad2d49b38ead45a70628722af90d2e36f5d48f3ce1cdadf1783567a9304fc93
+  metadata.gz: 41ab55ab65c5750ae7d2b4a4ec844da5049612f4a8e59419177dd1e85a43ff4e992a622dc6be9ee335dffd8a39ae5526fb16eaf10af27981f5494106f132441a
+  data.tar.gz: af65c10503038935f1a892aef9a231611196ea8c06a683b54e3cd6cf07f944e91cce13f19832b49070e7850a65ffd790a822091eb002dc7dfc6c575bb311f874

data/CHANGELOG.md

@@ -1,6 +1,8 @@
 # Karafka Sidekiq Backend
 
-## 1.2.0.beta2
+## 1.2.0.beta3
+- ```#load``` and ```#parse``` are renamed to ```#encode``` and ```#decode``` in interchangers
+- Default interchanger is not Base64 Marshal dump
 - #274 - Rename controllers to consumers
 - Karafka 1.2 support
 - Parse data before pushing to Sidekiq

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-sidekiq-backend (1.2.0.beta2)
+    karafka-sidekiq-backend (1.2.0.beta3)
       karafka (>= 1.2.0.beta2)
       sidekiq (>= 4.2)
 
@@ -81,7 +81,7 @@ GEM
     multi_json (1.13.1)
     null-logger (0.1.4)
     rack (2.0.4)
-    rack-protection (2.0.0)
+    rack-protection (2.0.1)
       rack
     rake (12.3.0)
     redis (4.0.1)
@@ -116,13 +116,12 @@ GEM
     timecop (0.9.1)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    waterdrop (1.2.0.beta1)
-      delivery_boy (>= 0.2.3)
+    waterdrop (1.2.0)
+      delivery_boy (~> 0.2)
       dry-configurable (~> 0.7)
       dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       null-logger
-      ruby-kafka (>= 0.5.3)
 
 PLATFORMS
   ruby

data/README.md

@@ -85,9 +85,9 @@ Note that even then, you need to specify a controller that will schedule a backg
 Custom workers need to provide a ```#perform_async``` method. It needs to accept two arguments:
 
  - ```topic_id``` - first argument is a current topic id from which a given message comes
- - ```params``` - all the params that came from Kafka + additional metadata. This data format might be changed if you use custom interchangers. Otherwise it will be an instance of Karafka::Params::Params.
+ - ```params_batch``` - all the params that came from Kafka + additional metadata. This data format might be changed if you use custom interchangers. Otherwise it will be an instance of Karafka::Params::ParamsBatch.
 
-Keep in mind, that params might be in two states: parsed or unparsed when passed to #perform_async. This means, that if you use custom interchangers and/or custom workers, you might want to look into Karafka's sources to see exactly how it works.
+**Note**: If you use custom interchangers, keep in mind, that params inside params batch might be in two states: parsed or unparsed when passed to #perform_async. This means, that if you use custom interchangers and/or custom workers, you might want to look into Karafka's sources to see exactly how it works.
 
 ### Interchangers
 
@@ -95,29 +95,6 @@ Custom interchangers target issues with non-standard (binary, etc.) data that we
 
 **Warning**: if you decide to use slow interchangers, they might significantly slow down Karafka.
 
-```ruby
-class Base64Interchanger
-  class << self
-    def load(params_batch)
-      params_batch.map do |params|
-        Base64.encode64(Marshal.dump(params))
-      end
-    end
-
-    def parse(params_batch)
-      params_batch.map do |params|
-        Marshal.load(Base64.decode64(params))
-      end
-    end
-  end
-end
-
-topic :binary_video_details do
-  controller Videos::DetailsController
-  interchanger Base64Interchanger
-end
-```
-
 ## References
 
 * [Karafka framework](https://github.com/karafka/karafka)

data/lib/karafka/backends/sidekiq.rb

@@ -5,7 +5,7 @@ module Karafka
     # Sidekiq backend that schedules stuff to Sidekiq worker for delayed execution
     module Sidekiq
       # Karafka Sidekiq backend version
-      VERSION = '1.2.0.beta2'
+      VERSION = '1.2.0.beta3'
 
       # Enqueues the execution of perform method into a worker.
       # @note Each worker needs to have a class #perform_async method that will allow us to pass
@@ -13,10 +13,12 @@ module Karafka
       #   params_batch as a second one (we pass topic to be able to build back the consumer
       #   in the worker)
       def process
+        Karafka.monitor.instrument('backends.sidekiq.process', caller: self) do
           topic.worker.perform_async(
             topic.id,
-            topic.interchanger.load(params_batch)
+            topic.interchanger.encode(params_batch)
           )
+        end
       end
     end
   end

data/lib/karafka/base_worker.rb

@@ -25,7 +25,7 @@ module Karafka
     def consumer(topic_id, params_batch)
       topic = Karafka::Routing::Router.find(topic_id)
       consumer = topic.consumer.new
-      consumer.params_batch = topic.interchanger.parse(params_batch)
+      consumer.params_batch = topic.interchanger.decode(params_batch)
       consumer
     end
   end

data/lib/karafka/interchanger.rb

@@ -5,38 +5,28 @@ module Karafka
   # This is meant to target mostly issues with data encoding like this one:
   # https://github.com/mperham/sidekiq/issues/197
   # Each custom interchanger should implement following methods:
-  #   - load - it is meant to encode params before they get stored inside Redis
-  #   - parse - decoded params back to a hash format that we can use
+  #   - encode - it is meant to encode params before they get stored inside Redis
+  #   - decode - decoded params back to a hash format that we can use
+  #
+  # This interchanger is not the fastets but it handles many unusual cases and deals well with
+  # more complex Ruby and Rails objects
   class Interchanger
     class << self
       # @param params_batch [Karafka::Params::ParamsBatch] Karafka params batch object
       # @note Params might not be parsed because of lazy loading feature. If you implement your
       #   own interchanger logic, this method needs to return data that can be converted to
       #   json with default Sidekiqs logic
-      # @return [Karafka::Params::ParamsBatch] parsed params batch. There are too many problems
+      # @return [String] parsed params batch encoded into a string. There are too many problems
       #   with passing unparsed data from Karafka to Sidekiq, to make it a default. In case you
       #   need this, please implement your own interchanger.
-      def load(params_batch)
-        params_batch.parsed
+      def encode(params_batch)
+        Base64.encode64(Marshal.dump(params_batch.parsed))
       end
 
-      # @param params_batch [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
-      # @note Since Sidekiq does not like symbols, we restore symbolized keys for system keys, so
-      #   everything can work as expected. Keep in mind, that custom data will always be assigned
-      #   with string keys per design. To change it, please change this interchanger and create
-      #   your own custom parser
-      def parse(params_batch)
-        params_batch.map! do |params|
-          Karafka::Params::Params::SYSTEM_KEYS.each do |key|
-            stringified_key = key.to_s
-            next unless params.key?(stringified_key)
-            params[key] ||= params.delete(stringified_key)
-          end
-
-          params
-        end
-
-        params_batch
+      # @param params_batch [String] Encoded params batch string that we use to rebuild params
+      # @return [Karafka::Params::ParamsBatch] rebuilt params batch
+      def decode(params_batch)
+        Marshal.load(Base64.decode64(params_batch))
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: karafka-sidekiq-backend
 version: !ruby/object:Gem::Version
-  version: 1.2.0.beta2
+  version: 1.2.0.beta3
 platform: ruby
 authors:
 - Maciej Mensfeld
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-16 00:00:00.000000000 Z
+date: 2018-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka
@@ -87,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Karafka Sidekiq backend for background messages processing