karafka-sidekiq-backend 1.4.3 → 1.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df894f5849e760cf4fc34d38606fed44f0fef045336fd316b5a67af6f949a264
-  data.tar.gz: d2f26478a4ce8ff38423faf55874ed6f5127ce2895dd8907fc0559f008f8564c
+  metadata.gz: 4c4560c74ccb09e782379a950507c3989adef5c3be5415beeeb2003b9d641768
+  data.tar.gz: 22422624c731faa13641adf08a36cd067913826358a57a38e45f8a5a5be553af
 SHA512:
-  metadata.gz: 8fe7c353bce5b28ab7abf84136d7a0550387627a036c4a9f90e8bd567441931c765f48781f282cc74a0f47f23df91c80b605f614bcfab917ccd2c002a8471cfa
-  data.tar.gz: 1d7764a08a7c407457762acb6c84110c5ef43cd3f630d7d89d284629cb76d2bf0bfef98bf3587020c3ce640b85428b27830bb86737569a29cf946d4211b848b9
+  metadata.gz: 0ec03fbc6ece2e0664e61d44de00b30f3e0690cb0d3bc8e15eceb9af73b33272264927b8fccb025c742911fa4b506360a5bb7d1536e85f9ed2421a9ba78405ce
+  data.tar.gz: 5222810dc912bde65da7db269e07d3556a850e471b3b02217fba07764590e6673e3f975bbe7e697f2204668d24cc518ff97eae20b7d684803208b35a4017a8bb

data/.github/workflows/ci.yml

@@ -1,5 +1,7 @@
 name: ci
 
+concurrency: ci-${{ github.ref }}
+
 on:
   pull_request:
   push:
@@ -14,12 +16,11 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.1'
           - '3.0'
           - '2.7'
-          - '2.6'
-          - 'jruby-9.3.1.0'
         include:
-          - ruby: '3.0'
+          - ruby: '3.1'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v2
@@ -51,7 +52,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.0
+          ruby-version: 3.1
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin

data/.ruby-version

@@ -1 +1 @@
-3.0.3
+3.1.0

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Karafka Sidekiq Backend
 
+## 1.4.4 (2022-03-06)
+- Ruby 3.1 support
+- Drop support for Ruby 2.6
+- Fix unsafe JSON serialization issued by Sidekiq (#122)
+- Provide sub-second precision for data processed in Sidekiq
+
 ## 1.4.3 (2021-12-05)
 - Source code metadata url added to the gemspec
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-sidekiq-backend (1.4.3)
+    karafka-sidekiq-backend (1.4.4)
       karafka (~> 1.4.0)
       sidekiq (>= 4.2)
 
@@ -13,11 +13,11 @@ GEM
     delivery_boy (1.1.0)
       king_konf (~> 1.0)
       ruby-kafka (~> 1.0)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     digest-crc (0.6.4)
       rake (>= 12.0.0, < 14.0.0)
     docile (1.4.0)
-    dry-configurable (0.13.0)
+    dry-configurable (0.14.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.6)
     dry-container (0.9.0)
@@ -29,7 +29,7 @@ GEM
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
     dry-inflector (0.2.1)
-    dry-initializer (3.0.4)
+    dry-initializer (3.1.1)
     dry-logic (1.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
@@ -37,7 +37,7 @@ GEM
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.8.0)
+    dry-schema (1.9.1)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
@@ -50,24 +50,20 @@ GEM
       dry-core (~> 0.5, >= 0.5)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.7.0)
+    dry-validation (1.8.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
       dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.8, >= 1.8.0)
+      dry-schema (~> 1.9, >= 1.9.1)
     envlogic (1.1.4)
       dry-inflector (~> 0.1)
-    io-console (0.5.9)
-    irb (1.3.7)
-      reline (>= 0.2.7)
-    karafka (1.4.11)
+    karafka (1.4.13)
       dry-configurable (~> 0.13)
       dry-inflector (~> 0.2)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
       envlogic (~> 1.1)
-      irb (~> 1.3)
       ruby-kafka (>= 1.3.0)
       thor (>= 1.1)
       waterdrop (~> 1.4)
@@ -75,25 +71,23 @@ GEM
     king_konf (1.0.0)
     rack (2.2.3)
     rake (13.0.6)
-    redis (4.5.1)
-    reline (0.2.7)
-      io-console (~> 0.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    redis (4.6.0)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
     ruby-kafka (1.4.0)
       digest-crc
-    sidekiq (6.3.1)
+    sidekiq (6.4.1)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
@@ -102,8 +96,8 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.1)
     waterdrop (1.4.4)
       delivery_boy (>= 0.2, < 2.x)
       dry-configurable (~> 0.13)
@@ -111,10 +105,9 @@ GEM
       dry-validation (~> 1.7)
       ruby-kafka (>= 1.3.0)
       zeitwerk (~> 2.4)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.4)
 
 PLATFORMS
-  x86_64-darwin
   x86_64-linux
 
 DEPENDENCIES
@@ -123,4 +116,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.2.32
+   2.3.7

data/karafka-sidekiq-backend.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'karafka', '~> 1.4.0'
   spec.add_dependency 'sidekiq', '>= 4.2'
-  spec.required_ruby_version = '>= 2.6.0'
+  spec.required_ruby_version = '>= 2.7'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')

data/lib/karafka/backends/sidekiq.rb

@@ -6,7 +6,7 @@ module Karafka
     # Sidekiq backend that schedules stuff to Sidekiq worker for delayed execution
     module Sidekiq
       # Karafka Sidekiq backend version
-      VERSION = '1.4.3'
+      VERSION = '1.4.4'
 
       # Enqueues the execution of perform method into a worker.
       # @note Each worker needs to have a class #perform_async method that will allow us to pass
@@ -15,10 +15,18 @@ module Karafka
       #   in the worker)
       def process
         Karafka.monitor.instrument('backends.sidekiq.process', caller: self) do
+          # We add batch metadata only for batch worker
+          batch_metadata_hash = if respond_to?(:batch_metadata)
+                                  # We remove deserializer as it's not safe to convert it to json
+                                  # and we can rebuild it anyhow based on the routing data in the
+                                  # worker
+                                  batch_metadata.to_h.transform_keys(&:to_s).except('deserializer')
+                                end
+
           topic.worker.perform_async(
             topic.id,
             topic.interchanger.encode(params_batch),
-            respond_to?(:batch_metadata) ? batch_metadata.to_h : nil
+            batch_metadata_hash
           )
         end
       end

data/lib/karafka/interchanger.rb

@@ -14,9 +14,19 @@ module Karafka
     # @return [Array<Hash>] Array with hash built out of params data
     def encode(params_batch)
       params_batch.map do |param|
+        metadata_hash = param.metadata.to_h
+        # All the metadata must have stringified keys in order to safe serialize
+        metadata_hash.transform_keys!(&:to_s)
+        # This will be taken back from the routing and is not safe for serialization
+        metadata_hash.delete('deserializer')
+
+        # Cast times to strings, we will de-serialize it back in Sidekiq
+        metadata_hash['receive_time'] = metadata_hash['receive_time'].to_f
+        metadata_hash['create_time'] = metadata_hash['create_time'].to_f
+
         {
           'raw_payload' => param.raw_payload,
-          'metadata' => param.metadata.to_h
+          'metadata' => metadata_hash
         }
       end
     end
@@ -24,7 +34,16 @@ module Karafka
     # @param params_batch [Array<Hash>] Sidekiq params that are now an array
     # @return [Array<Hash>] exactly what we've fetched from Sidekiq
     def decode(params_batch)
-      params_batch
+      params_batch.map do |param|
+        metadata = param['metadata']
+        # Covert serialized dates back to what they were
+        metadata['receive_time'] = Time.at(metadata['receive_time']).to_time
+        metadata['create_time'] = Time.at(metadata['create_time']).to_time
+
+        param['metadata'] = metadata
+
+        param
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-sidekiq-backend
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.4.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2021-12-05 00:00:00.000000000 Z
+date: 2022-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka
@@ -113,14 +113,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: Karafka Sidekiq backend for background messages processing