karafka-sidekiq-backend 1.3.0.rc1 → 1.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (24) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +3 -2
  3. data.tar.gz.sig +0 -0
  4. data/.diffend.yml +3 -0
  5. data/.github/workflows/ci.yml +70 -0
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +18 -0
  8. data/Gemfile +2 -3
  9. data/Gemfile.lock +73 -81
  10. data/{MIT-LICENCE → MIT-LICENSE} +0 -0
  11. data/README.md +11 -15
  12. data/certs/mensfeld.pem +21 -21
  13. data/karafka-sidekiq-backend.gemspec +3 -3
  14. data/lib/karafka/backends/sidekiq.rb +2 -2
  15. data/lib/karafka/base_worker.rb +2 -1
  16. data/lib/karafka/extensions/{metadata_builder.rb → batch_metadata_builder.rb} +7 -5
  17. data/lib/karafka/extensions/params_batch_builder.rb +2 -2
  18. data/lib/karafka/extensions/params_builder.rb +8 -3
  19. data/lib/karafka/extensions/stdout_listener.rb +2 -2
  20. data/lib/karafka/interchanger.rb +7 -3
  21. data/lib/karafka_sidekiq_backend.rb +1 -1
  22. metadata +34 -33
  23. metadata.gz.sig +0 -0
  24. data/.travis.yml +0 -37
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a5fc5bf5aa80d215a0fa572fbe6853e046b4c5e9d6e39c1aac7a5178614e45f4
4
- data.tar.gz: 9fcbdb5dd8b193c48669656bcdad9b20b894cf982ae400f0a11de57610922373
3
+ metadata.gz: 59f0d78d87fab493807408fc060573046af103be1bc309c5f1fa53aa326e6565
4
+ data.tar.gz: 16c5fa9968f3c0fcbaac95a87ff07b8eb8b417cbf5e8941c13b8b712b5ac2d8e
5
5
  SHA512:
6
- metadata.gz: 37405ff9a618e5f21fe0bc38488f6ee6beef120f148e73cba28c78b91409656ef72d7dc2d28f0cf265af025e1b63c368dc645c2f86dae713d73147a0ddcedf12
7
- data.tar.gz: 406316953d9e34be8f173e72f4baf389b982c504019b9da557a853975ed14cfa49020146565d5de74672db62690cf1e5b44b18509b71e1b727e3c5049c88eac2
6
+ metadata.gz: a86bf1b17952a1281f54436373071c5e9f575c4bea12cf7f51748544bf0a7637eecc343443b01a46694358fdbea714095127d2cc66eb9ef64e8f2d8d818ccc7d
7
+ data.tar.gz: 9ad59e1ed9b67e5edeed76eaeb99c73090c48de55a5176db32c293401faa07d82f9fe4045f6ae8f3c995e618328bb09ca4d4065502748165025e0cf5e5647b31
checksums.yaml.gz.sig CHANGED
@@ -1,2 +1,3 @@
1
- >����St3z֒X�f��ɾ, ���v#.�����`���ǫÿ�*�?�� �I+�;�̽.t�m�v��:bd����M� (�k�NM���q���y��9U��p�G"�Mr�[><$`��<�AX"�2$��v�4����L�ݻf��T�bw���t�,8�,�P)P�]O���k��c}�Sg�}�b����0R��@8�2�7�iɸ:�[{@�G�g�*�+
2
- '�}c��P����/�#!�QcT�-읮�t+
1
+ ��6I����1�����lmµ#��w9�`,Z$�VV@�$�wyo������x7�(�B5�V�!��e�!W��0Ꮐ=�;��t�GU
2
+ �;�UCA�*ng|�ʰ�w����_-�����$�J�����[_�(��//]���~n���� �[�z�aQl8]šř�����-_���&7E:I;1D0H��5^�V�t�~-,XJ����c}BVቼ����!��C�DC��GS�ʽ!ڷ��O�Ry�����5a���'���[:���
3
+ Uh�3�S�^hV�m�0o��=��BЅ�w��8�Ѩ#�����0QXc�gˀr��y\�]y����
data.tar.gz.sig CHANGED
Binary file
data/.diffend.yml ADDED
@@ -0,0 +1,3 @@
1
+ project_id: '0dab66ec-affc-405a-b994-4077d0b20140'
2
+ shareable_id: '61e9572e-2947-4b2b-bdcb-0652853bfdb0'
3
+ shareable_key: 'c9ea8a34-51de-4175-919f-d734bdc3c775'
data/.github/workflows/ci.yml ADDED
@@ -0,0 +1,70 @@
1
+ name: ci
2
+
3
+ on:
4
+ pull_request:
5
+ push:
6
+ schedule:
7
+ - cron: '0 1 * * *'
8
+
9
+ jobs:
10
+ specs:
11
+ runs-on: ubuntu-latest
12
+ needs: diffend
13
+ strategy:
14
+ fail-fast: false
15
+ matrix:
16
+ ruby:
17
+ - '3.0'
18
+ - '2.7'
19
+ - '2.6'
20
+ include:
21
+ - ruby: '3.0'
22
+ coverage: 'true'
23
+ steps:
24
+ - uses: actions/checkout@v2
25
+ - name: Install package dependencies
26
+ run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
27
+ - name: Set up Ruby
28
+ uses: ruby/setup-ruby@v1
29
+ with:
30
+ ruby-version: ${{matrix.ruby}}
31
+ - name: Install latest bundler
32
+ run: |
33
+ gem install bundler --no-document
34
+ bundle config set without 'tools benchmarks docs'
35
+ - name: Bundle install
36
+ run: bundle install --jobs 4 --retry 3
37
+ - name: Run all tests
38
+ env:
39
+ GITHUB_COVERAGE: ${{matrix.coverage}}
40
+ run: bundle exec rspec
41
+
42
+ diffend:
43
+ runs-on: ubuntu-latest
44
+ strategy:
45
+ fail-fast: false
46
+ steps:
47
+ - uses: actions/checkout@v2
48
+ with:
49
+ fetch-depth: 0
50
+ - name: Set up Ruby
51
+ uses: ruby/setup-ruby@v1
52
+ with:
53
+ ruby-version: 3.0
54
+ - name: Install latest bundler
55
+ run: gem install bundler --no-document
56
+ - name: Install Diffend plugin
57
+ run: bundle plugin install diffend
58
+ - name: Bundle Secure
59
+ run: bundle secure
60
+
61
+ coditsu:
62
+ runs-on: ubuntu-latest
63
+ strategy:
64
+ fail-fast: false
65
+ steps:
66
+ - uses: actions/checkout@v2
67
+ with:
68
+ fetch-depth: 0
69
+ - name: Run Coditsu
70
+ run: \curl -sSL https://api.coditsu.io/run/ci | bash
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 2.6.3
1
+ 3.0.1
data/CHANGELOG.md CHANGED
@@ -1,5 +1,23 @@
1
1
  # Karafka Sidekiq Backend
2
2
 
3
+ ## 1.4.2 (2021-04-21)
4
+ - Remove Ruby 2.5 support and update minimum Ruby requirement to 2.6
5
+
6
+ ## 1.4.1 (2020-27-10)
7
+ - Corrected interchanger documentation (Jack12816)
8
+ - Corrected the interchanger key reference (Jack12816)
9
+
10
+ ## 1.4.0 (2020-09-05)
11
+ - Update to match Karafka 1.4.0 params and batch metadata setup
12
+
13
+ ## 1.3.1 (2020-04-27)
14
+ - Ruby 2.6.5 support
15
+ - Ruby 2.7.1 support
16
+ - Change license to LGPL-3.0
17
+
18
+ ## 1.3.0 (2019-09-09)
19
+ - Drop Ruby 2.4 support
20
+
3
21
  ## 1.3.0.rc1 (2019-07-31)
4
22
  - Ruby 2.6.3 support
5
23
  - Drop Ruby 2.3 support
data/Gemfile CHANGED
@@ -2,10 +2,9 @@
2
2
 
3
3
  source 'https://rubygems.org'
4
4
 
5
- gemspec
5
+ plugin 'diffend'
6
6
 
7
- gem 'karafka', '1.3.0.rc1'
8
- gem 'waterdrop', '1.3.0.rc1'
7
+ gemspec
9
8
 
10
9
  group :test do
11
10
  gem 'rspec'
data/Gemfile.lock CHANGED
@@ -1,135 +1,127 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- karafka-sidekiq-backend (1.3.0.rc1)
5
- karafka (~> 1.3.0.rc1)
4
+ karafka-sidekiq-backend (1.4.2)
5
+ karafka (~> 1.4.0)
6
6
  sidekiq (>= 4.2)
7
7
 
8
8
  GEM
9
9
  remote: https://rubygems.org/
10
10
  specs:
11
- concurrent-ruby (1.1.5)
12
- connection_pool (2.2.2)
13
- delivery_boy (0.2.8)
14
- king_konf (~> 0.3)
15
- ruby-kafka (~> 0.7.8)
16
- diff-lcs (1.3)
17
- digest-crc (0.4.1)
18
- docile (1.3.2)
19
- dry-configurable (0.8.3)
11
+ concurrent-ruby (1.1.8)
12
+ connection_pool (2.2.5)
13
+ delivery_boy (1.1.0)
14
+ king_konf (~> 1.0)
15
+ ruby-kafka (~> 1.0)
16
+ diff-lcs (1.4.4)
17
+ digest-crc (0.6.3)
18
+ rake (>= 12.0.0, < 14.0.0)
19
+ docile (1.3.5)
20
+ dry-configurable (0.12.1)
20
21
  concurrent-ruby (~> 1.0)
21
- dry-core (~> 0.4, >= 0.4.7)
22
+ dry-core (~> 0.5, >= 0.5.0)
22
23
  dry-container (0.7.2)
23
24
  concurrent-ruby (~> 1.0)
24
25
  dry-configurable (~> 0.1, >= 0.1.3)
25
- dry-core (0.4.8)
26
+ dry-core (0.5.0)
26
27
  concurrent-ruby (~> 1.0)
27
- dry-equalizer (0.2.2)
28
- dry-events (0.2.0)
28
+ dry-equalizer (0.3.0)
29
+ dry-events (0.3.0)
29
30
  concurrent-ruby (~> 1.0)
30
- dry-core (~> 0.4)
31
- dry-equalizer (~> 0.2)
32
- dry-inflector (0.1.2)
33
- dry-initializer (3.0.1)
34
- dry-logic (1.0.2)
31
+ dry-core (~> 0.5, >= 0.5)
32
+ dry-inflector (0.2.0)
33
+ dry-initializer (3.0.4)
34
+ dry-logic (1.1.1)
35
35
  concurrent-ruby (~> 1.0)
36
- dry-core (~> 0.2)
37
- dry-equalizer (~> 0.2)
38
- dry-monitor (0.3.1)
36
+ dry-core (~> 0.5, >= 0.5)
37
+ dry-monitor (0.4.0)
39
38
  dry-configurable (~> 0.5)
40
- dry-core (~> 0.4)
41
- dry-equalizer (~> 0.2)
42
- dry-events (~> 0.1)
43
- dry-schema (1.3.1)
39
+ dry-core (~> 0.5, >= 0.5)
40
+ dry-events (~> 0.2)
41
+ dry-schema (1.6.2)
44
42
  concurrent-ruby (~> 1.0)
45
43
  dry-configurable (~> 0.8, >= 0.8.3)
46
- dry-core (~> 0.4)
47
- dry-equalizer (~> 0.2)
44
+ dry-core (~> 0.5, >= 0.5)
48
45
  dry-initializer (~> 3.0)
49
46
  dry-logic (~> 1.0)
50
- dry-types (~> 1.0)
51
- dry-types (1.1.1)
47
+ dry-types (~> 1.5)
48
+ dry-types (1.5.1)
52
49
  concurrent-ruby (~> 1.0)
53
50
  dry-container (~> 0.3)
54
- dry-core (~> 0.4, >= 0.4.4)
55
- dry-equalizer (~> 0.2, >= 0.2.2)
51
+ dry-core (~> 0.5, >= 0.5)
56
52
  dry-inflector (~> 0.1, >= 0.1.2)
57
53
  dry-logic (~> 1.0, >= 1.0.2)
58
- dry-validation (1.2.1)
54
+ dry-validation (1.6.0)
59
55
  concurrent-ruby (~> 1.0)
60
56
  dry-container (~> 0.7, >= 0.7.1)
61
57
  dry-core (~> 0.4)
62
58
  dry-equalizer (~> 0.2)
63
59
  dry-initializer (~> 3.0)
64
- dry-schema (~> 1.0, >= 1.3.1)
65
- envlogic (1.1.0)
60
+ dry-schema (~> 1.5, >= 1.5.2)
61
+ envlogic (1.1.3)
66
62
  dry-inflector (~> 0.1)
67
- irb (1.0.0)
68
- json (2.2.0)
69
- karafka (1.3.0.rc1)
63
+ io-console (0.5.9)
64
+ irb (1.3.5)
65
+ reline (>= 0.1.5)
66
+ karafka (1.4.4)
70
67
  dry-configurable (~> 0.8)
71
68
  dry-inflector (~> 0.1)
72
69
  dry-monitor (~> 0.3)
73
70
  dry-validation (~> 1.2)
74
- envlogic (~> 1.0)
71
+ envlogic (~> 1.1)
75
72
  irb (~> 1.0)
76
- multi_json (>= 1.12)
77
- rake (>= 11.3)
78
- ruby-kafka (>= 0.7.8)
79
- thor (~> 0.20)
80
- waterdrop (~> 1.3.0.rc1)
73
+ ruby-kafka (>= 1.0.0)
74
+ thor (>= 0.20)
75
+ waterdrop (~> 1.4.0)
81
76
  zeitwerk (~> 2.1)
82
- king_konf (0.3.7)
83
- multi_json (1.13.1)
84
- rack (2.0.7)
85
- rack-protection (2.0.5)
86
- rack
87
- rake (12.3.3)
88
- redis (4.1.2)
89
- rspec (3.8.0)
90
- rspec-core (~> 3.8.0)
91
- rspec-expectations (~> 3.8.0)
92
- rspec-mocks (~> 3.8.0)
93
- rspec-core (3.8.2)
94
- rspec-support (~> 3.8.0)
95
- rspec-expectations (3.8.4)
77
+ king_konf (1.0.0)
78
+ rack (2.2.3)
79
+ rake (13.0.3)
80
+ redis (4.2.5)
81
+ reline (0.2.5)
82
+ io-console (~> 0.5)
83
+ rspec (3.10.0)
84
+ rspec-core (~> 3.10.0)
85
+ rspec-expectations (~> 3.10.0)
86
+ rspec-mocks (~> 3.10.0)
87
+ rspec-core (3.10.1)
88
+ rspec-support (~> 3.10.0)
89
+ rspec-expectations (3.10.1)
96
90
  diff-lcs (>= 1.2.0, < 2.0)
97
- rspec-support (~> 3.8.0)
98
- rspec-mocks (3.8.1)
91
+ rspec-support (~> 3.10.0)
92
+ rspec-mocks (3.10.2)
99
93
  diff-lcs (>= 1.2.0, < 2.0)
100
- rspec-support (~> 3.8.0)
101
- rspec-support (3.8.2)
102
- ruby-kafka (0.7.9)
94
+ rspec-support (~> 3.10.0)
95
+ rspec-support (3.10.2)
96
+ ruby-kafka (1.3.0)
103
97
  digest-crc
104
- sidekiq (5.2.7)
105
- connection_pool (~> 2.2, >= 2.2.2)
106
- rack (>= 1.5.0)
107
- rack-protection (>= 1.5.0)
108
- redis (>= 3.3.5, < 5)
109
- simplecov (0.17.0)
98
+ sidekiq (6.2.1)
99
+ connection_pool (>= 2.2.2)
100
+ rack (~> 2.0)
101
+ redis (>= 4.2.0)
102
+ simplecov (0.21.2)
110
103
  docile (~> 1.1)
111
- json (>= 1.8, < 3)
112
- simplecov-html (~> 0.10.0)
113
- simplecov-html (0.10.2)
114
- thor (0.20.3)
115
- waterdrop (1.3.0.rc1)
116
- delivery_boy (~> 0.2)
104
+ simplecov-html (~> 0.11)
105
+ simplecov_json_formatter (~> 0.1)
106
+ simplecov-html (0.12.3)
107
+ simplecov_json_formatter (0.1.2)
108
+ thor (1.1.0)
109
+ waterdrop (1.4.2)
110
+ delivery_boy (>= 0.2, < 2.x)
117
111
  dry-configurable (~> 0.8)
118
112
  dry-monitor (~> 0.3)
119
113
  dry-validation (~> 1.2)
120
114
  ruby-kafka (>= 0.7.8)
121
115
  zeitwerk (~> 2.1)
122
- zeitwerk (2.1.9)
116
+ zeitwerk (2.4.2)
123
117
 
124
118
  PLATFORMS
125
- ruby
119
+ x86_64-linux
126
120
 
127
121
  DEPENDENCIES
128
- karafka (= 1.3.0.rc1)
129
122
  karafka-sidekiq-backend!
130
123
  rspec
131
124
  simplecov
132
- waterdrop (= 1.3.0.rc1)
133
125
 
134
126
  BUNDLED WITH
135
- 2.0.2
127
+ 2.2.16
data/{MIT-LICENCE → MIT-LICENSE} RENAMED
File without changes
data/README.md CHANGED
@@ -1,7 +1,8 @@
1
1
  # Karafka Sidekiq Backend
2
2
 
3
- [![Build Status](https://travis-ci.org/karafka/sidekiq-backend.svg)](https://travis-ci.org/karafka/sidekiq-backend)
4
- [![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
3
+ [![Build Status](https://github.com/karafka/sidekiq-backend/workflows/ci/badge.svg)](https://github.com/karafka/sidekiq-backend/actions?query=workflow%3Aci)
4
+ [![Gem Version](https://badge.fury.io/rb/karafka-sidekiq-backend.svg)](http://badge.fury.io/rb/karafka-sidekiq-backend)
5
+ [![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka)
5
6
 
6
7
  [Karafka Sidekiq Backend](https://github.com/karafka/sidekiq-backend) provides support for consuming (processing) received Kafka messages inside of Sidekiq workers.
7
8
 
@@ -45,6 +46,7 @@ or on a per topic level:
45
46
  App.routes.draw do
46
47
  consumer_group :videos_consumer do
47
48
  topic :binary_video_details do
49
+ backend :sidekiq
48
50
  consumer Videos::DetailsConsumer
49
51
  worker Workers::DetailsWorker
50
52
  interchanger Interchangers::MyCustomInterchanger
@@ -108,17 +110,13 @@ end
108
110
  Each custom interchanger should define `encode` to encode params before they get stored in Redis, and `decode` to convert the params to hash format, as shown below:
109
111
 
110
112
  ```ruby
111
- class Base64Interchanger
112
- class << self
113
- def encode(params_batch)
114
- # Note, that you need to cast the params_batch to an array in order to get it work
115
- # in sidekiq later
116
- Base64.encode64(Marshal.dump(params_batch.to_a))
117
- end
113
+ class Base64Interchanger < ::Karafka::Interchanger
114
+ def encode(params_batch)
115
+ Base64.encode64(Marshal.dump(super))
116
+ end
118
117
 
119
- def decode(params_string)
120
- Marshal.load(Base64.decode64(params_string))
121
- end
118
+ def decode(params_string)
119
+ Marshal.load(Base64.decode64(super))
122
120
  end
123
121
  end
124
122
 
@@ -129,7 +127,7 @@ end
129
127
  ## References
130
128
 
131
129
  * [Karafka framework](https://github.com/karafka/karafka)
132
- * [Karafka Sidekiq Backend Travis CI](https://travis-ci.org/karafka/karafka-sidekiq-backend)
130
+ * [Karafka Sidekiq Backend Actions CI](https://github.com/karafka/sidekiq-backend/actions?query=workflow%3Aci)
133
131
  * [Karafka Sidekiq Backend Coditsu](https://app.coditsu.io/karafka/repositories/karafka-sidekiq-backend)
134
132
 
135
133
  ## Note on contributions
@@ -141,5 +139,3 @@ Each pull request must pass all the RSpec specs and meet our quality requirement
141
139
  To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
142
140
 
143
141
  Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/repositories/karafka-sidekiq-backend/builds/commit_builds) of Karafka Sidekiq Backend repository.
144
-
145
- [![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/repositories/karafka-sidekiq-backend/builds/commit_builds)
data/certs/mensfeld.pem CHANGED
@@ -1,25 +1,25 @@
1
1
  -----BEGIN CERTIFICATE-----
2
2
  MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
3
- ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
4
- NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
5
- CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
6
- dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
7
- uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
8
- MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
9
- y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
10
- seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
11
- 94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
12
- TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
13
- Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
14
- BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
3
+ ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
4
+ NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
5
+ CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
6
+ 2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
7
+ N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
8
+ hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
9
+ sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
10
+ VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
11
+ tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
12
+ Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
13
+ LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
14
+ BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
15
15
  EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
16
- c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
17
- ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
18
- xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
19
- A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
20
- hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
21
- CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
22
- i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
23
- KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
24
- nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
16
+ c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
17
+ gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
18
+ 2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
19
+ fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
20
+ zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
21
+ EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
22
+ 65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
23
+ 2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
24
+ nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
25
25
  -----END CERTIFICATE-----
data/karafka-sidekiq-backend.gemspec CHANGED
@@ -9,15 +9,15 @@ Gem::Specification.new do |spec|
9
9
  spec.version = Karafka::Backends::Sidekiq::VERSION
10
10
  spec.platform = Gem::Platform::RUBY
11
11
  spec.authors = ['Maciej Mensfeld']
12
- spec.email = %w[maciej@coditsu.io]
12
+ spec.email = %w[maciej@mensfeld.pl]
13
13
  spec.homepage = 'https://github.com/karafka/karafka-sidekiq-backend'
14
14
  spec.summary = 'Karafka Sidekiq backend for background messages processing'
15
15
  spec.description = 'Karafka Sidekiq backend for background messages processing'
16
16
  spec.license = 'MIT'
17
17
 
18
- spec.add_dependency 'karafka', '~> 1.3.0.rc1'
18
+ spec.add_dependency 'karafka', '~> 1.4.0'
19
19
  spec.add_dependency 'sidekiq', '>= 4.2'
20
- spec.required_ruby_version = '>= 2.4.0'
20
+ spec.required_ruby_version = '>= 2.6.0'
21
21
 
22
22
  if $PROGRAM_NAME.end_with?('gem')
23
23
  spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
data/lib/karafka/backends/sidekiq.rb CHANGED
@@ -6,7 +6,7 @@ module Karafka
6
6
  # Sidekiq backend that schedules stuff to Sidekiq worker for delayed execution
7
7
  module Sidekiq
8
8
  # Karafka Sidekiq backend version
9
- VERSION = '1.3.0.rc1'
9
+ VERSION = '1.4.2'
10
10
 
11
11
  # Enqueues the execution of perform method into a worker.
12
12
  # @note Each worker needs to have a class #perform_async method that will allow us to pass
@@ -18,7 +18,7 @@ module Karafka
18
18
  topic.worker.perform_async(
19
19
  topic.id,
20
20
  topic.interchanger.encode(params_batch),
21
- respond_to?(:metadata) ? metadata : nil
21
+ respond_to?(:batch_metadata) ? batch_metadata.to_h : nil
22
22
  )
23
23
  end
24
24
  end
data/lib/karafka/base_worker.rb CHANGED
@@ -19,6 +19,7 @@ module Karafka
19
19
  # @param subclass [Class] subclass of the worker
20
20
  # @return [Class] subclass of the worker that was selected
21
21
  def inherited(subclass)
22
+ super
22
23
  @inherited ||= subclass
23
24
  end
24
25
  end
@@ -54,7 +55,7 @@ module Karafka
54
55
  )
55
56
 
56
57
  if topic.batch_fetching
57
- consumer.metadata = Params::Builders::Metadata.from_hash(
58
+ consumer.batch_metadata = Params::Builders::BatchMetadata.from_hash(
58
59
  metadata,
59
60
  topic
60
61
  )
data/lib/karafka/extensions/{metadata_builder.rb → batch_metadata_builder.rb} RENAMED
@@ -3,7 +3,7 @@
3
3
  module Karafka
4
4
  module Extensions
5
5
  # Extension for metadata builder to allow building metadata from a hash
6
- module MetadataBuilder
6
+ module BatchMetadataBuilder
7
7
  # Builds metadata from hash
8
8
  # @param hash [Hash] hash with metadata
9
9
  # @param topic [Karafka::Routing::Topic] topic instance
@@ -11,10 +11,12 @@ module Karafka
11
11
  def from_hash(hash, topic)
12
12
  # Parser needs to be merged as this is the only non-serializable object
13
13
  # so it gets reconstructed from the topic
14
- Karafka::Params::Metadata
15
- .new
16
- .merge!(hash)
17
- .merge!('deserializer' => topic.deserializer)
14
+ Karafka::Params::BatchMetadata
15
+ .new(
16
+ **hash
17
+ .merge('deserializer' => topic.deserializer)
18
+ .transform_keys(&:to_sym)
19
+ )
18
20
  end
19
21
  end
20
22
  end
data/lib/karafka/extensions/params_batch_builder.rb CHANGED
@@ -10,11 +10,11 @@ module Karafka
10
10
  # @return [Karafka::Params::ParamsBatch] built batch
11
11
  # @note We rebuild the params batch from array after the serialization
12
12
  def from_array(array, topic)
13
- params_array = array.map! do |hash|
13
+ params_array = array.map do |hash|
14
14
  Karafka::Params::Builders::Params.from_hash(hash, topic)
15
15
  end
16
16
 
17
- Karafka::Params::ParamsBatch.new(params_array)
17
+ Karafka::Params::ParamsBatch.new(params_array).freeze
18
18
  end
19
19
  end
20
20
  end
data/lib/karafka/extensions/params_builder.rb CHANGED
@@ -9,10 +9,15 @@ module Karafka
9
9
  # @param topic [Karafka::Routing::Topic] topic for which we build the params
10
10
  # @return [Karafka::Params::Params] built params
11
11
  def from_hash(hash, topic)
12
+ metadata = Karafka::Params::Metadata.new(
13
+ **hash
14
+ .fetch('metadata')
15
+ .merge('deserializer' => topic.deserializer)
16
+ .transform_keys(&:to_sym)
17
+ ).freeze
18
+
12
19
  Karafka::Params::Params
13
- .new
14
- .merge!(hash)
15
- .merge!('deserializer' => topic.deserializer)
20
+ .new(hash.fetch('raw_payload'), metadata)
16
21
  end
17
22
  end
18
23
  end
data/lib/karafka/extensions/stdout_listener.rb CHANGED
@@ -8,7 +8,7 @@ module Karafka
8
8
  # Logs info about scheduling of a certain dataset with a Sidekiq backend
9
9
  # @param event [Dry::Events::Event] event details including payload
10
10
  def on_backends_sidekiq_process(event)
11
- count = event[:caller].send(:params_batch).to_a.size
11
+ count = event[:caller].send(:params_batch).size
12
12
  topic = event[:caller].topic.name
13
13
  time = event[:time]
14
14
  info "Scheduling of #{count} messages to Sidekiq on topic #{topic} took #{time} ms"
@@ -17,7 +17,7 @@ module Karafka
17
17
  # Logs ino about processing certain events with a given Sidekiq worker
18
18
  # @param event [Dry::Events::Event] event details including payload
19
19
  def on_backends_sidekiq_base_worker_perform(event)
20
- count = event[:consumer].send(:params_batch).to_a.size
20
+ count = event[:consumer].send(:params_batch).size
21
21
  topic = event[:consumer].topic.name
22
22
  time = event[:time]
23
23
  info "Sidekiq processing of topic #{topic} with #{count} messages took #{time} ms"
data/lib/karafka/interchanger.rb CHANGED
@@ -11,10 +11,14 @@ module Karafka
11
11
  # This interchanger uses default Sidekiq options to exchange data
12
12
  class Interchanger
13
13
  # @param params_batch [Karafka::Params::ParamsBatch] Karafka params batch object
14
- # @return [Array<Karafka::Params::Params>] Array with hash/hashwithindiff values that will
15
- # be serialized using Sidekiq serialization engine
14
+ # @return [Array<Hash>] Array with hash built out of params data
16
15
  def encode(params_batch)
17
- params_batch.to_a
16
+ params_batch.map do |param|
17
+ {
18
+ 'raw_payload' => param.raw_payload,
19
+ 'metadata' => param.metadata.to_h
20
+ }
21
+ end
18
22
  end
19
23
 
20
24
  # @param params_batch [Array<Hash>] Sidekiq params that are now an array
data/lib/karafka_sidekiq_backend.rb CHANGED
@@ -16,7 +16,7 @@ Zeitwerk::Loader
16
16
 
17
17
  Karafka::Params::Builders::Params.extend(Karafka::Extensions::ParamsBuilder)
18
18
  Karafka::Params::Builders::ParamsBatch.extend(Karafka::Extensions::ParamsBatchBuilder)
19
- Karafka::Params::Builders::Metadata.extend(Karafka::Extensions::MetadataBuilder)
19
+ Karafka::Params::Builders::BatchMetadata.extend(Karafka::Extensions::BatchMetadataBuilder)
20
20
  Karafka::Routing::Topic.include(Karafka::Extensions::SidekiqTopicAttributes)
21
21
  Karafka::AttributesMap.prepend(Karafka::Extensions::SidekiqAttributesMap)
22
22
  Karafka::Instrumentation::StdoutListener.include(Karafka::Extensions::StdoutListener)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka-sidekiq-backend
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0.rc1
4
+ version: 1.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -11,30 +11,30 @@ cert_chain:
11
11
  - |
12
12
  -----BEGIN CERTIFICATE-----
13
13
  MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
14
- ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
15
- NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
16
- CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
17
- dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
18
- uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
19
- MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
20
- y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
21
- seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
22
- 94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
23
- TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
24
- Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
25
- BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
14
+ ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
15
+ NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
16
+ CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
17
+ 2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
18
+ N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
19
+ hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
20
+ sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
21
+ VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
22
+ tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
23
+ Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
24
+ LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
25
+ BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
26
26
  EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
27
- c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
28
- ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
29
- xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
30
- A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
31
- hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
32
- CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
33
- i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
34
- KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
35
- nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
27
+ c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
28
+ gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
29
+ 2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
30
+ fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
31
+ zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
32
+ EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
33
+ 65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
34
+ 2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
35
+ nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
36
36
  -----END CERTIFICATE-----
37
- date: 2019-07-31 00:00:00.000000000 Z
37
+ date: 2021-04-21 00:00:00.000000000 Z
38
38
  dependencies:
39
39
  - !ruby/object:Gem::Dependency
40
40
  name: karafka
@@ -42,14 +42,14 @@ dependencies:
42
42
  requirements:
43
43
  - - "~>"
44
44
  - !ruby/object:Gem::Version
45
- version: 1.3.0.rc1
45
+ version: 1.4.0
46
46
  type: :runtime
47
47
  prerelease: false
48
48
  version_requirements: !ruby/object:Gem::Requirement
49
49
  requirements:
50
50
  - - "~>"
51
51
  - !ruby/object:Gem::Version
52
- version: 1.3.0.rc1
52
+ version: 1.4.0
53
53
  - !ruby/object:Gem::Dependency
54
54
  name: sidekiq
55
55
  requirement: !ruby/object:Gem::Requirement
@@ -66,22 +66,23 @@ dependencies:
66
66
  version: '4.2'
67
67
  description: Karafka Sidekiq backend for background messages processing
68
68
  email:
69
- - maciej@coditsu.io
69
+ - maciej@mensfeld.pl
70
70
  executables: []
71
71
  extensions: []
72
72
  extra_rdoc_files: []
73
73
  files:
74
74
  - ".coditsu/ci.yml"
75
+ - ".diffend.yml"
75
76
  - ".github/FUNDING.yml"
77
+ - ".github/workflows/ci.yml"
76
78
  - ".gitignore"
77
79
  - ".rspec"
78
80
  - ".ruby-gemset"
79
81
  - ".ruby-version"
80
- - ".travis.yml"
81
82
  - CHANGELOG.md
82
83
  - Gemfile
83
84
  - Gemfile.lock
84
- - MIT-LICENCE
85
+ - MIT-LICENSE
85
86
  - README.md
86
87
  - Rakefile
87
88
  - certs/mensfeld.pem
@@ -91,7 +92,7 @@ files:
91
92
  - lib/karafka/base_worker.rb
92
93
  - lib/karafka/cli/worker.rb
93
94
  - lib/karafka/errors.rb
94
- - lib/karafka/extensions/metadata_builder.rb
95
+ - lib/karafka/extensions/batch_metadata_builder.rb
95
96
  - lib/karafka/extensions/params_batch_builder.rb
96
97
  - lib/karafka/extensions/params_builder.rb
97
98
  - lib/karafka/extensions/sidekiq_attributes_map.rb
@@ -112,14 +113,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
112
113
  requirements:
113
114
  - - ">="
114
115
  - !ruby/object:Gem::Version
115
- version: 2.4.0
116
+ version: 2.6.0
116
117
  required_rubygems_version: !ruby/object:Gem::Requirement
117
118
  requirements:
118
- - - ">"
119
+ - - ">="
119
120
  - !ruby/object:Gem::Version
120
- version: 1.3.1
121
+ version: '0'
121
122
  requirements: []
122
- rubygems_version: 3.0.3
123
+ rubygems_version: 3.2.15
123
124
  signing_key:
124
125
  specification_version: 4
125
126
  summary: Karafka Sidekiq backend for background messages processing
metadata.gz.sig CHANGED
Binary file
data/.travis.yml DELETED
@@ -1,37 +0,0 @@
1
- services:
2
- - docker
3
-
4
- dist: trusty
5
- cache: bundler
6
-
7
- git:
8
- depth: false
9
-
10
- test: &test
11
- stage: Test
12
- language: ruby
13
- before_install:
14
- - gem install bundler
15
- - gem update --system
16
- script: bundle exec rspec
17
-
18
- jobs:
19
- include:
20
- - <<: *test
21
- rvm: 2.6.3
22
- - <<: *test
23
- rvm: 2.5.5
24
- - <<: *test
25
- rvm: 2.4.6
26
-
27
- - stage: coditsu
28
- language: ruby
29
- rvm: 2.6.3
30
- before_install:
31
- - gem update --system
32
- - gem install bundler
33
- script: \curl -sSL https://api.coditsu.io/run/ci | bash
34
-
35
- stages:
36
- - coditsu
37
- - test