karafka-sidekiq-backend 1.0.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +68 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +16 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +11 -0
- data/Gemfile.lock +140 -0
- data/MIT-LICENCE +18 -0
- data/README.md +114 -0
- data/Rakefile +3 -0
- data/karafka-sidekiq-backend.gemspec +25 -0
- data/lib/karafka/backends/sidekiq.rb +25 -0
- data/lib/karafka/base_worker.rb +27 -0
- data/lib/karafka/cli/worker.rb +28 -0
- data/lib/karafka/errors.rb +10 -0
- data/lib/karafka/extensions/sidekiq_attributes_map.rb +21 -0
- data/lib/karafka/extensions/sidekiq_topic_attributes.rb +25 -0
- data/lib/karafka/interchanger.rb +33 -0
- data/lib/karafka/workers/builder.rb +51 -0
- data/lib/karafka-sidekiq-backend.rb +20 -0
- metadata +93 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: fb7035c44bd2a5c8ed71073da1b0ea7ccf962027
|
4
|
+
data.tar.gz: be6d7a538ce48a5334ae4f8f5cc249ad8053446c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8d4cc805a0715bb35a54c5b7a960010ac56bab5883b32eed70f6f8d08b4edba543473a6c695bc79548234484bf079b4c295caecaf7370024d27a07633fa22ce7
|
7
|
+
data.tar.gz: 64fd8c8dd2257cc8b4a7738ef09031c70aaae6141d6016b82dd1d2ae422ff3b1904a01cb314ca383358220ffbd6e032468902c550890efd60da27ae2eaae15dc
|
data/.gitignore
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
# bundler state
|
2
|
+
/.bundle
|
3
|
+
/vendor/bundle/
|
4
|
+
/vendor/ruby/
|
5
|
+
/ruby/
|
6
|
+
app.god
|
7
|
+
|
8
|
+
# minimal Rails specific artifacts
|
9
|
+
db/*.sqlite3
|
10
|
+
/log/development.log
|
11
|
+
/log/production.log
|
12
|
+
/log/test.log
|
13
|
+
/tmp/*
|
14
|
+
*.gem
|
15
|
+
*.~
|
16
|
+
|
17
|
+
# various artifacts
|
18
|
+
**.war
|
19
|
+
*.rbc
|
20
|
+
*.sassc
|
21
|
+
.byebug_history
|
22
|
+
.redcar/
|
23
|
+
.capistrano/
|
24
|
+
.sass-cache
|
25
|
+
/config/god/sidekiq.rb
|
26
|
+
/config/puma.rb
|
27
|
+
/coverage.data
|
28
|
+
/coverage/
|
29
|
+
/doc/api/
|
30
|
+
/doc/app/
|
31
|
+
/doc/yard
|
32
|
+
/doc/features.html
|
33
|
+
/doc/specs.html
|
34
|
+
/spec/tmp/*
|
35
|
+
/cache
|
36
|
+
/capybara*
|
37
|
+
/capybara-*.html
|
38
|
+
/gems
|
39
|
+
/specifications
|
40
|
+
rerun.txt
|
41
|
+
pickle-email-*.html
|
42
|
+
|
43
|
+
# If you find yourself ignoring temporary files generated by your text editor
|
44
|
+
# or operating system, you probably want to add a global ignore instead:
|
45
|
+
# git config --global core.excludesfile ~/.gitignore_global
|
46
|
+
#
|
47
|
+
# Here are some files you may want to ignore globally:
|
48
|
+
|
49
|
+
# scm revert files
|
50
|
+
**.orig
|
51
|
+
|
52
|
+
# Mac finder artifacts
|
53
|
+
.DS_Store
|
54
|
+
|
55
|
+
# Netbeans project directory
|
56
|
+
/nbproject
|
57
|
+
|
58
|
+
# RubyMine project files
|
59
|
+
.idea
|
60
|
+
|
61
|
+
# Textmate project files
|
62
|
+
/*.tmproj
|
63
|
+
|
64
|
+
# vim artifacts
|
65
|
+
**.swp
|
66
|
+
|
67
|
+
# documentation
|
68
|
+
.yardoc
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper
|
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
karafka-sidekiq-backend
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.4.1
|
data/.travis.yml
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
language: ruby
|
2
|
+
sudo: false
|
3
|
+
rvm:
|
4
|
+
- 2.3.0
|
5
|
+
- 2.3.1
|
6
|
+
- 2.3.2
|
7
|
+
- 2.3.3
|
8
|
+
- 2.3.4
|
9
|
+
- 2.4.0
|
10
|
+
- 2.4.1
|
11
|
+
- jruby-9.1.12.0
|
12
|
+
script: bundle exec rspec spec/
|
13
|
+
env:
|
14
|
+
global:
|
15
|
+
- JRUBY_OPTS='--debug'
|
16
|
+
install: bundle install --jobs=3 --retry=3
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,140 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
karafka-sidekiq-backend (1.0.0.pre1)
|
5
|
+
karafka (>= 1.0.0.pre1)
|
6
|
+
sidekiq (>= 4.2)
|
7
|
+
|
8
|
+
GEM
|
9
|
+
remote: https://rubygems.org/
|
10
|
+
specs:
|
11
|
+
activesupport (5.1.3)
|
12
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
13
|
+
i18n (~> 0.7)
|
14
|
+
minitest (~> 5.1)
|
15
|
+
tzinfo (~> 1.1)
|
16
|
+
celluloid (0.17.3)
|
17
|
+
celluloid-essentials
|
18
|
+
celluloid-extras
|
19
|
+
celluloid-fsm
|
20
|
+
celluloid-pool
|
21
|
+
celluloid-supervision
|
22
|
+
timers (>= 4.1.1)
|
23
|
+
celluloid-essentials (0.20.5)
|
24
|
+
timers (>= 4.1.1)
|
25
|
+
celluloid-extras (0.20.5)
|
26
|
+
timers (>= 4.1.1)
|
27
|
+
celluloid-fsm (0.20.5)
|
28
|
+
timers (>= 4.1.1)
|
29
|
+
celluloid-pool (0.20.5)
|
30
|
+
timers (>= 4.1.1)
|
31
|
+
celluloid-supervision (0.20.6)
|
32
|
+
timers (>= 4.1.1)
|
33
|
+
concurrent-ruby (1.0.5)
|
34
|
+
connection_pool (2.2.1)
|
35
|
+
diff-lcs (1.3)
|
36
|
+
docile (1.1.5)
|
37
|
+
dry-configurable (0.7.0)
|
38
|
+
concurrent-ruby (~> 1.0)
|
39
|
+
dry-container (0.6.0)
|
40
|
+
concurrent-ruby (~> 1.0)
|
41
|
+
dry-configurable (~> 0.1, >= 0.1.3)
|
42
|
+
dry-core (0.3.3)
|
43
|
+
concurrent-ruby (~> 1.0)
|
44
|
+
dry-equalizer (0.2.0)
|
45
|
+
dry-logic (0.4.1)
|
46
|
+
dry-container (~> 0.2, >= 0.2.6)
|
47
|
+
dry-core (~> 0.2)
|
48
|
+
dry-equalizer (~> 0.2)
|
49
|
+
dry-types (0.11.1)
|
50
|
+
concurrent-ruby (~> 1.0)
|
51
|
+
dry-configurable (~> 0.1)
|
52
|
+
dry-container (~> 0.3)
|
53
|
+
dry-core (~> 0.2, >= 0.2.1)
|
54
|
+
dry-equalizer (~> 0.2)
|
55
|
+
dry-logic (~> 0.4, >= 0.4.0)
|
56
|
+
inflecto (~> 0.0.0, >= 0.0.2)
|
57
|
+
dry-validation (0.11.0)
|
58
|
+
concurrent-ruby (~> 1.0)
|
59
|
+
dry-configurable (~> 0.1, >= 0.1.3)
|
60
|
+
dry-core (~> 0.2, >= 0.2.1)
|
61
|
+
dry-equalizer (~> 0.2)
|
62
|
+
dry-logic (~> 0.4, >= 0.4.0)
|
63
|
+
dry-types (~> 0.11.0)
|
64
|
+
envlogic (1.0.4)
|
65
|
+
activesupport
|
66
|
+
hitimes (1.2.6)
|
67
|
+
i18n (0.8.6)
|
68
|
+
inflecto (0.0.2)
|
69
|
+
json (2.1.0)
|
70
|
+
karafka (1.0.0.rc1)
|
71
|
+
activesupport (>= 5.0)
|
72
|
+
celluloid
|
73
|
+
dry-configurable (~> 0.7)
|
74
|
+
dry-validation (~> 0.11)
|
75
|
+
envlogic (~> 1.0)
|
76
|
+
multi_json (>= 1.12)
|
77
|
+
rake (>= 11.3)
|
78
|
+
require_all (>= 1.4)
|
79
|
+
ruby-kafka (>= 0.4)
|
80
|
+
thor (~> 0.19)
|
81
|
+
waterdrop (>= 0.4)
|
82
|
+
minitest (5.10.3)
|
83
|
+
multi_json (1.12.1)
|
84
|
+
null-logger (0.1.4)
|
85
|
+
rack (2.0.3)
|
86
|
+
rack-protection (2.0.0)
|
87
|
+
rack
|
88
|
+
rake (12.0.0)
|
89
|
+
redis (3.3.3)
|
90
|
+
require_all (1.4.0)
|
91
|
+
rspec (3.6.0)
|
92
|
+
rspec-core (~> 3.6.0)
|
93
|
+
rspec-expectations (~> 3.6.0)
|
94
|
+
rspec-mocks (~> 3.6.0)
|
95
|
+
rspec-core (3.6.0)
|
96
|
+
rspec-support (~> 3.6.0)
|
97
|
+
rspec-expectations (3.6.0)
|
98
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
99
|
+
rspec-support (~> 3.6.0)
|
100
|
+
rspec-mocks (3.6.0)
|
101
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
102
|
+
rspec-support (~> 3.6.0)
|
103
|
+
rspec-support (3.6.0)
|
104
|
+
ruby-kafka (0.4.1)
|
105
|
+
sidekiq (5.0.4)
|
106
|
+
concurrent-ruby (~> 1.0)
|
107
|
+
connection_pool (~> 2.2, >= 2.2.0)
|
108
|
+
rack-protection (>= 1.5.0)
|
109
|
+
redis (~> 3.3, >= 3.3.3)
|
110
|
+
simplecov (0.15.0)
|
111
|
+
docile (~> 1.1.0)
|
112
|
+
json (>= 1.8, < 3)
|
113
|
+
simplecov-html (~> 0.10.0)
|
114
|
+
simplecov-html (0.10.2)
|
115
|
+
thor (0.20.0)
|
116
|
+
thread_safe (0.3.6)
|
117
|
+
timecop (0.9.1)
|
118
|
+
timers (4.1.2)
|
119
|
+
hitimes
|
120
|
+
tzinfo (1.2.3)
|
121
|
+
thread_safe (~> 0.1)
|
122
|
+
waterdrop (0.4.0)
|
123
|
+
bundler
|
124
|
+
connection_pool
|
125
|
+
dry-configurable (~> 0.6)
|
126
|
+
null-logger
|
127
|
+
rake
|
128
|
+
ruby-kafka (~> 0.4)
|
129
|
+
|
130
|
+
PLATFORMS
|
131
|
+
ruby
|
132
|
+
|
133
|
+
DEPENDENCIES
|
134
|
+
karafka-sidekiq-backend!
|
135
|
+
rspec
|
136
|
+
simplecov
|
137
|
+
timecop
|
138
|
+
|
139
|
+
BUNDLED WITH
|
140
|
+
1.15.4
|
data/MIT-LICENCE
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
2
|
+
a copy of this software and associated documentation files (the
|
3
|
+
"Software"), to deal in the Software without restriction, including
|
4
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
5
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
6
|
+
permit persons to whom the Software is furnished to do so, subject to
|
7
|
+
the following conditions:
|
8
|
+
|
9
|
+
The above copyright notice and this permission notice shall be
|
10
|
+
included in all copies or substantial portions of the Software.
|
11
|
+
|
12
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
13
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
14
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
15
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
16
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
17
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
18
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
# Karafka Sidekiq Backend
|
2
|
+
|
3
|
+
[![Build Status](https://travis-ci.org/karafka/karafka-sidekiq-backend.png)](https://travis-ci.org/karafka/karafka-sidekiq-backend)
|
4
|
+
|
5
|
+
Karafka Sidekiq Backend provides support for processing received Kafka messages inside of Sidekiq workers.
|
6
|
+
|
7
|
+
## Installations
|
8
|
+
|
9
|
+
Add this to your gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'karafka-sidekiq-backend'
|
13
|
+
```
|
14
|
+
|
15
|
+
and create a file called ```application_worker.rb``` inside of your ```app/workers``` directory, that looks like that:
|
16
|
+
|
17
|
+
```ruby
|
18
|
+
class ApplicationWorker < Karafka::BaseWorker
|
19
|
+
end
|
20
|
+
```
|
21
|
+
|
22
|
+
and you are ready to go. Karafka Sidekiq Backend integrates with Karafka automatically
|
23
|
+
|
24
|
+
Note: You can name your application worker base class with any name you want. The only thing that is required is a direct inheritance from the ```Karafka::BaseWorker``` class.
|
25
|
+
|
26
|
+
## Usage
|
27
|
+
|
28
|
+
If you want to process messages with Sidekiq backend, you need to tell this to Karafka.
|
29
|
+
|
30
|
+
To do so, you can either configure that in a configuration block:
|
31
|
+
|
32
|
+
```ruby
|
33
|
+
class App < Karafka::App
|
34
|
+
setup do |config|
|
35
|
+
config.backend = :sidekiq
|
36
|
+
# Other config options...
|
37
|
+
end
|
38
|
+
end
|
39
|
+
```
|
40
|
+
|
41
|
+
or on a per topic level:
|
42
|
+
|
43
|
+
```ruby
|
44
|
+
App.routes.draw do
|
45
|
+
consumer_group :videos_consumer do
|
46
|
+
topic :binary_video_details do
|
47
|
+
controller Videos::DetailsController
|
48
|
+
worker Workers::DetailsWorker
|
49
|
+
interchanger Interchangers::Binary
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
```
|
54
|
+
|
55
|
+
You don't need to do anything beyond that. Karafka will know, that you want to run your controllers ```#perform``` method in a background job.
|
56
|
+
|
57
|
+
## Configuration
|
58
|
+
|
59
|
+
There are two options you can set inside of the ```topic``` block:
|
60
|
+
|
61
|
+
| Option | Value type | Description |
|
62
|
+
|--------------|------------|-------------------------------------------------------------------------------------------------------------------|
|
63
|
+
| worker | Class | Name of a worker class that we want to use to schedule perform code |
|
64
|
+
| interchanger | Class | Name of a parser class that we want to use to parse incoming data |
|
65
|
+
|
66
|
+
|
67
|
+
### Workers
|
68
|
+
|
69
|
+
Karafka by default will build a worker that will correspond to each of your controllers (so you will have a pair - controller and a worker). All of them will inherit from ```ApplicationWorker``` and will share all its settings.
|
70
|
+
|
71
|
+
To run Sidekiq you should have sidekiq.yml file in *config* folder. The example of ```sidekiq.yml``` file will be generated to config/sidekiq.yml.example once you run ```bundle exec karafka install```.
|
72
|
+
|
73
|
+
However, if you want to use a raw Sidekiq worker (without any Karafka additional magic), or you want to use SidekiqPro (or any other queuing engine that has the same API as Sidekiq), you can assign your own custom worker:
|
74
|
+
|
75
|
+
```ruby
|
76
|
+
topic :incoming_messages do
|
77
|
+
controller MessagesController
|
78
|
+
worker MyCustomWorker
|
79
|
+
end
|
80
|
+
```
|
81
|
+
|
82
|
+
Note that even then, you need to specify a controller that will schedule a background task.
|
83
|
+
|
84
|
+
Custom workers need to provide a ```#perform_async``` method. It needs to accept two arguments:
|
85
|
+
|
86
|
+
- ```topic_id``` - first argument is a current topic id from which a given message comes
|
87
|
+
- ```params``` - all the params that came from Kafka + additional metadata. This data format might be changed if you use custom interchangers. Otherwise it will be an instance of Karafka::Params::Params.
|
88
|
+
|
89
|
+
Keep in mind, that params might be in two states: parsed or unparsed when passed to #perform_async. This means, that if you use custom interchangers and/or custom workers, you might want to look into Karafka's sources to see exactly how it works.
|
90
|
+
|
91
|
+
### Interchangers
|
92
|
+
|
93
|
+
Custom interchangers target issues with non-standard (binary, etc.) data that we want to store when we do ```#perform_async```. This data might be corrupted when fetched in a worker (see [this](https://github.com/karafka/karafka/issues/30) issue). With custom interchangers, you can encode/compress data before it is being passed to scheduling and decode/decompress it when it gets into the worker.
|
94
|
+
|
95
|
+
**Warning**: if you decide to use slow interchangers, they might significantly slow down Karafka.
|
96
|
+
|
97
|
+
```ruby
|
98
|
+
class Base64Interchanger
|
99
|
+
class << self
|
100
|
+
def load(params)
|
101
|
+
Base64.encode64(Marshal.dump(params))
|
102
|
+
end
|
103
|
+
|
104
|
+
def parse(params)
|
105
|
+
Marshal.load(Base64.decode64(params))
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
topic :binary_video_details do
|
111
|
+
controller Videos::DetailsController
|
112
|
+
interchanger Base64Interchanger
|
113
|
+
end
|
114
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require 'karafka/backends/sidekiq'
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = 'karafka-sidekiq-backend'
|
9
|
+
spec.version = Karafka::Backends::Sidekiq::VERSION
|
10
|
+
spec.platform = Gem::Platform::RUBY
|
11
|
+
spec.authors = ['Maciej Mensfeld']
|
12
|
+
spec.email = %w[maciej@coditsu.io]
|
13
|
+
spec.homepage = 'https://github.com/karafka/karafka-sidekiq-backend'
|
14
|
+
spec.summary = 'Karafka Sidekiq backend for background messages processing'
|
15
|
+
spec.description = 'Karafka Sidekiq backend for background messages processing'
|
16
|
+
spec.license = 'MIT'
|
17
|
+
|
18
|
+
spec.add_dependency 'karafka', '>= 1.0.0.pre1'
|
19
|
+
spec.add_dependency 'sidekiq', '>= 4.2'
|
20
|
+
spec.required_ruby_version = '>= 2.3.0'
|
21
|
+
|
22
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
|
23
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
24
|
+
spec.require_paths = %w[lib]
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Backends
|
5
|
+
# Sidekiq backend that schedules stuff to Sidekiq worker for delayed execution
|
6
|
+
module Sidekiq
|
7
|
+
VERSION = '1.0.0.pre1'
|
8
|
+
|
9
|
+
private
|
10
|
+
|
11
|
+
# Enqueues the execution of perform method into a worker.
|
12
|
+
# @note Each worker needs to have a class #perform_async method that will allow us to pass
|
13
|
+
# parameters into it. We always pass topic as a first argument and this request
|
14
|
+
# params_batch as a second one (we pass topic to be able to build back the controller
|
15
|
+
# in the worker)
|
16
|
+
def process
|
17
|
+
Karafka.monitor.notice(self.class, params_batch)
|
18
|
+
topic.worker.perform_async(
|
19
|
+
topic.id,
|
20
|
+
topic.interchanger.load(params_batch.to_a)
|
21
|
+
)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Worker wrapper for Sidekiq workers
|
5
|
+
class BaseWorker
|
6
|
+
include Sidekiq::Worker
|
7
|
+
|
8
|
+
# Executes the logic that lies in #perform Karafka controller method
|
9
|
+
# @param topic_id [String] Unique topic id that we will use to find a proper topic
|
10
|
+
# @param params_batch [Array] Array with messages batch
|
11
|
+
def perform(topic_id, params_batch)
|
12
|
+
Karafka.monitor.notice(self.class, params_batch)
|
13
|
+
controller(topic_id, params_batch).perform
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
# @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
|
19
|
+
# with params_batch assigned already (controller is ready to use)
|
20
|
+
def controller(topic_id, params_batch)
|
21
|
+
topic = Karafka::Routing::Router.find(topic_id)
|
22
|
+
controller = topic.controller.new
|
23
|
+
controller.params_batch = topic.interchanger.parse(params_batch)
|
24
|
+
controller
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Karafka framework Cli
|
5
|
+
class Cli < Thor
|
6
|
+
# Worker Karafka Cli action
|
7
|
+
class Worker < Base
|
8
|
+
desc 'Start the Karafka Sidekiq worker (short-cut alias: "w")'
|
9
|
+
option aliases: 'w'
|
10
|
+
|
11
|
+
# Start the Karafka Sidekiq worker
|
12
|
+
# @param params [Array<String>] additional params that will be passed to sidekiq, that way we
|
13
|
+
# can override any default Karafka settings
|
14
|
+
def call(*params)
|
15
|
+
puts 'Starting Karafka worker'
|
16
|
+
config = "-C #{Karafka::App.root.join('config/sidekiq.yml')}"
|
17
|
+
req = "-r #{Karafka.boot_file}"
|
18
|
+
env = "-e #{Karafka.env}"
|
19
|
+
|
20
|
+
cli.info
|
21
|
+
|
22
|
+
cmd = "bundle exec sidekiq #{env} #{req} #{config} #{params.join(' ')}"
|
23
|
+
puts(cmd)
|
24
|
+
exec(cmd)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Namespace used to encapsulate all the internal errors of Karafka
|
5
|
+
module Errors
|
6
|
+
# Raised when application does not have ApplicationWorker or other class that directly
|
7
|
+
# inherits from Karafka::BaseWorker
|
8
|
+
BaseWorkerDescentantMissing = Class.new(BaseError)
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Extensions
|
5
|
+
module SidekiqAttributesMap
|
6
|
+
module ClassMethods
|
7
|
+
def topic
|
8
|
+
super + %i[interchanger worker]
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.prepended(base)
|
13
|
+
class << base
|
14
|
+
prepend ClassMethods
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
Karafka::AttributesMap.prepend Karafka::Extensions::SidekiqAttributesMap
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Karafka
|
2
|
+
module Extensions
|
3
|
+
module SidekiqTopicAttributes
|
4
|
+
# @return [Class] Class (not an instance) of a worker that should be used to schedule the
|
5
|
+
# background job
|
6
|
+
# @note If not provided - will be built based on the provided controller
|
7
|
+
def worker
|
8
|
+
@worker ||= backend == :sidekiq ? Karafka::Workers::Builder.new(controller).build : nil
|
9
|
+
end
|
10
|
+
|
11
|
+
# @return [Class] Interchanger class (not an instance) that we want to use to interchange
|
12
|
+
# params between Karafka server and Karafka background job
|
13
|
+
def interchanger
|
14
|
+
@interchanger ||= Karafka::Interchanger
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.included(base)
|
18
|
+
base.send :attr_writer, :worker
|
19
|
+
base.send :attr_writer, :interchanger
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
Karafka::Routing::Topic.include Karafka::Extensions::SidekiqTopicAttributes
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Interchangers allow us to format/encode/pack data that is being send to perform_async
|
5
|
+
# This is meant to target mostly issues with data encoding like this one:
|
6
|
+
# https://github.com/mperham/sidekiq/issues/197
|
7
|
+
# Each custom interchanger should implement following methods:
|
8
|
+
# - load - it is meant to encode params before they get stored inside Redis
|
9
|
+
# - parse - decoded params back to a hash format that we can use
|
10
|
+
class Interchanger
|
11
|
+
class << self
|
12
|
+
# @param params [Karafka::Params::Params] Karafka params object
|
13
|
+
# @note Params might not be parsed because of lazy loading feature. If you implement your
|
14
|
+
# own interchanger logic, this method needs to return data that can be converted to
|
15
|
+
# json with default Sidekiqs logic
|
16
|
+
# @return [Karafka::Params::Params] same as input. We assume that our incoming data is
|
17
|
+
# jsonable-safe and we can rely on a direct Sidekiq encoding logic
|
18
|
+
def load(params)
|
19
|
+
params
|
20
|
+
end
|
21
|
+
|
22
|
+
# @param params [Hash] Sidekiqs params that are now a Hash (after they were JSON#parse)
|
23
|
+
# @note Hash is what we need to build Karafka::Params::Params, so we do nothing
|
24
|
+
# with it. If you implement your own interchanger logic, this method needs to return
|
25
|
+
# a hash with appropriate data that will be used to build Karafka::Params::Params
|
26
|
+
# @return [Hash] We return exactly what we received. We rely on sidekiqs default
|
27
|
+
# interchanging format
|
28
|
+
def parse(params)
|
29
|
+
params
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Internal stuff related to workers
|
5
|
+
module Workers
|
6
|
+
# Builder is used to check if there is a proper controller with the same name as
|
7
|
+
# a controller and if not, it will create a default one using Karafka::BaseWorker
|
8
|
+
# This is used as a building layer between controllers and workers. it will be only used
|
9
|
+
# when user does not provide his own worker that should perform controller stuff
|
10
|
+
class Builder
|
11
|
+
# @param controller_class [Karafka::BaseController] descendant of Karafka::BaseController
|
12
|
+
# @example Create a worker builder
|
13
|
+
# Karafka::Workers::Builder.new(SuperController)
|
14
|
+
def initialize(controller_class)
|
15
|
+
@controller_class = controller_class
|
16
|
+
end
|
17
|
+
|
18
|
+
# @return [Class] Sidekiq worker class that already exists or new build based
|
19
|
+
# on the provided controller_class name
|
20
|
+
# @example Controller: SuperController
|
21
|
+
# build #=> SuperWorker
|
22
|
+
# @example Controller: Videos::NewVideosController
|
23
|
+
# build #=> Videos::NewVideosWorker
|
24
|
+
def build
|
25
|
+
return matcher.match if matcher.match
|
26
|
+
klass = Class.new(base)
|
27
|
+
matcher.scope.const_set(matcher.name, klass)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
# @return [Class] descendant of Karafka::BaseWorker from which all other workers
|
33
|
+
# should inherit
|
34
|
+
# @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when Karafka cannot detect
|
35
|
+
# direct Karafka::BaseWorker descendant from which it could build workers
|
36
|
+
def base
|
37
|
+
Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
|
38
|
+
end
|
39
|
+
|
40
|
+
# @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between controller
|
41
|
+
# and appropriate worker
|
42
|
+
def matcher
|
43
|
+
@matcher ||= Helpers::ClassMatcher.new(
|
44
|
+
@controller_class,
|
45
|
+
from: 'Controller',
|
46
|
+
to: 'Worker'
|
47
|
+
)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
%w[
|
4
|
+
sidekiq
|
5
|
+
].each(&method(:require))
|
6
|
+
|
7
|
+
module Karafka
|
8
|
+
module Backends
|
9
|
+
module Sidekiq
|
10
|
+
class << self
|
11
|
+
# @return [String] path to Karafka gem root core
|
12
|
+
def core_root
|
13
|
+
Pathname.new(File.expand_path('../karafka', __FILE__))
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
Karafka::Loader.load!(Karafka::Backends::Sidekiq.core_root)
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: karafka-sidekiq-backend
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0.pre1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Maciej Mensfeld
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-09-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: karafka
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.0.0.pre1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.0.0.pre1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: sidekiq
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '4.2'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '4.2'
|
41
|
+
description: Karafka Sidekiq backend for background messages processing
|
42
|
+
email:
|
43
|
+
- maciej@coditsu.io
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- ".gitignore"
|
49
|
+
- ".rspec"
|
50
|
+
- ".ruby-gemset"
|
51
|
+
- ".ruby-version"
|
52
|
+
- ".travis.yml"
|
53
|
+
- CHANGELOG.md
|
54
|
+
- Gemfile
|
55
|
+
- Gemfile.lock
|
56
|
+
- MIT-LICENCE
|
57
|
+
- README.md
|
58
|
+
- Rakefile
|
59
|
+
- karafka-sidekiq-backend.gemspec
|
60
|
+
- lib/karafka-sidekiq-backend.rb
|
61
|
+
- lib/karafka/backends/sidekiq.rb
|
62
|
+
- lib/karafka/base_worker.rb
|
63
|
+
- lib/karafka/cli/worker.rb
|
64
|
+
- lib/karafka/errors.rb
|
65
|
+
- lib/karafka/extensions/sidekiq_attributes_map.rb
|
66
|
+
- lib/karafka/extensions/sidekiq_topic_attributes.rb
|
67
|
+
- lib/karafka/interchanger.rb
|
68
|
+
- lib/karafka/workers/builder.rb
|
69
|
+
homepage: https://github.com/karafka/karafka-sidekiq-backend
|
70
|
+
licenses:
|
71
|
+
- MIT
|
72
|
+
metadata: {}
|
73
|
+
post_install_message:
|
74
|
+
rdoc_options: []
|
75
|
+
require_paths:
|
76
|
+
- lib
|
77
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 2.3.0
|
82
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">"
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: 1.3.1
|
87
|
+
requirements: []
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 2.6.13
|
90
|
+
signing_key:
|
91
|
+
specification_version: 4
|
92
|
+
summary: Karafka Sidekiq backend for background messages processing
|
93
|
+
test_files: []
|