karafka-sidekiq-backend 1.2.0.beta4 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.coditsu/ci.yml +3 -0
- data/.diffend.yml +3 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +48 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +21 -2
- data/Gemfile +3 -2
- data/Gemfile.lock +96 -100
- data/LICENSE +165 -0
- data/README.md +41 -9
- data/certs/mensfeld.pem +25 -0
- data/karafka-sidekiq-backend.gemspec +9 -4
- data/lib/karafka-sidekiq-backend.rb +1 -32
- data/lib/karafka/backends/sidekiq.rb +4 -2
- data/lib/karafka/base_worker.rb +40 -6
- data/lib/karafka/extensions/batch_metadata_builder.rb +23 -0
- data/lib/karafka/extensions/params_batch_builder.rb +21 -0
- data/lib/karafka/extensions/params_builder.rb +24 -0
- data/lib/karafka/extensions/sidekiq_attributes_map.rb +1 -0
- data/lib/karafka/extensions/sidekiq_topic_attributes.rb +1 -5
- data/lib/karafka/extensions/stdout_listener.rb +27 -0
- data/lib/karafka/interchanger.rb +14 -18
- data/lib/karafka/workers/builder.rb +2 -9
- data/lib/karafka_sidekiq_backend.rb +28 -0
- metadata +50 -17
- metadata.gz.sig +2 -0
- data/.travis.yml +0 -18
- data/MIT-LICENCE +0 -18
- data/lib/karafka/instrumentation/listener.rb +0 -30
@@ -23,20 +23,13 @@ module Karafka
|
|
23
23
|
# build #=> Videos::NewVideosWorker
|
24
24
|
def build
|
25
25
|
return matcher.match if matcher.match
|
26
|
-
|
26
|
+
|
27
|
+
klass = Class.new(Karafka::BaseWorker.base_worker)
|
27
28
|
matcher.scope.const_set(matcher.name, klass)
|
28
29
|
end
|
29
30
|
|
30
31
|
private
|
31
32
|
|
32
|
-
# @return [Class] descendant of Karafka::BaseWorker from which all other workers
|
33
|
-
# should inherit
|
34
|
-
# @raise [Karafka::Errors::BaseWorkerDescentantMissing] raised when Karafka cannot detect
|
35
|
-
# direct Karafka::BaseWorker descendant from which it could build workers
|
36
|
-
def base
|
37
|
-
Karafka::BaseWorker.subclasses.first || raise(Errors::BaseWorkerDescentantMissing)
|
38
|
-
end
|
39
|
-
|
40
33
|
# @return [Karafka::Helpers::ClassMatcher] matcher instance for matching between consumer
|
41
34
|
# and appropriate worker
|
42
35
|
def matcher
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
%w[
|
4
|
+
karafka
|
5
|
+
sidekiq
|
6
|
+
].each(&method(:require))
|
7
|
+
|
8
|
+
require_relative 'karafka/errors'
|
9
|
+
|
10
|
+
Zeitwerk::Loader
|
11
|
+
.for_gem
|
12
|
+
.tap { |loader| loader.ignore("#{__dir__}/karafka_sidekiq_backend.rb") }
|
13
|
+
.tap { |loader| loader.ignore("#{__dir__}/karafka-sidekiq-backend.rb") }
|
14
|
+
.tap(&:setup)
|
15
|
+
.tap(&:eager_load)
|
16
|
+
|
17
|
+
Karafka::Params::Builders::Params.extend(Karafka::Extensions::ParamsBuilder)
|
18
|
+
Karafka::Params::Builders::ParamsBatch.extend(Karafka::Extensions::ParamsBatchBuilder)
|
19
|
+
Karafka::Params::Builders::BatchMetadata.extend(Karafka::Extensions::BatchMetadataBuilder)
|
20
|
+
Karafka::Routing::Topic.include(Karafka::Extensions::SidekiqTopicAttributes)
|
21
|
+
Karafka::AttributesMap.prepend(Karafka::Extensions::SidekiqAttributesMap)
|
22
|
+
Karafka::Instrumentation::StdoutListener.include(Karafka::Extensions::StdoutListener)
|
23
|
+
|
24
|
+
# Register internal events for instrumentation
|
25
|
+
%w[
|
26
|
+
backends.sidekiq.process
|
27
|
+
backends.sidekiq.base_worker.perform
|
28
|
+
].each(&Karafka.monitor.method(:register_event))
|
metadata
CHANGED
@@ -1,29 +1,55 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-sidekiq-backend
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
|
-
cert_chain:
|
11
|
-
|
10
|
+
cert_chain:
|
11
|
+
- |
|
12
|
+
-----BEGIN CERTIFICATE-----
|
13
|
+
MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
|
14
|
+
ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
|
15
|
+
NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
|
16
|
+
CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
|
17
|
+
2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
|
18
|
+
N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
|
19
|
+
hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
|
20
|
+
sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
|
21
|
+
VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
|
22
|
+
tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
|
23
|
+
Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
|
24
|
+
LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
|
25
|
+
BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
|
26
|
+
EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
|
27
|
+
c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
|
28
|
+
gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
|
29
|
+
2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
|
30
|
+
fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
|
31
|
+
zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
|
32
|
+
EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
|
33
|
+
65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
|
34
|
+
2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
|
35
|
+
nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
|
36
|
+
-----END CERTIFICATE-----
|
37
|
+
date: 2020-09-05 00:00:00.000000000 Z
|
12
38
|
dependencies:
|
13
39
|
- !ruby/object:Gem::Dependency
|
14
40
|
name: karafka
|
15
41
|
requirement: !ruby/object:Gem::Requirement
|
16
42
|
requirements:
|
17
|
-
- - "
|
43
|
+
- - "~>"
|
18
44
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.
|
45
|
+
version: 1.4.0.rc2
|
20
46
|
type: :runtime
|
21
47
|
prerelease: false
|
22
48
|
version_requirements: !ruby/object:Gem::Requirement
|
23
49
|
requirements:
|
24
|
-
- - "
|
50
|
+
- - "~>"
|
25
51
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.
|
52
|
+
version: 1.4.0.rc2
|
27
53
|
- !ruby/object:Gem::Dependency
|
28
54
|
name: sidekiq
|
29
55
|
requirement: !ruby/object:Gem::Requirement
|
@@ -40,36 +66,44 @@ dependencies:
|
|
40
66
|
version: '4.2'
|
41
67
|
description: Karafka Sidekiq backend for background messages processing
|
42
68
|
email:
|
43
|
-
- maciej@
|
69
|
+
- maciej@mensfeld.pl
|
44
70
|
executables: []
|
45
71
|
extensions: []
|
46
72
|
extra_rdoc_files: []
|
47
73
|
files:
|
74
|
+
- ".coditsu/ci.yml"
|
75
|
+
- ".diffend.yml"
|
76
|
+
- ".github/FUNDING.yml"
|
77
|
+
- ".github/workflows/ci.yml"
|
48
78
|
- ".gitignore"
|
49
79
|
- ".rspec"
|
50
80
|
- ".ruby-gemset"
|
51
81
|
- ".ruby-version"
|
52
|
-
- ".travis.yml"
|
53
82
|
- CHANGELOG.md
|
54
83
|
- Gemfile
|
55
84
|
- Gemfile.lock
|
56
|
-
-
|
85
|
+
- LICENSE
|
57
86
|
- README.md
|
58
87
|
- Rakefile
|
88
|
+
- certs/mensfeld.pem
|
59
89
|
- karafka-sidekiq-backend.gemspec
|
60
90
|
- lib/karafka-sidekiq-backend.rb
|
61
91
|
- lib/karafka/backends/sidekiq.rb
|
62
92
|
- lib/karafka/base_worker.rb
|
63
93
|
- lib/karafka/cli/worker.rb
|
64
94
|
- lib/karafka/errors.rb
|
95
|
+
- lib/karafka/extensions/batch_metadata_builder.rb
|
96
|
+
- lib/karafka/extensions/params_batch_builder.rb
|
97
|
+
- lib/karafka/extensions/params_builder.rb
|
65
98
|
- lib/karafka/extensions/sidekiq_attributes_map.rb
|
66
99
|
- lib/karafka/extensions/sidekiq_topic_attributes.rb
|
67
|
-
- lib/karafka/
|
100
|
+
- lib/karafka/extensions/stdout_listener.rb
|
68
101
|
- lib/karafka/interchanger.rb
|
69
102
|
- lib/karafka/workers/builder.rb
|
103
|
+
- lib/karafka_sidekiq_backend.rb
|
70
104
|
homepage: https://github.com/karafka/karafka-sidekiq-backend
|
71
105
|
licenses:
|
72
|
-
-
|
106
|
+
- LGPL-3.0
|
73
107
|
metadata: {}
|
74
108
|
post_install_message:
|
75
109
|
rdoc_options: []
|
@@ -79,15 +113,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
79
113
|
requirements:
|
80
114
|
- - ">="
|
81
115
|
- !ruby/object:Gem::Version
|
82
|
-
version: 2.
|
116
|
+
version: 2.5.0
|
83
117
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
84
118
|
requirements:
|
85
|
-
- - "
|
119
|
+
- - ">="
|
86
120
|
- !ruby/object:Gem::Version
|
87
|
-
version:
|
121
|
+
version: '0'
|
88
122
|
requirements: []
|
89
|
-
|
90
|
-
rubygems_version: 2.7.6
|
123
|
+
rubygems_version: 3.1.4
|
91
124
|
signing_key:
|
92
125
|
specification_version: 4
|
93
126
|
summary: Karafka Sidekiq backend for background messages processing
|
metadata.gz.sig
ADDED
data/.travis.yml
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
sudo: false
|
3
|
-
rvm:
|
4
|
-
- 2.3.0
|
5
|
-
- 2.3.1
|
6
|
-
- 2.3.2
|
7
|
-
- 2.3.3
|
8
|
-
- 2.3.4
|
9
|
-
- 2.4.0
|
10
|
-
- 2.4.1
|
11
|
-
- 2.4.2
|
12
|
-
- 2.5.0
|
13
|
-
- jruby-head
|
14
|
-
script: bundle exec rspec spec/
|
15
|
-
env:
|
16
|
-
global:
|
17
|
-
- JRUBY_OPTS='--debug'
|
18
|
-
install: bundle install --jobs=3 --retry=3
|
data/MIT-LICENCE
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
2
|
-
a copy of this software and associated documentation files (the
|
3
|
-
"Software"), to deal in the Software without restriction, including
|
4
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
5
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
6
|
-
permit persons to whom the Software is furnished to do so, subject to
|
7
|
-
the following conditions:
|
8
|
-
|
9
|
-
The above copyright notice and this permission notice shall be
|
10
|
-
included in all copies or substantial portions of the Software.
|
11
|
-
|
12
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
13
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
14
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
15
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
16
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
17
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
18
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
@@ -1,30 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
# Namespace for extensions of Karafka core framework instrumentation
|
5
|
-
module Instrumentation
|
6
|
-
# Additional methods for listener that listen on instrumentation related to the Sidekiq
|
7
|
-
# backend of Karafka
|
8
|
-
module Listener
|
9
|
-
class << self
|
10
|
-
# Logs info about scheduling of a certain dataset with a Sidekiq backend
|
11
|
-
# @param event [Dry::Events::Event] event details including payload
|
12
|
-
def on_backends_sidekiq_process(event)
|
13
|
-
count = event[:caller].send(:params_batch).to_a.size
|
14
|
-
topic = event[:caller].topic.name
|
15
|
-
time = event[:time]
|
16
|
-
info "Scheduling of #{count} messages to Sidekiq on topic #{topic} took #{time} ms"
|
17
|
-
end
|
18
|
-
|
19
|
-
# Logs ino about processing certain events with a given Sidekiq worker
|
20
|
-
# @param event [Dry::Events::Event] event details including payload
|
21
|
-
def on_backends_sidekiq_base_worker_perform(event)
|
22
|
-
count = event[:consumer].send(:params_batch).to_a.size
|
23
|
-
topic = event[:consumer].topic.name
|
24
|
-
time = event[:time]
|
25
|
-
info "Sidekiq processing of topic #{topic} with #{count} messages took #{time} ms"
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|