phobos 1.8.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +26 -0
- data/.rubocop_common.yml +29 -0
- data/.rubocop_todo.yml +7 -0
- data/.rubosync.yml +2 -0
- data/CHANGELOG.md +4 -0
- data/Gemfile +2 -0
- data/README.md +11 -7
- data/Rakefile +5 -3
- data/bin/console +3 -1
- data/bin/phobos +1 -0
- data/examples/handler_saving_events_database.rb +4 -2
- data/examples/handler_using_async_producer.rb +3 -1
- data/examples/publishing_messages_without_consumer.rb +8 -4
- data/lib/phobos.rb +61 -31
- data/lib/phobos/actions/process_batch.rb +3 -1
- data/lib/phobos/actions/process_message.rb +54 -31
- data/lib/phobos/cli.rb +2 -1
- data/lib/phobos/cli/runner.rb +3 -3
- data/lib/phobos/cli/start.rb +17 -25
- data/lib/phobos/constants.rb +33 -0
- data/lib/phobos/deep_struct.rb +13 -14
- data/lib/phobos/echo_handler.rb +2 -0
- data/lib/phobos/errors.rb +2 -0
- data/lib/phobos/executor.rb +39 -42
- data/lib/phobos/handler.rb +7 -7
- data/lib/phobos/instrumentation.rb +4 -2
- data/lib/phobos/listener.rb +81 -74
- data/lib/phobos/log.rb +23 -0
- data/lib/phobos/producer.rb +18 -14
- data/lib/phobos/test.rb +2 -0
- data/lib/phobos/test/helper.rb +4 -4
- data/lib/phobos/version.rb +3 -1
- data/phobos.gemspec +21 -14
- metadata +44 -10
data/lib/phobos/log.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Phobos
|
4
|
+
module Log
|
5
|
+
def log_info(msg, metadata = {})
|
6
|
+
LoggerHelper.log(:info, msg, metadata)
|
7
|
+
end
|
8
|
+
|
9
|
+
def log_debug(msg, metadata = {})
|
10
|
+
LoggerHelper.log(:debug, msg, metadata)
|
11
|
+
end
|
12
|
+
|
13
|
+
def log_error(msg, metadata)
|
14
|
+
LoggerHelper.log(:error, msg, metadata)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
module LoggerHelper
|
19
|
+
def self.log(method, msg, metadata)
|
20
|
+
Phobos.logger.send(method, Hash(message: msg).merge(metadata))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/phobos/producer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Phobos
|
2
4
|
module Producer
|
3
5
|
def self.included(base)
|
@@ -13,12 +15,12 @@ module Phobos
|
|
13
15
|
@host_obj = host_obj
|
14
16
|
end
|
15
17
|
|
16
|
-
def publish(topic, payload, key = nil)
|
17
|
-
class_producer.publish(topic, payload, key)
|
18
|
+
def publish(topic, payload, key = nil, partition_key = nil)
|
19
|
+
class_producer.publish(topic, payload, key, partition_key)
|
18
20
|
end
|
19
21
|
|
20
|
-
def async_publish(topic, payload, key = nil)
|
21
|
-
class_producer.async_publish(topic, payload, key)
|
22
|
+
def async_publish(topic, payload, key = nil, partition_key = nil)
|
23
|
+
class_producer.async_publish(topic, payload, key, partition_key)
|
22
24
|
end
|
23
25
|
|
24
26
|
# @param messages [Array(Hash(:topic, :payload, :key))]
|
@@ -49,7 +51,7 @@ module Phobos
|
|
49
51
|
|
50
52
|
class PublicAPI
|
51
53
|
NAMESPACE = :phobos_producer_store
|
52
|
-
ASYNC_PRODUCER_PARAMS =
|
54
|
+
ASYNC_PRODUCER_PARAMS = [:max_queue_size, :delivery_threshold, :delivery_interval].freeze
|
53
55
|
|
54
56
|
# This method configures the kafka client used with publish operations
|
55
57
|
# performed by the host class
|
@@ -65,8 +67,9 @@ module Phobos
|
|
65
67
|
producer_store[:kafka_client]
|
66
68
|
end
|
67
69
|
|
68
|
-
def publish(topic, payload, key = nil)
|
69
|
-
publish_list([{ topic: topic, payload: payload, key: key
|
70
|
+
def publish(topic, payload, key = nil, partition_key = nil)
|
71
|
+
publish_list([{ topic: topic, payload: payload, key: key,
|
72
|
+
partition_key: partition_key }])
|
70
73
|
end
|
71
74
|
|
72
75
|
def publish_list(messages)
|
@@ -88,8 +91,9 @@ module Phobos
|
|
88
91
|
producer_store[:async_producer]
|
89
92
|
end
|
90
93
|
|
91
|
-
def async_publish(topic, payload, key = nil)
|
92
|
-
async_publish_list([{ topic: topic, payload: payload, key: key
|
94
|
+
def async_publish(topic, payload, key = nil, partition_key = nil)
|
95
|
+
async_publish_list([{ topic: topic, payload: payload, key: key,
|
96
|
+
partition_key: partition_key }])
|
93
97
|
end
|
94
98
|
|
95
99
|
def async_publish_list(messages)
|
@@ -105,7 +109,7 @@ module Phobos
|
|
105
109
|
end
|
106
110
|
|
107
111
|
def regular_configs
|
108
|
-
Phobos.config.producer_hash.reject { |k, _| ASYNC_PRODUCER_PARAMS.include?(k)}
|
112
|
+
Phobos.config.producer_hash.reject { |k, _| ASYNC_PRODUCER_PARAMS.include?(k) }
|
109
113
|
end
|
110
114
|
|
111
115
|
def async_configs
|
@@ -116,16 +120,16 @@ module Phobos
|
|
116
120
|
|
117
121
|
def produce_messages(producer, messages)
|
118
122
|
messages.each do |message|
|
123
|
+
partition_key = message[:partition_key] || message[:key]
|
119
124
|
producer.produce(message[:payload], topic: message[:topic],
|
120
125
|
key: message[:key],
|
121
|
-
partition_key:
|
122
|
-
)
|
126
|
+
partition_key: partition_key)
|
123
127
|
end
|
124
128
|
end
|
125
129
|
|
126
130
|
def async_automatic_delivery?
|
127
|
-
async_configs.fetch(:delivery_threshold, 0)
|
128
|
-
async_configs.fetch(:delivery_interval, 0)
|
131
|
+
async_configs.fetch(:delivery_threshold, 0).positive? ||
|
132
|
+
async_configs.fetch(:delivery_interval, 0).positive?
|
129
133
|
end
|
130
134
|
|
131
135
|
def producer_store
|
data/lib/phobos/test.rb
CHANGED
data/lib/phobos/test/helper.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Phobos
|
2
4
|
module Test
|
3
5
|
module Helper
|
@@ -8,14 +10,12 @@ module Phobos
|
|
8
10
|
listener = Phobos::Listener.new(
|
9
11
|
handler: handler,
|
10
12
|
group_id: GROUP,
|
11
|
-
topic: TOPIC,
|
12
|
-
force_encoding: force_encoding
|
13
|
+
topic: TOPIC, force_encoding: force_encoding
|
13
14
|
)
|
14
15
|
|
15
16
|
message = Kafka::FetchedMessage.new(
|
16
17
|
message: Kafka::Protocol::Message.new(value: payload, key: nil, offset: 13),
|
17
|
-
topic: TOPIC,
|
18
|
-
partition: 0
|
18
|
+
topic: TOPIC, partition: 0
|
19
19
|
)
|
20
20
|
|
21
21
|
Phobos::Actions::ProcessMessage.new(
|
data/lib/phobos/version.rb
CHANGED
data/phobos.gemspec
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'phobos/version'
|
5
6
|
|
@@ -15,7 +16,7 @@ Gem::Specification.new do |spec|
|
|
15
16
|
'Francisco Juan',
|
16
17
|
'Tommy Gustafsson'
|
17
18
|
]
|
18
|
-
spec.email
|
19
|
+
spec.email = [
|
19
20
|
'ornelas.tulio@gmail.com',
|
20
21
|
'mathias.klippinge@gmail.com',
|
21
22
|
'sergey.evstifeev@gmail.com',
|
@@ -25,37 +26,43 @@ Gem::Specification.new do |spec|
|
|
25
26
|
'tommydgustafsson@gmail.com'
|
26
27
|
]
|
27
28
|
|
28
|
-
spec.summary =
|
29
|
-
spec.description =
|
29
|
+
spec.summary = 'Simplifying Kafka for ruby apps'
|
30
|
+
spec.description = 'Phobos is a microframework and library for kafka based applications, '\
|
31
|
+
'it wraps common behaviors needed by consumers/producers in an easy an convenient API. '\
|
32
|
+
'It uses ruby-kafka as its kafka client and core component.'
|
30
33
|
spec.homepage = 'https://github.com/klarna/phobos'
|
31
34
|
spec.license = 'Apache License Version 2.0'
|
32
35
|
|
33
36
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
34
37
|
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
35
|
-
|
36
|
-
|
37
|
-
else
|
38
|
-
raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
|
38
|
+
unless spec.respond_to?(:metadata)
|
39
|
+
raise('RubyGems 2.0 or newer is required to protect against public gem pushes.')
|
39
40
|
end
|
40
41
|
|
41
|
-
spec.
|
42
|
+
spec.metadata['allowed_push_host'] = 'https://rubygems.org'
|
43
|
+
|
44
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
45
|
+
f.match(%r{^(test|spec|features)/})
|
46
|
+
end
|
42
47
|
spec.bindir = 'bin'
|
43
48
|
spec.executables = spec.files.grep(%r{^bin/phobos}) { |f| File.basename(f) }
|
44
49
|
spec.require_paths = ['lib']
|
45
50
|
spec.required_ruby_version = '>= 2.3'
|
46
51
|
|
47
52
|
spec.add_development_dependency 'bundler'
|
53
|
+
spec.add_development_dependency 'pry-byebug'
|
48
54
|
spec.add_development_dependency 'rake', '~> 12.3'
|
49
55
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
50
|
-
spec.add_development_dependency '
|
56
|
+
spec.add_development_dependency 'rubocop', '0.60.0'
|
57
|
+
spec.add_development_dependency 'rubocop_rules'
|
51
58
|
spec.add_development_dependency 'simplecov'
|
52
59
|
spec.add_development_dependency 'timecop'
|
53
60
|
|
54
|
-
spec.add_dependency '
|
61
|
+
spec.add_dependency 'activesupport', '>= 3.0.0'
|
55
62
|
spec.add_dependency 'concurrent-ruby', '>= 1.0.2'
|
56
63
|
spec.add_dependency 'concurrent-ruby-ext', '>= 1.0.2'
|
57
|
-
spec.add_dependency 'activesupport', '>= 3.0.0'
|
58
|
-
spec.add_dependency 'logging'
|
59
64
|
spec.add_dependency 'exponential-backoff'
|
65
|
+
spec.add_dependency 'logging'
|
66
|
+
spec.add_dependency 'ruby-kafka'
|
60
67
|
spec.add_dependency 'thor'
|
61
68
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: phobos
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.8.
|
4
|
+
version: 1.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Túlio Ornelas
|
@@ -14,7 +14,7 @@ authors:
|
|
14
14
|
autorequire:
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date: 2018-
|
17
|
+
date: 2018-11-23 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: bundler
|
@@ -30,6 +30,20 @@ dependencies:
|
|
30
30
|
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '0'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: pry-byebug
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
40
|
+
type: :development
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
name: rake
|
35
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -59,7 +73,21 @@ dependencies:
|
|
59
73
|
- !ruby/object:Gem::Version
|
60
74
|
version: '3.0'
|
61
75
|
- !ruby/object:Gem::Dependency
|
62
|
-
name:
|
76
|
+
name: rubocop
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - '='
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 0.60.0
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - '='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 0.60.0
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: rubocop_rules
|
63
91
|
requirement: !ruby/object:Gem::Requirement
|
64
92
|
requirements:
|
65
93
|
- - ">="
|
@@ -101,19 +129,19 @@ dependencies:
|
|
101
129
|
- !ruby/object:Gem::Version
|
102
130
|
version: '0'
|
103
131
|
- !ruby/object:Gem::Dependency
|
104
|
-
name:
|
132
|
+
name: activesupport
|
105
133
|
requirement: !ruby/object:Gem::Requirement
|
106
134
|
requirements:
|
107
135
|
- - ">="
|
108
136
|
- !ruby/object:Gem::Version
|
109
|
-
version: 0.
|
137
|
+
version: 3.0.0
|
110
138
|
type: :runtime
|
111
139
|
prerelease: false
|
112
140
|
version_requirements: !ruby/object:Gem::Requirement
|
113
141
|
requirements:
|
114
142
|
- - ">="
|
115
143
|
- !ruby/object:Gem::Version
|
116
|
-
version: 0.
|
144
|
+
version: 3.0.0
|
117
145
|
- !ruby/object:Gem::Dependency
|
118
146
|
name: concurrent-ruby
|
119
147
|
requirement: !ruby/object:Gem::Requirement
|
@@ -143,19 +171,19 @@ dependencies:
|
|
143
171
|
- !ruby/object:Gem::Version
|
144
172
|
version: 1.0.2
|
145
173
|
- !ruby/object:Gem::Dependency
|
146
|
-
name:
|
174
|
+
name: exponential-backoff
|
147
175
|
requirement: !ruby/object:Gem::Requirement
|
148
176
|
requirements:
|
149
177
|
- - ">="
|
150
178
|
- !ruby/object:Gem::Version
|
151
|
-
version:
|
179
|
+
version: '0'
|
152
180
|
type: :runtime
|
153
181
|
prerelease: false
|
154
182
|
version_requirements: !ruby/object:Gem::Requirement
|
155
183
|
requirements:
|
156
184
|
- - ">="
|
157
185
|
- !ruby/object:Gem::Version
|
158
|
-
version:
|
186
|
+
version: '0'
|
159
187
|
- !ruby/object:Gem::Dependency
|
160
188
|
name: logging
|
161
189
|
requirement: !ruby/object:Gem::Requirement
|
@@ -171,7 +199,7 @@ dependencies:
|
|
171
199
|
- !ruby/object:Gem::Version
|
172
200
|
version: '0'
|
173
201
|
- !ruby/object:Gem::Dependency
|
174
|
-
name:
|
202
|
+
name: ruby-kafka
|
175
203
|
requirement: !ruby/object:Gem::Requirement
|
176
204
|
requirements:
|
177
205
|
- - ">="
|
@@ -218,6 +246,10 @@ files:
|
|
218
246
|
- ".env"
|
219
247
|
- ".gitignore"
|
220
248
|
- ".rspec"
|
249
|
+
- ".rubocop.yml"
|
250
|
+
- ".rubocop_common.yml"
|
251
|
+
- ".rubocop_todo.yml"
|
252
|
+
- ".rubosync.yml"
|
221
253
|
- ".ruby-version"
|
222
254
|
- ".travis.yml"
|
223
255
|
- CHANGELOG.md
|
@@ -240,6 +272,7 @@ files:
|
|
240
272
|
- lib/phobos/cli.rb
|
241
273
|
- lib/phobos/cli/runner.rb
|
242
274
|
- lib/phobos/cli/start.rb
|
275
|
+
- lib/phobos/constants.rb
|
243
276
|
- lib/phobos/deep_struct.rb
|
244
277
|
- lib/phobos/echo_handler.rb
|
245
278
|
- lib/phobos/errors.rb
|
@@ -247,6 +280,7 @@ files:
|
|
247
280
|
- lib/phobos/handler.rb
|
248
281
|
- lib/phobos/instrumentation.rb
|
249
282
|
- lib/phobos/listener.rb
|
283
|
+
- lib/phobos/log.rb
|
250
284
|
- lib/phobos/producer.rb
|
251
285
|
- lib/phobos/test.rb
|
252
286
|
- lib/phobos/test/helper.rb
|