manageiq-messaging 0.1.5 → 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.codeclimate.yml +25 -29
- data/.rubocop.yml +3 -3
- data/.rubocop_cc.yml +3 -4
- data/.travis.yml +2 -3
- data/CHANGES +17 -0
- data/README.md +14 -2
- data/lib/manageiq/messaging.rb +1 -0
- data/lib/manageiq/messaging/client.rb +7 -4
- data/lib/manageiq/messaging/common.rb +5 -1
- data/lib/manageiq/messaging/kafka/client.rb +31 -31
- data/lib/manageiq/messaging/kafka/common.rb +33 -49
- data/lib/manageiq/messaging/kafka/queue.rb +11 -25
- data/lib/manageiq/messaging/kafka/topic.rb +12 -14
- data/lib/manageiq/messaging/stomp/queue.rb +1 -0
- data/lib/manageiq/messaging/stomp/topic.rb +6 -1
- data/lib/manageiq/messaging/version.rb +1 -1
- data/manageiq-messaging.gemspec +8 -8
- metadata +32 -41
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 64efdc1c23146264a24fb6464a2d72480cacd3db28ebc729394ac511d82dbec4
|
4
|
+
data.tar.gz: bb21c6264ee6c05c1d5ea6151f49fe81a156b2f2f9180682a2a14a5597047969
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6f8823df33b9cf386e2ff0188e6df0aa1a28bfb90b6b81b6e6112cc1249429cbc2548601ebf81e2970635535fb4c81b76559e203210b4a97d7a4c6513dad84a8
|
7
|
+
data.tar.gz: 93ec75aa96126b9938318f107e0b789aa5a02f557137668d563506abf24e3d78af333f334318ea114e2bfcc4e1ebdfe9a2659f5d81319a914b9c31a476ad87e4
|
data/.codeclimate.yml
CHANGED
@@ -1,18 +1,21 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
-
|
4
|
-
|
5
|
-
-
|
6
|
-
|
7
|
-
-
|
8
|
-
|
9
|
-
-
|
10
|
-
|
1
|
+
prepare:
|
2
|
+
fetch:
|
3
|
+
- url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_base.yml
|
4
|
+
path: ".rubocop_base.yml"
|
5
|
+
- url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_cc_base.yml
|
6
|
+
path: ".rubocop_cc_base.yml"
|
7
|
+
- url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/base.yml
|
8
|
+
path: styles/base.yml
|
9
|
+
- url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/cc_base.yml
|
10
|
+
path: styles/cc_base.yml
|
11
|
+
plugins:
|
12
|
+
rubocop:
|
13
|
+
enabled: true
|
14
|
+
config: ".rubocop_cc.yml"
|
15
|
+
channel: rubocop-0-82
|
11
16
|
brakeman:
|
12
|
-
# very slow :sad_panda:
|
13
17
|
enabled: false
|
14
18
|
bundler-audit:
|
15
|
-
# requires Gemfile.lock
|
16
19
|
enabled: false
|
17
20
|
csslint:
|
18
21
|
enabled: false
|
@@ -24,24 +27,17 @@ engines:
|
|
24
27
|
- javascript
|
25
28
|
eslint:
|
26
29
|
enabled: false
|
27
|
-
channel:
|
30
|
+
channel: eslint-3
|
28
31
|
fixme:
|
29
|
-
# let's enable later
|
30
32
|
enabled: false
|
31
33
|
markdownlint:
|
32
|
-
# let's enable later
|
33
34
|
enabled: false
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
ratings:
|
44
|
-
paths:
|
45
|
-
- Gemfile.lock
|
46
|
-
- "**.rake"
|
47
|
-
- "**.rb"
|
35
|
+
exclude_patterns:
|
36
|
+
- ".git/"
|
37
|
+
- "**.xml"
|
38
|
+
- "**.yaml"
|
39
|
+
- "**.yml"
|
40
|
+
- locale/
|
41
|
+
- spec/
|
42
|
+
- tools/
|
43
|
+
version: '2'
|
data/.rubocop.yml
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
+
inherit_gem:
|
2
|
+
manageiq-style: ".rubocop_base.yml"
|
1
3
|
inherit_from:
|
2
|
-
-
|
3
|
-
# put all local rubocop config into .rubocop_local.yml as it will be loaded by .rubocop_cc.yml as well
|
4
|
-
- .rubocop_local.yml
|
4
|
+
- ".rubocop_local.yml"
|
data/.rubocop_cc.yml
CHANGED
data/.travis.yml
CHANGED
data/CHANGES
CHANGED
@@ -14,3 +14,20 @@
|
|
14
14
|
|
15
15
|
= 0.1.5 - 6-Jun-2019
|
16
16
|
* Allow caller to provide extra headers to the message
|
17
|
+
|
18
|
+
= 0.1.6 - 6-July-2020
|
19
|
+
* Rescue message body decoding errors. Re-raise errors raised by users code of processing received messages.
|
20
|
+
|
21
|
+
= 1.0.0 - 28-Sep-2020
|
22
|
+
* Switch to use rdkafka client
|
23
|
+
|
24
|
+
= 1.0.1 - 15-Dec-2020
|
25
|
+
* Allow all kafka options to be passed
|
26
|
+
* remove sudo:false from .travis.yml
|
27
|
+
* Use manageiq-style
|
28
|
+
|
29
|
+
= 1.0.2 - 4-Jan-2021
|
30
|
+
* Rails 6.0 Support
|
31
|
+
|
32
|
+
= 1.0.3 - 12-May-2021
|
33
|
+
* Allow bulk publish of messages to a topic
|
data/README.md
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
# ManageIQ Messaging Client
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/manageiq-messaging.svg)](http://badge.fury.io/rb/manageiq-messaging)
|
4
|
-
[![Build Status](https://travis-ci.
|
4
|
+
[![Build Status](https://travis-ci.com/ManageIQ/manageiq-messaging.svg)](https://travis-ci.com/ManageIQ/manageiq-messaging)
|
5
5
|
[![Code Climate](https://codeclimate.com/github/ManageIQ/manageiq-messaging.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging)
|
6
6
|
[![Test Coverage](https://codeclimate.com/github/ManageIQ/manageiq-messaging/badges/coverage.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging/coverage)
|
7
|
-
[![Dependency Status](https://gemnasium.com/ManageIQ/manageiq-messaging.svg)](https://gemnasium.com/ManageIQ/manageiq-messaging)
|
8
7
|
[![Security](https://hakiri.io/github/ManageIQ/manageiq-messaging/master.svg)](https://hakiri.io/github/ManageIQ/manageiq-messaging/master)
|
9
8
|
|
10
9
|
Client library for ManageIQ components to exchange messages through its internal message bus.
|
@@ -156,6 +155,19 @@ This is the one-to-many publish/subscribe pattern. Multiple subscribers can subs
|
|
156
155
|
|
157
156
|
By default, events are delivered to live subscribers only. Some messaging systems support persistence with options.
|
158
157
|
|
158
|
+
### Publish bulk messages to a topic
|
159
|
+
|
160
|
+
Often it is more efficient to publish messages in bulk rather than one-at-a-time. To do this you can pass an array of messages to the `publish_topic` API:
|
161
|
+
|
162
|
+
```ruby
|
163
|
+
client.publish_topic(
|
164
|
+
[
|
165
|
+
{:service => 'provider_events', :event => 'powered_off', :payload => {:ems_ref => 'uid987', :timestamp => '1501091391'}},
|
166
|
+
{:service => 'provider_events', :event => 'powered_on', :payload => {:ems_ref => 'uid987', :timestamp => '1501091429'}},
|
167
|
+
]
|
168
|
+
)
|
169
|
+
```
|
170
|
+
|
159
171
|
### Add your own headers to a message (Queue or Topic)
|
160
172
|
|
161
173
|
If you want you can add in your own headers to the send message
|
data/lib/manageiq/messaging.rb
CHANGED
@@ -40,6 +40,7 @@ module ManageIQ
|
|
40
40
|
def self.open(options)
|
41
41
|
protocol = options[:protocol] || :Stomp
|
42
42
|
client = Object.const_get("ManageIQ::Messaging::#{protocol}::Client").new(options)
|
43
|
+
|
43
44
|
return client unless block_given?
|
44
45
|
|
45
46
|
begin
|
@@ -160,7 +161,8 @@ module ManageIQ
|
|
160
161
|
end
|
161
162
|
|
162
163
|
# Publish a message as a topic. All subscribers will receive a copy of the message.
|
163
|
-
#
|
164
|
+
# +messages+ can be either a hash or an array of hashes.
|
165
|
+
# Expected keys are:
|
164
166
|
# * :service (service is used to determine the topic address)
|
165
167
|
# * :event (event name)
|
166
168
|
# * :payload (message body, a string or an user object that can be serialized)
|
@@ -168,10 +170,11 @@ module ManageIQ
|
|
168
170
|
# * :headers (optional, additional headers to add to the message)
|
169
171
|
# Other options are underlying messaging system specific.
|
170
172
|
#
|
171
|
-
def publish_topic(
|
172
|
-
|
173
|
+
def publish_topic(messages)
|
174
|
+
messages = Array.wrap(messages)
|
175
|
+
messages.each { |msg| assert_options(msg, [:event, :service]) }
|
173
176
|
|
174
|
-
publish_topic_impl(
|
177
|
+
publish_topic_impl(messages)
|
175
178
|
end
|
176
179
|
|
177
180
|
# Subscribe to receive topic type messages.
|
@@ -18,7 +18,7 @@ module ManageIQ
|
|
18
18
|
|
19
19
|
def decode_body(headers, raw_body)
|
20
20
|
return raw_body unless headers.kind_of?(Hash)
|
21
|
-
case headers["encoding"]
|
21
|
+
case headers["encoding"] || headers[:encoding]
|
22
22
|
when "json"
|
23
23
|
JSON.parse(raw_body)
|
24
24
|
when "yaml"
|
@@ -26,6 +26,10 @@ module ManageIQ
|
|
26
26
|
else
|
27
27
|
raw_body
|
28
28
|
end
|
29
|
+
rescue => e # JSON or YAML parsing error
|
30
|
+
logger.error("Error decoding message body: #{e.message}")
|
31
|
+
logger.error(e.backtrace.join("\n"))
|
32
|
+
raw_body
|
29
33
|
end
|
30
34
|
|
31
35
|
def payload_log(payload)
|
@@ -10,16 +10,11 @@ module ManageIQ
|
|
10
10
|
# * :hosts (Array of Kafka cluster hosts, or)
|
11
11
|
# * :host (Single host name)
|
12
12
|
# * :port (host port number)
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
# * :sasl_plain_username
|
19
|
-
# * :sasl_plain_password
|
20
|
-
# * :sasl_scram_username
|
21
|
-
# * :sasl_scram_password
|
22
|
-
# * :sasl_scram_mechanism
|
13
|
+
#
|
14
|
+
# For additional security options, please refer to
|
15
|
+
# https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka and
|
16
|
+
# https://github.com/edenhill/librdkafka/wiki/Using-SASL-with-librdkafka
|
17
|
+
#
|
23
18
|
#
|
24
19
|
# Kafka specific +publish_message+ options:
|
25
20
|
# * :group_name (Used as Kafka partition_key)
|
@@ -42,7 +37,7 @@ module ManageIQ
|
|
42
37
|
#
|
43
38
|
# +subscribe_background_job+ is currently not implemented.
|
44
39
|
class Client < ManageIQ::Messaging::Client
|
45
|
-
require '
|
40
|
+
require 'rdkafka'
|
46
41
|
require 'manageiq/messaging/kafka/common'
|
47
42
|
require 'manageiq/messaging/kafka/queue'
|
48
43
|
require 'manageiq/messaging/kafka/background_job'
|
@@ -53,27 +48,27 @@ module ManageIQ
|
|
53
48
|
include BackgroundJob
|
54
49
|
include Topic
|
55
50
|
|
56
|
-
private *delegate(:subscribe, :unsubscribe, :publish, :to => :kafka_client)
|
57
|
-
delegate :close, :to => :kafka_client
|
58
|
-
|
59
51
|
attr_accessor :encoding
|
60
52
|
|
61
53
|
def ack(ack_ref)
|
62
|
-
|
63
|
-
|
54
|
+
ack_ref.commit
|
55
|
+
rescue Rdkafka::RdkafkaError => e
|
56
|
+
logger.warn("ack failed with error #{e.message}")
|
57
|
+
raise unless e.message =~ /no_offset/
|
64
58
|
end
|
65
59
|
|
66
60
|
def close
|
67
|
-
@
|
68
|
-
@topic_consumer = nil
|
69
|
-
@queue_consumer.try(:stop)
|
70
|
-
@queue_consumer = nil
|
71
|
-
|
72
|
-
@producer.try(:shutdown)
|
61
|
+
@producer&.close
|
73
62
|
@producer = nil
|
74
63
|
|
75
|
-
|
76
|
-
@
|
64
|
+
@consumer&.close
|
65
|
+
@consumer = nil
|
66
|
+
end
|
67
|
+
|
68
|
+
# list all topics
|
69
|
+
def topics
|
70
|
+
native_kafka = producer.instance_variable_get(:@native_kafka)
|
71
|
+
Rdkafka::Metadata.new(native_kafka).topics.collect { |topic| topic[:topic_name] }
|
77
72
|
end
|
78
73
|
|
79
74
|
private
|
@@ -81,18 +76,23 @@ module ManageIQ
|
|
81
76
|
attr_reader :kafka_client
|
82
77
|
|
83
78
|
def initialize(options)
|
84
|
-
hosts = Array(options[:hosts] || options[:host])
|
85
|
-
hosts.collect! { |host| "#{host}:#{options[:port]}" }
|
86
|
-
|
87
79
|
@encoding = options[:encoding] || 'yaml'
|
88
80
|
require "json" if @encoding == "json"
|
89
81
|
|
90
|
-
|
91
|
-
|
82
|
+
::Rdkafka::Config.logger = logger
|
83
|
+
@kafka_client = ::Rdkafka::Config.new(rdkafka_connection_opts(options))
|
84
|
+
end
|
85
|
+
|
86
|
+
def rdkafka_connection_opts(options)
|
87
|
+
hosts = Array(options[:hosts] || options[:host])
|
88
|
+
hosts.collect! { |host| "#{host}:#{options[:port]}" }
|
92
89
|
|
93
|
-
|
90
|
+
result = {:"bootstrap.servers" => hosts.join(',')}
|
91
|
+
result[:"client.id"] = options[:client_ref] if options[:client_ref]
|
92
|
+
result[:"sasl.username"] = options[:username] if options[:username]
|
93
|
+
result[:"sasl.password"] = options[:password] if options[:password]
|
94
94
|
|
95
|
-
|
95
|
+
result.merge(options.except(:port, :host, :hosts, :encoding, :protocol, :client_ref, :username, :password))
|
96
96
|
end
|
97
97
|
end
|
98
98
|
end
|
@@ -5,48 +5,25 @@ module ManageIQ
|
|
5
5
|
require 'manageiq/messaging/common'
|
6
6
|
include ManageIQ::Messaging::Common
|
7
7
|
|
8
|
-
GROUP_FOR_QUEUE_MESSAGES = 'manageiq_messaging_queue_group_'.freeze
|
9
|
-
|
10
8
|
private
|
11
9
|
|
12
10
|
def producer
|
13
11
|
@producer ||= kafka_client.producer
|
14
12
|
end
|
15
13
|
|
16
|
-
def
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
@topic_consumer ||= kafka_client.consumer(consumer_opts)
|
26
|
-
end
|
27
|
-
|
28
|
-
def queue_consumer(topic, session_timeout = nil)
|
29
|
-
# all queue consumers join the same group so that each message can be processed by one and only one consumer
|
30
|
-
@queue_consumer.try(:stop) unless @queue_topic == topic
|
31
|
-
@queue_topic = topic
|
32
|
-
|
33
|
-
consumer_opts = {:group_id => GROUP_FOR_QUEUE_MESSAGES + topic}
|
34
|
-
consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
|
35
|
-
|
36
|
-
@queue_consumer ||= kafka_client.consumer(consumer_opts)
|
37
|
-
end
|
38
|
-
|
39
|
-
trap("TERM") do
|
40
|
-
@topic_consumer.try(:stop)
|
41
|
-
@topic_consumer = nil
|
42
|
-
@queue_consumer.try(:stop)
|
43
|
-
@queue_consumer = nil
|
14
|
+
def consumer(beginning, options)
|
15
|
+
@consumer&.close
|
16
|
+
kafka_client[:"group.id"] = options[:persist_ref]
|
17
|
+
kafka_client[:"auto.offset.reset"] = beginning ? 'smallest' : 'largest'
|
18
|
+
kafka_client[:"enable.auto.commit"] = !!auto_ack?(options)
|
19
|
+
kafka_client[:"session.timeout.ms"] = options[:session_timeout] * 1000 if options[:session_timeout].present?
|
20
|
+
kafka_client[:"group.instance.id"] = options[:group_instance_id] if options[:group_instance_id].present?
|
21
|
+
@consumer = kafka_client.consumer
|
44
22
|
end
|
45
23
|
|
46
|
-
def raw_publish(
|
47
|
-
|
48
|
-
producer.
|
49
|
-
logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
|
24
|
+
def raw_publish(body, options)
|
25
|
+
options[:payload] = encode_body(options[:headers], body)
|
26
|
+
producer.produce(options)
|
50
27
|
end
|
51
28
|
|
52
29
|
def queue_for_publish(options)
|
@@ -68,8 +45,8 @@ module ManageIQ
|
|
68
45
|
|
69
46
|
def for_publish(options)
|
70
47
|
kafka_opts = {:topic => address(options)}
|
71
|
-
kafka_opts[:partition_key]
|
72
|
-
kafka_opts[:headers]
|
48
|
+
kafka_opts[:partition_key] = options[:group_name] if options[:group_name]
|
49
|
+
kafka_opts[:headers] = {}
|
73
50
|
kafka_opts[:headers][:sender] = options[:sender] if options[:sender]
|
74
51
|
|
75
52
|
body = options[:payload] || ''
|
@@ -85,23 +62,30 @@ module ManageIQ
|
|
85
62
|
end
|
86
63
|
end
|
87
64
|
|
88
|
-
def process_queue_message(queue, message)
|
89
|
-
|
90
|
-
|
91
|
-
|
65
|
+
def process_queue_message(queue_consumer, queue, message)
|
66
|
+
begin
|
67
|
+
payload = decode_body(message.headers, message.payload)
|
68
|
+
sender, message_type, _class_name = parse_message_headers(message.headers)
|
69
|
+
client_headers = message.headers.except(*message_header_keys).with_indifferent_access
|
92
70
|
|
93
|
-
|
94
|
-
|
71
|
+
logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
|
72
|
+
yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, client_headers, queue_consumer, self)]
|
73
|
+
logger.info("Messsage processed")
|
74
|
+
rescue StandardError => e
|
75
|
+
logger.error("Message processing error: #{e.message}")
|
76
|
+
logger.error(e.backtrace.join("\n"))
|
77
|
+
raise
|
78
|
+
end
|
95
79
|
end
|
96
80
|
|
97
|
-
def process_topic_message(topic, message)
|
81
|
+
def process_topic_message(topic_consumer, topic, message)
|
98
82
|
begin
|
99
|
-
payload = decode_body(message.headers, message.
|
83
|
+
payload = decode_body(message.headers, message.payload)
|
100
84
|
sender, event_type = parse_event_headers(message.headers)
|
101
|
-
client_headers = message.headers.except(*event_header_keys)
|
85
|
+
client_headers = message.headers.except(*event_header_keys).with_indifferent_access
|
102
86
|
|
103
87
|
logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
|
104
|
-
yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers,
|
88
|
+
yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers, topic_consumer, self)
|
105
89
|
logger.info("Event processed")
|
106
90
|
rescue StandardError => e
|
107
91
|
logger.error("Event processing error: #{e.message}")
|
@@ -111,7 +95,7 @@ module ManageIQ
|
|
111
95
|
end
|
112
96
|
|
113
97
|
def message_header_keys
|
114
|
-
[
|
98
|
+
[:sender, :message_type, :class_name]
|
115
99
|
end
|
116
100
|
|
117
101
|
def parse_message_headers(headers)
|
@@ -120,12 +104,12 @@ module ManageIQ
|
|
120
104
|
end
|
121
105
|
|
122
106
|
def event_header_keys
|
123
|
-
[
|
107
|
+
[:sender, :event_type]
|
124
108
|
end
|
125
109
|
|
126
110
|
def parse_event_headers(headers)
|
127
111
|
return [nil, nil] unless headers.kind_of?(Hash)
|
128
|
-
headers.values_at(
|
112
|
+
headers.values_at(*event_header_keys)
|
129
113
|
end
|
130
114
|
end
|
131
115
|
end
|
@@ -2,42 +2,28 @@ module ManageIQ
|
|
2
2
|
module Messaging
|
3
3
|
module Kafka
|
4
4
|
module Queue
|
5
|
+
GROUP_FOR_QUEUE_MESSAGES = ENV['QUEUE_MESSAGES_GROUP_PREFIX'].freeze || 'manageiq_messaging_queue_group_'.freeze
|
6
|
+
|
5
7
|
private
|
6
8
|
|
7
9
|
def publish_message_impl(options)
|
8
10
|
raise ArgumentError, "Kafka messaging implementation does not take a block" if block_given?
|
9
|
-
raw_publish(
|
11
|
+
raw_publish(*queue_for_publish(options)).wait
|
10
12
|
end
|
11
13
|
|
12
14
|
def publish_messages_impl(messages)
|
13
|
-
messages.
|
14
|
-
|
15
|
+
handles = messages.collect { |msg_options| raw_publish(*queue_for_publish(msg_options)) }
|
16
|
+
handles.each(&:wait)
|
15
17
|
end
|
16
18
|
|
17
|
-
def subscribe_messages_impl(options)
|
19
|
+
def subscribe_messages_impl(options, &block)
|
18
20
|
topic = address(options)
|
19
|
-
|
20
|
-
|
21
|
-
batch_options = {}
|
22
|
-
batch_options[:automatically_mark_as_processed] = auto_ack?(options)
|
23
|
-
batch_options[:max_bytes] = options[:max_bytes] if options.key?(:max_bytes)
|
24
|
-
|
25
|
-
consumer = queue_consumer(topic, session_timeout)
|
26
|
-
consumer.subscribe(topic)
|
27
|
-
consumer.each_batch(batch_options) do |batch|
|
28
|
-
logger.info("Batch message received: queue(#{topic})")
|
29
|
-
begin
|
30
|
-
messages = batch.messages.collect do |message|
|
31
|
-
sender, message_type, _class_name, payload = process_queue_message(topic, message)
|
32
|
-
ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, headers, message, self)
|
33
|
-
end
|
21
|
+
options[:persist_ref] = GROUP_FOR_QUEUE_MESSAGES + topic
|
34
22
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
end
|
40
|
-
logger.info("Batch message processed")
|
23
|
+
queue_consumer = consumer(true, options)
|
24
|
+
queue_consumer.subscribe(topic)
|
25
|
+
queue_consumer.each do |message|
|
26
|
+
process_queue_message(queue_consumer, topic, message, &block)
|
41
27
|
end
|
42
28
|
end
|
43
29
|
end
|
@@ -1,28 +1,26 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
1
3
|
module ManageIQ
|
2
4
|
module Messaging
|
3
5
|
module Kafka
|
4
6
|
module Topic
|
7
|
+
GROUP_FOR_ADHOC_LISTENERS = Socket.gethostname.freeze
|
8
|
+
|
5
9
|
private
|
6
10
|
|
7
|
-
def publish_topic_impl(
|
8
|
-
raw_publish(
|
11
|
+
def publish_topic_impl(messages)
|
12
|
+
handles = messages.collect { |message| raw_publish(*topic_for_publish(message)) }
|
13
|
+
handles.each(&:wait)
|
9
14
|
end
|
10
15
|
|
11
16
|
def subscribe_topic_impl(options, &block)
|
12
17
|
topic = address(options)
|
13
|
-
persist_ref = options[:persist_ref]
|
14
|
-
session_timeout = options[:session_timeout]
|
15
18
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
else
|
23
|
-
kafka_client.each_message(:topic => topic, :start_from_beginning => false) do |message|
|
24
|
-
process_topic_message(topic, message, &block)
|
25
|
-
end
|
19
|
+
options[:persist_ref] = "#{GROUP_FOR_ADHOC_LISTENERS}_#{Time.now.to_i}" unless options[:persist_ref]
|
20
|
+
topic_consumer = consumer(false, options)
|
21
|
+
topic_consumer.subscribe(topic)
|
22
|
+
topic_consumer.each do |message|
|
23
|
+
process_topic_message(topic_consumer, topic, message, &block)
|
26
24
|
end
|
27
25
|
end
|
28
26
|
end
|
@@ -4,7 +4,7 @@ module ManageIQ
|
|
4
4
|
module Topic
|
5
5
|
private
|
6
6
|
|
7
|
-
def
|
7
|
+
def publish_topic_single(options)
|
8
8
|
address, headers = topic_for_publish(options)
|
9
9
|
headers[:sender] = options[:sender] if options[:sender]
|
10
10
|
headers[:event_type] = options[:event] if options[:event]
|
@@ -12,6 +12,10 @@ module ManageIQ
|
|
12
12
|
raw_publish(address, options[:payload], headers)
|
13
13
|
end
|
14
14
|
|
15
|
+
def publish_topic_impl(messages)
|
16
|
+
messages.each { |message| publish_topic_single(message) }
|
17
|
+
end
|
18
|
+
|
15
19
|
def subscribe_topic_impl(options)
|
16
20
|
queue_name, headers = topic_for_subscribe(options)
|
17
21
|
|
@@ -30,6 +34,7 @@ module ManageIQ
|
|
30
34
|
rescue => e
|
31
35
|
logger.error("Event processing error: #{e.message}")
|
32
36
|
logger.error(e.backtrace.join("\n"))
|
37
|
+
raise
|
33
38
|
end
|
34
39
|
end
|
35
40
|
end
|
data/manageiq-messaging.gemspec
CHANGED
@@ -4,9 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
require 'manageiq/messaging/version'
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name
|
8
|
-
spec.version
|
9
|
-
spec.
|
7
|
+
spec.name = "manageiq-messaging"
|
8
|
+
spec.version = ManageIQ::Messaging::VERSION
|
9
|
+
spec.required_ruby_version = '>= 2.4'
|
10
|
+
spec.authors = ["ManageIQ Authors"]
|
10
11
|
|
11
12
|
spec.summary = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
|
12
13
|
spec.description = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
|
@@ -20,15 +21,14 @@ Gem::Specification.new do |spec|
|
|
20
21
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
22
|
spec.require_paths = ["lib"]
|
22
23
|
|
23
|
-
spec.add_dependency 'activesupport', '>=
|
24
|
-
spec.add_dependency '
|
24
|
+
spec.add_dependency 'activesupport', '>= 5.2.4.3', "< 6.1"
|
25
|
+
spec.add_dependency 'rdkafka', '~> 0.8'
|
25
26
|
spec.add_dependency 'stomp', '~> 1.4.4'
|
26
27
|
|
27
28
|
spec.add_development_dependency "bundler"
|
28
29
|
spec.add_development_dependency "codeclimate-test-reporter", "~> 1.0.0"
|
29
|
-
spec.add_development_dependency "
|
30
|
+
spec.add_development_dependency "manageiq-style"
|
31
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
30
32
|
spec.add_development_dependency "rspec", "~> 3.0"
|
31
|
-
spec.add_development_dependency "rubocop"
|
32
|
-
spec.add_development_dependency "rubocop-performance"
|
33
33
|
spec.add_development_dependency "simplecov"
|
34
34
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: manageiq-messaging
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ManageIQ Authors
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -16,28 +16,34 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 5.2.4.3
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '6.1'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
27
|
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
29
|
+
version: 5.2.4.3
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '6.1'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
34
|
+
name: rdkafka
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
37
|
- - "~>"
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.
|
39
|
+
version: '0.8'
|
34
40
|
type: :runtime
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
44
|
- - "~>"
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version: 0.
|
46
|
+
version: '0.8'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: stomp
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -81,61 +87,47 @@ dependencies:
|
|
81
87
|
- !ruby/object:Gem::Version
|
82
88
|
version: 1.0.0
|
83
89
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - "~>"
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '10.0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - "~>"
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '10.0'
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
|
-
name: rspec
|
90
|
+
name: manageiq-style
|
99
91
|
requirement: !ruby/object:Gem::Requirement
|
100
92
|
requirements:
|
101
|
-
- - "
|
93
|
+
- - ">="
|
102
94
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
95
|
+
version: '0'
|
104
96
|
type: :development
|
105
97
|
prerelease: false
|
106
98
|
version_requirements: !ruby/object:Gem::Requirement
|
107
99
|
requirements:
|
108
|
-
- - "
|
100
|
+
- - ">="
|
109
101
|
- !ruby/object:Gem::Version
|
110
|
-
version: '
|
102
|
+
version: '0'
|
111
103
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
104
|
+
name: rake
|
113
105
|
requirement: !ruby/object:Gem::Requirement
|
114
106
|
requirements:
|
115
107
|
- - ">="
|
116
108
|
- !ruby/object:Gem::Version
|
117
|
-
version:
|
109
|
+
version: 12.3.3
|
118
110
|
type: :development
|
119
111
|
prerelease: false
|
120
112
|
version_requirements: !ruby/object:Gem::Requirement
|
121
113
|
requirements:
|
122
114
|
- - ">="
|
123
115
|
- !ruby/object:Gem::Version
|
124
|
-
version:
|
116
|
+
version: 12.3.3
|
125
117
|
- !ruby/object:Gem::Dependency
|
126
|
-
name:
|
118
|
+
name: rspec
|
127
119
|
requirement: !ruby/object:Gem::Requirement
|
128
120
|
requirements:
|
129
|
-
- - "
|
121
|
+
- - "~>"
|
130
122
|
- !ruby/object:Gem::Version
|
131
|
-
version: '0'
|
123
|
+
version: '3.0'
|
132
124
|
type: :development
|
133
125
|
prerelease: false
|
134
126
|
version_requirements: !ruby/object:Gem::Requirement
|
135
127
|
requirements:
|
136
|
-
- - "
|
128
|
+
- - "~>"
|
137
129
|
- !ruby/object:Gem::Version
|
138
|
-
version: '0'
|
130
|
+
version: '3.0'
|
139
131
|
- !ruby/object:Gem::Dependency
|
140
132
|
name: simplecov
|
141
133
|
requirement: !ruby/object:Gem::Requirement
|
@@ -152,7 +144,7 @@ dependencies:
|
|
152
144
|
version: '0'
|
153
145
|
description: Client library for ManageIQ components to exchange messages through its
|
154
146
|
internal message bus.
|
155
|
-
email:
|
147
|
+
email:
|
156
148
|
executables: []
|
157
149
|
extensions: []
|
158
150
|
extra_rdoc_files: []
|
@@ -200,7 +192,7 @@ homepage: http://github.com/ManageIQ/manageiq-messaging
|
|
200
192
|
licenses:
|
201
193
|
- MIT
|
202
194
|
metadata: {}
|
203
|
-
post_install_message:
|
195
|
+
post_install_message:
|
204
196
|
rdoc_options: []
|
205
197
|
require_paths:
|
206
198
|
- lib
|
@@ -208,16 +200,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
208
200
|
requirements:
|
209
201
|
- - ">="
|
210
202
|
- !ruby/object:Gem::Version
|
211
|
-
version: '
|
203
|
+
version: '2.4'
|
212
204
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
213
205
|
requirements:
|
214
206
|
- - ">="
|
215
207
|
- !ruby/object:Gem::Version
|
216
208
|
version: '0'
|
217
209
|
requirements: []
|
218
|
-
|
219
|
-
|
220
|
-
signing_key:
|
210
|
+
rubygems_version: 3.2.5
|
211
|
+
signing_key:
|
221
212
|
specification_version: 4
|
222
213
|
summary: Client library for ManageIQ components to exchange messages through its internal
|
223
214
|
message bus.
|