manageiq-messaging 0.1.4 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.codeclimate.yml +1 -0
- data/.rubocop.yml +2 -1
- data/.travis.yml +2 -3
- data/CHANGES +17 -0
- data/README.md +19 -2
- data/lib/manageiq/messaging/client.rb +3 -0
- data/lib/manageiq/messaging/common.rb +5 -1
- data/lib/manageiq/messaging/kafka/client.rb +24 -29
- data/lib/manageiq/messaging/kafka/common.rb +48 -45
- data/lib/manageiq/messaging/kafka/queue.rb +10 -24
- data/lib/manageiq/messaging/kafka/topic.rb +9 -12
- data/lib/manageiq/messaging/received_message.rb +3 -3
- data/lib/manageiq/messaging/stomp/common.rb +8 -0
- data/lib/manageiq/messaging/stomp/queue.rb +8 -1
- data/lib/manageiq/messaging/stomp/topic.rb +4 -1
- data/lib/manageiq/messaging/version.rb +1 -1
- data/manageiq-messaging.gemspec +9 -8
- metadata +38 -33
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 56402b3a8736e2983e711c6521c39e12c5afac32dc431e1fc64c9a608c59cc09
|
4
|
+
data.tar.gz: cfe1fb310fbf5971c98f69e866d2637e0b3639822bd05dbc8a16a607c0fbd133
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d107b4468ef6e1b6a37a6a2ab3b294c2fa6f2a6482949bec3b01cc38aa75f8cc4ccf95eeb7e0a22802afcf0e8158dc4cb68de0ea26b22713c520febd8457b1be
|
7
|
+
data.tar.gz: 0fd07ec8805d22ac6cdfef59f58b5e30393e011e822a534f6f6e2159104aa7cba28f1403e1b78872f4f73e5a506e466c25d061318510090bd8fc836345cb8c85
|
data/.codeclimate.yml
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
+
inherit_gem:
|
2
|
+
manageiq-style: .rubocop_base.yml
|
1
3
|
inherit_from:
|
2
|
-
- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
|
3
4
|
# put all local rubocop config into .rubocop_local.yml as it will be loaded by .rubocop_cc.yml as well
|
4
5
|
- .rubocop_local.yml
|
data/.travis.yml
CHANGED
data/CHANGES
CHANGED
@@ -11,3 +11,20 @@
|
|
11
11
|
= 0.1.4 - 3-Apr-2019
|
12
12
|
* Add an #ack method to a ReceivedMessage to simplify manual acknowledgements
|
13
13
|
* Allow caller to provide a session_timeout to kafka consumers, default of 30sec
|
14
|
+
|
15
|
+
= 0.1.5 - 6-Jun-2019
|
16
|
+
* Allow caller to provide extra headers to the message
|
17
|
+
|
18
|
+
= 0.1.6 - 6-July-2020
|
19
|
+
* Rescue message body decoding errors. Re-raise errors raised by users code of processing received messages.
|
20
|
+
|
21
|
+
= 1.0.0 - 28-Sep-2020
|
22
|
+
* Switch to use rdkafka client
|
23
|
+
|
24
|
+
= 1.0.1 - 15-Dec-2020
|
25
|
+
* Allow all kafka options to be passed
|
26
|
+
* remove sudo:false from .travis.yml
|
27
|
+
* Use manageiq-style
|
28
|
+
|
29
|
+
= 1.0.2 - 4-Jan-2021
|
30
|
+
* Rails 6.0 Support
|
data/README.md
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
# ManageIQ Messaging Client
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/manageiq-messaging.svg)](http://badge.fury.io/rb/manageiq-messaging)
|
4
|
-
[![Build Status](https://travis-ci.
|
4
|
+
[![Build Status](https://travis-ci.com/ManageIQ/manageiq-messaging.svg)](https://travis-ci.com/ManageIQ/manageiq-messaging)
|
5
5
|
[![Code Climate](https://codeclimate.com/github/ManageIQ/manageiq-messaging.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging)
|
6
6
|
[![Test Coverage](https://codeclimate.com/github/ManageIQ/manageiq-messaging/badges/coverage.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging/coverage)
|
7
|
-
[![Dependency Status](https://gemnasium.com/ManageIQ/manageiq-messaging.svg)](https://gemnasium.com/ManageIQ/manageiq-messaging)
|
8
7
|
[![Security](https://hakiri.io/github/ManageIQ/manageiq-messaging/master.svg)](https://hakiri.io/github/ManageIQ/manageiq-messaging/master)
|
9
8
|
|
10
9
|
Client library for ManageIQ components to exchange messages through its internal message bus.
|
@@ -156,6 +155,24 @@ This is the one-to-many publish/subscribe pattern. Multiple subscribers can subs
|
|
156
155
|
|
157
156
|
By default, events are delivered to live subscribers only. Some messaging systems support persistence with options.
|
158
157
|
|
158
|
+
### Add your own headers to a message (Queue or Topic)
|
159
|
+
|
160
|
+
If you want you can add in your own headers to the send message
|
161
|
+
|
162
|
+
```
|
163
|
+
client.publish_topic(
|
164
|
+
:service => 'provider_events',
|
165
|
+
:event => 'powered_on',
|
166
|
+
:headers => {:request_id => "12345"},
|
167
|
+
:payload => {:ems_ref => 'uid987'}
|
168
|
+
|
169
|
+
)
|
170
|
+
|
171
|
+
client.subscribe_topic(:service => 'provider_events', :persist_ref => 'automate_1') do |msg|
|
172
|
+
puts "Received event #{msg.message} with request-id: #{msg.headers['request_id']}"
|
173
|
+
end
|
174
|
+
```
|
175
|
+
|
159
176
|
## Development
|
160
177
|
|
161
178
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -40,6 +40,7 @@ module ManageIQ
|
|
40
40
|
def self.open(options)
|
41
41
|
protocol = options[:protocol] || :Stomp
|
42
42
|
client = Object.const_get("ManageIQ::Messaging::#{protocol}::Client").new(options)
|
43
|
+
|
43
44
|
return client unless block_given?
|
44
45
|
|
45
46
|
begin
|
@@ -58,6 +59,7 @@ module ManageIQ
|
|
58
59
|
# * :message (e.g. method name or message type)
|
59
60
|
# * :payload (message body, a string or an user object that can be serialized)
|
60
61
|
# * :sender (optional, identify the sender)
|
62
|
+
# * :headers (optional, additional headers to add to the message)
|
61
63
|
# Other options are underlying messaging system specific.
|
62
64
|
#
|
63
65
|
# Optionally a call back block can be provided to wait on the consumer to send
|
@@ -164,6 +166,7 @@ module ManageIQ
|
|
164
166
|
# * :event (event name)
|
165
167
|
# * :payload (message body, a string or an user object that can be serialized)
|
166
168
|
# * :sender (optional, identify the sender)
|
169
|
+
# * :headers (optional, additional headers to add to the message)
|
167
170
|
# Other options are underlying messaging system specific.
|
168
171
|
#
|
169
172
|
def publish_topic(options)
|
@@ -18,7 +18,7 @@ module ManageIQ
|
|
18
18
|
|
19
19
|
def decode_body(headers, raw_body)
|
20
20
|
return raw_body unless headers.kind_of?(Hash)
|
21
|
-
case headers["encoding"]
|
21
|
+
case headers["encoding"] || headers[:encoding]
|
22
22
|
when "json"
|
23
23
|
JSON.parse(raw_body)
|
24
24
|
when "yaml"
|
@@ -26,6 +26,10 @@ module ManageIQ
|
|
26
26
|
else
|
27
27
|
raw_body
|
28
28
|
end
|
29
|
+
rescue => e # JSON or YAML parsing error
|
30
|
+
logger.error("Error decoding message body: #{e.message}")
|
31
|
+
logger.error(e.backtrace.join("\n"))
|
32
|
+
raw_body
|
29
33
|
end
|
30
34
|
|
31
35
|
def payload_log(payload)
|
@@ -10,16 +10,11 @@ module ManageIQ
|
|
10
10
|
# * :hosts (Array of Kafka cluster hosts, or)
|
11
11
|
# * :host (Single host name)
|
12
12
|
# * :port (host port number)
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
# * :sasl_plain_username
|
19
|
-
# * :sasl_plain_password
|
20
|
-
# * :sasl_scram_username
|
21
|
-
# * :sasl_scram_password
|
22
|
-
# * :sasl_scram_mechanism
|
13
|
+
#
|
14
|
+
# For additional security options, please refer to
|
15
|
+
# https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka and
|
16
|
+
# https://github.com/edenhill/librdkafka/wiki/Using-SASL-with-librdkafka
|
17
|
+
#
|
23
18
|
#
|
24
19
|
# Kafka specific +publish_message+ options:
|
25
20
|
# * :group_name (Used as Kafka partition_key)
|
@@ -42,7 +37,7 @@ module ManageIQ
|
|
42
37
|
#
|
43
38
|
# +subscribe_background_job+ is currently not implemented.
|
44
39
|
class Client < ManageIQ::Messaging::Client
|
45
|
-
require '
|
40
|
+
require 'rdkafka'
|
46
41
|
require 'manageiq/messaging/kafka/common'
|
47
42
|
require 'manageiq/messaging/kafka/queue'
|
48
43
|
require 'manageiq/messaging/kafka/background_job'
|
@@ -53,27 +48,27 @@ module ManageIQ
|
|
53
48
|
include BackgroundJob
|
54
49
|
include Topic
|
55
50
|
|
56
|
-
private *delegate(:subscribe, :unsubscribe, :publish, :to => :kafka_client)
|
57
|
-
delegate :close, :to => :kafka_client
|
58
|
-
|
59
51
|
attr_accessor :encoding
|
60
52
|
|
61
53
|
def ack(ack_ref)
|
62
|
-
|
63
|
-
|
54
|
+
ack_ref.commit
|
55
|
+
rescue Rdkafka::RdkafkaError => e
|
56
|
+
logger.warn("ack failed with error #{e.message}")
|
57
|
+
raise unless e.message =~ /no_offset/
|
64
58
|
end
|
65
59
|
|
66
60
|
def close
|
67
|
-
@
|
68
|
-
@topic_consumer = nil
|
69
|
-
@queue_consumer.try(:stop)
|
70
|
-
@queue_consumer = nil
|
71
|
-
|
72
|
-
@producer.try(:shutdown)
|
61
|
+
@producer&.close
|
73
62
|
@producer = nil
|
74
63
|
|
75
|
-
|
76
|
-
@
|
64
|
+
@consumer&.close
|
65
|
+
@consumer = nil
|
66
|
+
end
|
67
|
+
|
68
|
+
# list all topics
|
69
|
+
def topics
|
70
|
+
native_kafka = producer.instance_variable_get(:@native_kafka)
|
71
|
+
Rdkafka::Metadata.new(native_kafka).topics.collect { |topic| topic[:topic_name] }
|
77
72
|
end
|
78
73
|
|
79
74
|
private
|
@@ -87,12 +82,12 @@ module ManageIQ
|
|
87
82
|
@encoding = options[:encoding] || 'yaml'
|
88
83
|
require "json" if @encoding == "json"
|
89
84
|
|
90
|
-
connection_opts = {}
|
91
|
-
connection_opts[:
|
92
|
-
|
93
|
-
connection_opts.merge!(options.slice(:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :sasl_gssapi_principal, :sasl_gssapi_keytab, :sasl_plain_username, :sasl_plain_password, :sasl_scram_username, :sasl_scram_password, :sasl_scram_mechanism))
|
85
|
+
connection_opts = {:"bootstrap.servers" => hosts.join(',')}
|
86
|
+
connection_opts[:"client.id"] = options[:client_ref] if options[:client_ref]
|
87
|
+
connection_opts.merge!(options.except(:port, :host, :hosts, :encoding, :protocol, :client_ref))
|
94
88
|
|
95
|
-
|
89
|
+
::Rdkafka::Config.logger = logger
|
90
|
+
@kafka_client = ::Rdkafka::Config.new(connection_opts)
|
96
91
|
end
|
97
92
|
end
|
98
93
|
end
|
@@ -5,54 +5,35 @@ module ManageIQ
|
|
5
5
|
require 'manageiq/messaging/common'
|
6
6
|
include ManageIQ::Messaging::Common
|
7
7
|
|
8
|
-
GROUP_FOR_QUEUE_MESSAGES = 'manageiq_messaging_queue_group_'.freeze
|
9
|
-
|
10
8
|
private
|
11
9
|
|
12
10
|
def producer
|
13
11
|
@producer ||= kafka_client.producer
|
14
12
|
end
|
15
13
|
|
16
|
-
def
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
@topic_consumer ||= kafka_client.consumer(consumer_opts)
|
26
|
-
end
|
27
|
-
|
28
|
-
def queue_consumer(topic, session_timeout = nil)
|
29
|
-
# all queue consumers join the same group so that each message can be processed by one and only one consumer
|
30
|
-
@queue_consumer.try(:stop) unless @queue_topic == topic
|
31
|
-
@queue_topic = topic
|
32
|
-
|
33
|
-
consumer_opts = {:group_id => GROUP_FOR_QUEUE_MESSAGES + topic}
|
34
|
-
consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
|
35
|
-
|
36
|
-
@queue_consumer ||= kafka_client.consumer(consumer_opts)
|
14
|
+
def consumer(beginning, options)
|
15
|
+
@consumer&.close
|
16
|
+
kafka_client[:"group.id"] = options[:persist_ref]
|
17
|
+
kafka_client[:"auto.offset.reset"] = beginning ? 'smallest' : 'largest'
|
18
|
+
kafka_client[:"enable.auto.commit"] = !!auto_ack?(options)
|
19
|
+
kafka_client[:"session.timeout.ms"] = options[:session_timeout] * 1000 if options[:session_timeout].present?
|
20
|
+
kafka_client[:"group.instance.id"] = options[:group_instance_id] if options[:group_instance_id].present?
|
21
|
+
@consumer = kafka_client.consumer
|
37
22
|
end
|
38
23
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
def raw_publish(commit, body, options)
|
47
|
-
producer.produce(encode_body(options[:headers], body), options)
|
48
|
-
producer.deliver_messages if commit
|
49
|
-
logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
|
24
|
+
def raw_publish(wait, body, options)
|
25
|
+
options[:payload] = encode_body(options[:headers], body)
|
26
|
+
producer.produce(options).tap do |handle|
|
27
|
+
handle.wait if wait
|
28
|
+
logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
|
29
|
+
end
|
50
30
|
end
|
51
31
|
|
52
32
|
def queue_for_publish(options)
|
53
33
|
body, kafka_opts = for_publish(options)
|
54
34
|
kafka_opts[:headers][:message_type] = options[:message] if options[:message]
|
55
35
|
kafka_opts[:headers][:class_name] = options[:class_name] if options[:class_name]
|
36
|
+
kafka_opts[:headers].merge!(options[:headers].except(*message_header_keys)) if options.key?(:headers)
|
56
37
|
|
57
38
|
[body, kafka_opts]
|
58
39
|
end
|
@@ -60,13 +41,15 @@ module ManageIQ
|
|
60
41
|
def topic_for_publish(options)
|
61
42
|
body, kafka_opts = for_publish(options)
|
62
43
|
kafka_opts[:headers][:event_type] = options[:event] if options[:event]
|
44
|
+
kafka_opts[:headers].merge!(options[:headers].except(*event_header_keys)) if options.key?(:headers)
|
63
45
|
|
64
46
|
[body, kafka_opts]
|
65
47
|
end
|
66
48
|
|
67
49
|
def for_publish(options)
|
68
|
-
kafka_opts = {:topic => address(options)
|
50
|
+
kafka_opts = {:topic => address(options)}
|
69
51
|
kafka_opts[:partition_key] = options[:group_name] if options[:group_name]
|
52
|
+
kafka_opts[:headers] = {}
|
70
53
|
kafka_opts[:headers][:sender] = options[:sender] if options[:sender]
|
71
54
|
|
72
55
|
body = options[:payload] || ''
|
@@ -82,34 +65,54 @@ module ManageIQ
|
|
82
65
|
end
|
83
66
|
end
|
84
67
|
|
85
|
-
def process_queue_message(queue, message)
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
68
|
+
def process_queue_message(queue_consumer, queue, message)
|
69
|
+
begin
|
70
|
+
payload = decode_body(message.headers, message.payload)
|
71
|
+
sender, message_type, _class_name = parse_message_headers(message.headers)
|
72
|
+
client_headers = message.headers.except(*message_header_keys).with_indifferent_access
|
73
|
+
|
74
|
+
logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
|
75
|
+
yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, client_headers, queue_consumer, self)]
|
76
|
+
logger.info("Messsage processed")
|
77
|
+
rescue StandardError => e
|
78
|
+
logger.error("Message processing error: #{e.message}")
|
79
|
+
logger.error(e.backtrace.join("\n"))
|
80
|
+
raise
|
81
|
+
end
|
90
82
|
end
|
91
83
|
|
92
|
-
def process_topic_message(topic, message)
|
84
|
+
def process_topic_message(topic_consumer, topic, message)
|
93
85
|
begin
|
94
|
-
payload = decode_body(message.headers, message.
|
86
|
+
payload = decode_body(message.headers, message.payload)
|
95
87
|
sender, event_type = parse_event_headers(message.headers)
|
88
|
+
client_headers = message.headers.except(*event_header_keys).with_indifferent_access
|
89
|
+
|
96
90
|
logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
|
97
|
-
yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload,
|
91
|
+
yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers, topic_consumer, self)
|
98
92
|
logger.info("Event processed")
|
99
93
|
rescue StandardError => e
|
100
94
|
logger.error("Event processing error: #{e.message}")
|
101
95
|
logger.error(e.backtrace.join("\n"))
|
96
|
+
raise
|
102
97
|
end
|
103
98
|
end
|
104
99
|
|
100
|
+
def message_header_keys
|
101
|
+
[:sender, :message_type, :class_name]
|
102
|
+
end
|
103
|
+
|
105
104
|
def parse_message_headers(headers)
|
106
105
|
return [nil, nil, nil] unless headers.kind_of?(Hash)
|
107
|
-
headers.values_at(
|
106
|
+
headers.values_at(*message_header_keys)
|
107
|
+
end
|
108
|
+
|
109
|
+
def event_header_keys
|
110
|
+
[:sender, :event_type]
|
108
111
|
end
|
109
112
|
|
110
113
|
def parse_event_headers(headers)
|
111
114
|
return [nil, nil] unless headers.kind_of?(Hash)
|
112
|
-
headers.values_at(
|
115
|
+
headers.values_at(*event_header_keys)
|
113
116
|
end
|
114
117
|
end
|
115
118
|
end
|
@@ -2,6 +2,8 @@ module ManageIQ
|
|
2
2
|
module Messaging
|
3
3
|
module Kafka
|
4
4
|
module Queue
|
5
|
+
GROUP_FOR_QUEUE_MESSAGES = ENV['QUEUE_MESSAGES_GROUP_PREFIX'].freeze || 'manageiq_messaging_queue_group_'.freeze
|
6
|
+
|
5
7
|
private
|
6
8
|
|
7
9
|
def publish_message_impl(options)
|
@@ -10,34 +12,18 @@ module ManageIQ
|
|
10
12
|
end
|
11
13
|
|
12
14
|
def publish_messages_impl(messages)
|
13
|
-
messages.
|
14
|
-
|
15
|
+
handles = messages.collect { |msg_options| raw_publish(false, *queue_for_publish(msg_options)) }
|
16
|
+
handles.each(&:wait)
|
15
17
|
end
|
16
18
|
|
17
|
-
def subscribe_messages_impl(options)
|
19
|
+
def subscribe_messages_impl(options, &block)
|
18
20
|
topic = address(options)
|
19
|
-
|
20
|
-
|
21
|
-
batch_options = {}
|
22
|
-
batch_options[:automatically_mark_as_processed] = auto_ack?(options)
|
23
|
-
batch_options[:max_bytes] = options[:max_bytes] if options.key?(:max_bytes)
|
24
|
-
|
25
|
-
consumer = queue_consumer(topic, session_timeout)
|
26
|
-
consumer.subscribe(topic)
|
27
|
-
consumer.each_batch(batch_options) do |batch|
|
28
|
-
logger.info("Batch message received: queue(#{topic})")
|
29
|
-
begin
|
30
|
-
messages = batch.messages.collect do |message|
|
31
|
-
sender, message_type, _class_name, payload = process_queue_message(topic, message)
|
32
|
-
ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, message, self)
|
33
|
-
end
|
21
|
+
options[:persist_ref] = GROUP_FOR_QUEUE_MESSAGES + topic
|
34
22
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
end
|
40
|
-
logger.info("Batch message processed")
|
23
|
+
queue_consumer = consumer(true, options)
|
24
|
+
queue_consumer.subscribe(topic)
|
25
|
+
queue_consumer.each do |message|
|
26
|
+
process_queue_message(queue_consumer, topic, message, &block)
|
41
27
|
end
|
42
28
|
end
|
43
29
|
end
|
@@ -1,7 +1,11 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
1
3
|
module ManageIQ
|
2
4
|
module Messaging
|
3
5
|
module Kafka
|
4
6
|
module Topic
|
7
|
+
GROUP_FOR_ADHOC_LISTENERS = Socket.gethostname.freeze
|
8
|
+
|
5
9
|
private
|
6
10
|
|
7
11
|
def publish_topic_impl(options)
|
@@ -10,19 +14,12 @@ module ManageIQ
|
|
10
14
|
|
11
15
|
def subscribe_topic_impl(options, &block)
|
12
16
|
topic = address(options)
|
13
|
-
persist_ref = options[:persist_ref]
|
14
|
-
session_timeout = options[:session_timeout]
|
15
17
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
else
|
23
|
-
kafka_client.each_message(:topic => topic, :start_from_beginning => false) do |message|
|
24
|
-
process_topic_message(topic, message, &block)
|
25
|
-
end
|
18
|
+
options[:persist_ref] = "#{GROUP_FOR_ADHOC_LISTENERS}_#{Time.now.to_i}" unless options[:persist_ref]
|
19
|
+
topic_consumer = consumer(false, options)
|
20
|
+
topic_consumer.subscribe(topic)
|
21
|
+
topic_consumer.each do |message|
|
22
|
+
process_topic_message(topic_consumer, topic, message, &block)
|
26
23
|
end
|
27
24
|
end
|
28
25
|
end
|
@@ -1,10 +1,10 @@
|
|
1
1
|
module ManageIQ
|
2
2
|
module Messaging
|
3
3
|
class ReceivedMessage
|
4
|
-
attr_accessor :sender, :message, :payload, :ack_ref, :client
|
4
|
+
attr_accessor :sender, :message, :payload, :headers, :ack_ref, :client
|
5
5
|
|
6
|
-
def initialize(sender, message, payload, ack_ref, client)
|
7
|
-
@sender, @message, @payload, @ack_ref, @client = sender, message, payload, ack_ref, client
|
6
|
+
def initialize(sender, message, payload, headers, ack_ref, client)
|
7
|
+
@sender, @message, @payload, @headers, @ack_ref, @client = sender, message, payload, headers, ack_ref, client
|
8
8
|
end
|
9
9
|
|
10
10
|
def ack
|
@@ -17,6 +17,8 @@ module ManageIQ
|
|
17
17
|
address = "queue/#{options[:service]}.#{affinity}"
|
18
18
|
|
19
19
|
headers = {:"destination-type" => 'ANYCAST', :persistent => true}
|
20
|
+
headers.merge!(options[:headers].except(*internal_header_keys)) if options.key?(:headers)
|
21
|
+
|
20
22
|
headers[:expires] = options[:expires_on].to_i * 1000 if options[:expires_on]
|
21
23
|
headers[:AMQ_SCHEDULED_TIME] = options[:deliver_on].to_i * 1000 if options[:deliver_on]
|
22
24
|
headers[:priority] = options[:priority] if options[:priority]
|
@@ -38,6 +40,8 @@ module ManageIQ
|
|
38
40
|
address = "topic/#{options[:service]}"
|
39
41
|
|
40
42
|
headers = {:"destination-type" => 'MULTICAST', :persistent => true}
|
43
|
+
headers.merge!(options[:headers].except(*internal_header_keys)) if options.key?(:headers)
|
44
|
+
|
41
45
|
headers[:expires] = options[:expires_on].to_i * 1000 if options[:expires_on]
|
42
46
|
headers[:AMQ_SCHEDULED_TIME] = options[:deliver_on].to_i * 1000 if options[:deliver_on]
|
43
47
|
headers[:priority] = options[:priority] if options[:priority]
|
@@ -63,6 +67,10 @@ module ManageIQ
|
|
63
67
|
raw_publish(address, result || '', response_headers.merge(:correlation_id => correlation_ref))
|
64
68
|
end
|
65
69
|
|
70
|
+
def internal_header_keys
|
71
|
+
[:"destination-type", :persistent, :expires, :AMQ_SCHEDULED_TIME, :priority, :_AMQ_GROUP_ID]
|
72
|
+
end
|
73
|
+
|
66
74
|
def receive_response(service, correlation_ref)
|
67
75
|
response_options = {
|
68
76
|
:service => "#{service}.response",
|
@@ -33,9 +33,15 @@ module ManageIQ
|
|
33
33
|
sender = msg.headers['sender']
|
34
34
|
message_type = msg.headers['message_type']
|
35
35
|
message_body = decode_body(msg.headers, msg.body)
|
36
|
+
client_headers = msg.headers.except(*internal_header_keys)
|
37
|
+
|
36
38
|
logger.info("Message received: queue(#{queue_name}), msg(#{payload_log(message_body)}), headers(#{msg.headers})")
|
37
39
|
|
38
|
-
|
40
|
+
messages = [
|
41
|
+
ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, message_body, client_headers, msg, self)
|
42
|
+
]
|
43
|
+
|
44
|
+
result = yield messages
|
39
45
|
logger.info("Message processed")
|
40
46
|
|
41
47
|
correlation_ref = msg.headers['correlation_id']
|
@@ -46,6 +52,7 @@ module ManageIQ
|
|
46
52
|
rescue => e
|
47
53
|
logger.error("Message processing error: #{e.message}")
|
48
54
|
logger.error(e.backtrace.join("\n"))
|
55
|
+
raise
|
49
56
|
end
|
50
57
|
end
|
51
58
|
end
|
@@ -21,13 +21,16 @@ module ManageIQ
|
|
21
21
|
|
22
22
|
sender = event.headers['sender']
|
23
23
|
event_type = event.headers['event_type']
|
24
|
+
client_headers = event.headers.except(*internal_header_keys)
|
25
|
+
|
24
26
|
event_body = decode_body(event.headers, event.body)
|
25
27
|
logger.info("Event received: queue(#{queue_name}), event(#{event_body}), headers(#{event.headers})")
|
26
|
-
yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, event_body, event, self)
|
28
|
+
yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, event_body, client_headers, event, self)
|
27
29
|
logger.info("Event processed")
|
28
30
|
rescue => e
|
29
31
|
logger.error("Event processing error: #{e.message}")
|
30
32
|
logger.error(e.backtrace.join("\n"))
|
33
|
+
raise
|
31
34
|
end
|
32
35
|
end
|
33
36
|
end
|
data/manageiq-messaging.gemspec
CHANGED
@@ -4,9 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
require 'manageiq/messaging/version'
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name
|
8
|
-
spec.version
|
9
|
-
spec.
|
7
|
+
spec.name = "manageiq-messaging"
|
8
|
+
spec.version = ManageIQ::Messaging::VERSION
|
9
|
+
spec.required_ruby_version = '>= 2.4'
|
10
|
+
spec.authors = ["ManageIQ Authors"]
|
10
11
|
|
11
12
|
spec.summary = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
|
12
13
|
spec.description = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
|
@@ -20,14 +21,14 @@ Gem::Specification.new do |spec|
|
|
20
21
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
22
|
spec.require_paths = ["lib"]
|
22
23
|
|
23
|
-
spec.add_dependency 'activesupport', '>=
|
24
|
-
spec.add_dependency '
|
24
|
+
spec.add_dependency 'activesupport', '>= 5.2.4.3', "< 6.1"
|
25
|
+
spec.add_dependency 'rdkafka', '~> 0.8'
|
25
26
|
spec.add_dependency 'stomp', '~> 1.4.4'
|
26
27
|
|
27
|
-
spec.add_development_dependency "bundler"
|
28
|
+
spec.add_development_dependency "bundler"
|
28
29
|
spec.add_development_dependency "codeclimate-test-reporter", "~> 1.0.0"
|
29
|
-
spec.add_development_dependency "
|
30
|
+
spec.add_development_dependency "manageiq-style"
|
31
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
30
32
|
spec.add_development_dependency "rspec", "~> 3.0"
|
31
|
-
spec.add_development_dependency "rubocop"
|
32
33
|
spec.add_development_dependency "simplecov"
|
33
34
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: manageiq-messaging
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ManageIQ Authors
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -16,28 +16,34 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 5.2.4.3
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '6.1'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
27
|
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
29
|
+
version: 5.2.4.3
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '6.1'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
34
|
+
name: rdkafka
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
30
36
|
requirements:
|
31
37
|
- - "~>"
|
32
38
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.
|
39
|
+
version: '0.8'
|
34
40
|
type: :runtime
|
35
41
|
prerelease: false
|
36
42
|
version_requirements: !ruby/object:Gem::Requirement
|
37
43
|
requirements:
|
38
44
|
- - "~>"
|
39
45
|
- !ruby/object:Gem::Version
|
40
|
-
version: 0.
|
46
|
+
version: '0.8'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: stomp
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -56,16 +62,16 @@ dependencies:
|
|
56
62
|
name: bundler
|
57
63
|
requirement: !ruby/object:Gem::Requirement
|
58
64
|
requirements:
|
59
|
-
- - "
|
65
|
+
- - ">="
|
60
66
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
67
|
+
version: '0'
|
62
68
|
type: :development
|
63
69
|
prerelease: false
|
64
70
|
version_requirements: !ruby/object:Gem::Requirement
|
65
71
|
requirements:
|
66
|
-
- - "
|
72
|
+
- - ">="
|
67
73
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
74
|
+
version: '0'
|
69
75
|
- !ruby/object:Gem::Dependency
|
70
76
|
name: codeclimate-test-reporter
|
71
77
|
requirement: !ruby/object:Gem::Requirement
|
@@ -81,47 +87,47 @@ dependencies:
|
|
81
87
|
- !ruby/object:Gem::Version
|
82
88
|
version: 1.0.0
|
83
89
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
90
|
+
name: manageiq-style
|
85
91
|
requirement: !ruby/object:Gem::Requirement
|
86
92
|
requirements:
|
87
|
-
- - "
|
93
|
+
- - ">="
|
88
94
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
95
|
+
version: '0'
|
90
96
|
type: :development
|
91
97
|
prerelease: false
|
92
98
|
version_requirements: !ruby/object:Gem::Requirement
|
93
99
|
requirements:
|
94
|
-
- - "
|
100
|
+
- - ">="
|
95
101
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
102
|
+
version: '0'
|
97
103
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
104
|
+
name: rake
|
99
105
|
requirement: !ruby/object:Gem::Requirement
|
100
106
|
requirements:
|
101
|
-
- - "
|
107
|
+
- - ">="
|
102
108
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
109
|
+
version: 12.3.3
|
104
110
|
type: :development
|
105
111
|
prerelease: false
|
106
112
|
version_requirements: !ruby/object:Gem::Requirement
|
107
113
|
requirements:
|
108
|
-
- - "
|
114
|
+
- - ">="
|
109
115
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
116
|
+
version: 12.3.3
|
111
117
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
118
|
+
name: rspec
|
113
119
|
requirement: !ruby/object:Gem::Requirement
|
114
120
|
requirements:
|
115
|
-
- - "
|
121
|
+
- - "~>"
|
116
122
|
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
123
|
+
version: '3.0'
|
118
124
|
type: :development
|
119
125
|
prerelease: false
|
120
126
|
version_requirements: !ruby/object:Gem::Requirement
|
121
127
|
requirements:
|
122
|
-
- - "
|
128
|
+
- - "~>"
|
123
129
|
- !ruby/object:Gem::Version
|
124
|
-
version: '0'
|
130
|
+
version: '3.0'
|
125
131
|
- !ruby/object:Gem::Dependency
|
126
132
|
name: simplecov
|
127
133
|
requirement: !ruby/object:Gem::Requirement
|
@@ -138,7 +144,7 @@ dependencies:
|
|
138
144
|
version: '0'
|
139
145
|
description: Client library for ManageIQ components to exchange messages through its
|
140
146
|
internal message bus.
|
141
|
-
email:
|
147
|
+
email:
|
142
148
|
executables: []
|
143
149
|
extensions: []
|
144
150
|
extra_rdoc_files: []
|
@@ -186,7 +192,7 @@ homepage: http://github.com/ManageIQ/manageiq-messaging
|
|
186
192
|
licenses:
|
187
193
|
- MIT
|
188
194
|
metadata: {}
|
189
|
-
post_install_message:
|
195
|
+
post_install_message:
|
190
196
|
rdoc_options: []
|
191
197
|
require_paths:
|
192
198
|
- lib
|
@@ -194,16 +200,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
194
200
|
requirements:
|
195
201
|
- - ">="
|
196
202
|
- !ruby/object:Gem::Version
|
197
|
-
version: '
|
203
|
+
version: '2.4'
|
198
204
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
199
205
|
requirements:
|
200
206
|
- - ">="
|
201
207
|
- !ruby/object:Gem::Version
|
202
208
|
version: '0'
|
203
209
|
requirements: []
|
204
|
-
|
205
|
-
|
206
|
-
signing_key:
|
210
|
+
rubygems_version: 3.2.0.rc.2
|
211
|
+
signing_key:
|
207
212
|
specification_version: 4
|
208
213
|
summary: Client library for ManageIQ components to exchange messages through its internal
|
209
214
|
message bus.
|