manageiq-messaging 0.1.5 → 1.0.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6d0c0df7c314aa421d82df4122b949a0c3a1bdb10619e77de5e56453b363d339
4
- data.tar.gz: 41d110a2e0625e016ba6e9744cfa0c2cf3ae58dae9944aa0ac144b5dbc144905
3
+ metadata.gz: 64efdc1c23146264a24fb6464a2d72480cacd3db28ebc729394ac511d82dbec4
4
+ data.tar.gz: bb21c6264ee6c05c1d5ea6151f49fe81a156b2f2f9180682a2a14a5597047969
5
5
  SHA512:
6
- metadata.gz: 9abed064106ee030cd2dd04a7571f91f51f75d2a97bf0c2c15be5a36c8150f8cf910c6c8e4b47c47a659ca4e2134e45798d2c78a62fbe1e92c10adafdfde9b31
7
- data.tar.gz: ea8f2f8c8c7712cc91a7f27a47af03ebdbee2fe1366c05b185bf0293bab04d967e190c5b8914389c9c8db2ce82dd13a9b927ae57c9076a0af0cada7e36294481
6
+ metadata.gz: 6f8823df33b9cf386e2ff0188e6df0aa1a28bfb90b6b81b6e6112cc1249429cbc2548601ebf81e2970635535fb4c81b76559e203210b4a97d7a4c6513dad84a8
7
+ data.tar.gz: 93ec75aa96126b9938318f107e0b789aa5a02f557137668d563506abf24e3d78af333f334318ea114e2bfcc4e1ebdfe9a2659f5d81319a914b9c31a476ad87e4
data/.codeclimate.yml CHANGED
@@ -1,18 +1,21 @@
1
- ---
2
- exclude_paths:
3
- - ".git/"
4
- - "**.xml"
5
- - "**.yaml"
6
- - "**.yml"
7
- - "locale/"
8
- - "spec/"
9
- - "tools/"
10
- engines:
1
+ prepare:
2
+ fetch:
3
+ - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_base.yml
4
+ path: ".rubocop_base.yml"
5
+ - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_cc_base.yml
6
+ path: ".rubocop_cc_base.yml"
7
+ - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/base.yml
8
+ path: styles/base.yml
9
+ - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/cc_base.yml
10
+ path: styles/cc_base.yml
11
+ plugins:
12
+ rubocop:
13
+ enabled: true
14
+ config: ".rubocop_cc.yml"
15
+ channel: rubocop-0-82
11
16
  brakeman:
12
- # very slow :sad_panda:
13
17
  enabled: false
14
18
  bundler-audit:
15
- # requires Gemfile.lock
16
19
  enabled: false
17
20
  csslint:
18
21
  enabled: false
@@ -24,24 +27,17 @@ engines:
24
27
  - javascript
25
28
  eslint:
26
29
  enabled: false
27
- channel: "eslint-3"
30
+ channel: eslint-3
28
31
  fixme:
29
- # let's enable later
30
32
  enabled: false
31
33
  markdownlint:
32
- # let's enable later
33
34
  enabled: false
34
- rubocop:
35
- enabled: true
36
- config: '.rubocop_cc.yml'
37
- prepare:
38
- fetch:
39
- - url: "https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml"
40
- path: ".rubocop_base.yml"
41
- - url: "https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_cc_base.yml"
42
- path: ".rubocop_cc_base.yml"
43
- ratings:
44
- paths:
45
- - Gemfile.lock
46
- - "**.rake"
47
- - "**.rb"
35
+ exclude_patterns:
36
+ - ".git/"
37
+ - "**.xml"
38
+ - "**.yaml"
39
+ - "**.yml"
40
+ - locale/
41
+ - spec/
42
+ - tools/
43
+ version: '2'
data/.rubocop.yml CHANGED
@@ -1,4 +1,4 @@
1
+ inherit_gem:
2
+ manageiq-style: ".rubocop_base.yml"
1
3
  inherit_from:
2
- - https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
3
- # put all local rubocop config into .rubocop_local.yml as it will be loaded by .rubocop_cc.yml as well
4
- - .rubocop_local.yml
4
+ - ".rubocop_local.yml"
data/.rubocop_cc.yml CHANGED
@@ -1,5 +1,4 @@
1
1
  inherit_from:
2
- # this is downloaded by .codeclimate.yml
3
- - .rubocop_base.yml
4
- - .rubocop_cc_base.yml
5
- - .rubocop_local.yml
2
+ - ".rubocop_base.yml"
3
+ - ".rubocop_cc_base.yml"
4
+ - ".rubocop_local.yml"
data/.travis.yml CHANGED
@@ -1,9 +1,8 @@
1
- sudo: false
2
1
  language: ruby
3
2
  rvm:
4
- - 2.3.1
5
3
  - 2.4.5
4
+ - 2.5.7
5
+ - 2.6.5
6
6
  before_install: gem install bundler -v 1.13.0
7
7
  after_script: bundle exec codeclimate-test-reporter
8
- sudo: false
9
8
  cache: bundler
data/CHANGES CHANGED
@@ -14,3 +14,20 @@
14
14
 
15
15
  = 0.1.5 - 6-Jun-2019
16
16
  * Allow caller to provide extra headers to the message
17
+
18
+ = 0.1.6 - 6-July-2020
19
+ * Rescue message body decoding errors. Re-raise errors raised by users code of processing received messages.
20
+
21
+ = 1.0.0 - 28-Sep-2020
22
+ * Switch to use rdkafka client
23
+
24
+ = 1.0.1 - 15-Dec-2020
25
+ * Allow all kafka options to be passed
26
+ * remove sudo:false from .travis.yml
27
+ * Use manageiq-style
28
+
29
+ = 1.0.2 - 4-Jan-2021
30
+ * Rails 6.0 Support
31
+
32
+ = 1.0.3 - 12-May-2021
33
+ * Allow bulk publish of messages to a topic
data/README.md CHANGED
@@ -1,10 +1,9 @@
1
1
  # ManageIQ Messaging Client
2
2
 
3
3
  [![Gem Version](https://badge.fury.io/rb/manageiq-messaging.svg)](http://badge.fury.io/rb/manageiq-messaging)
4
- [![Build Status](https://travis-ci.org/ManageIQ/manageiq-messaging.svg)](https://travis-ci.org/ManageIQ/manageiq-messaging)
4
+ [![Build Status](https://travis-ci.com/ManageIQ/manageiq-messaging.svg)](https://travis-ci.com/ManageIQ/manageiq-messaging)
5
5
  [![Code Climate](https://codeclimate.com/github/ManageIQ/manageiq-messaging.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging)
6
6
  [![Test Coverage](https://codeclimate.com/github/ManageIQ/manageiq-messaging/badges/coverage.svg)](https://codeclimate.com/github/ManageIQ/manageiq-messaging/coverage)
7
- [![Dependency Status](https://gemnasium.com/ManageIQ/manageiq-messaging.svg)](https://gemnasium.com/ManageIQ/manageiq-messaging)
8
7
  [![Security](https://hakiri.io/github/ManageIQ/manageiq-messaging/master.svg)](https://hakiri.io/github/ManageIQ/manageiq-messaging/master)
9
8
 
10
9
  Client library for ManageIQ components to exchange messages through its internal message bus.
@@ -156,6 +155,19 @@ This is the one-to-many publish/subscribe pattern. Multiple subscribers can subs
156
155
 
157
156
  By default, events are delivered to live subscribers only. Some messaging systems support persistence with options.
158
157
 
158
+ ### Publish bulk messages to a topic
159
+
160
+ Often it is more efficient to publish messages in bulk rather than one-at-a-time. To do this you can pass an array of messages to the `publish_topic` API:
161
+
162
+ ```ruby
163
+ client.publish_topic(
164
+ [
165
+ {:service => 'provider_events', :event => 'powered_off', :payload => {:ems_ref => 'uid987', :timestamp => '1501091391'}},
166
+ {:service => 'provider_events', :event => 'powered_on', :payload => {:ems_ref => 'uid987', :timestamp => '1501091429'}},
167
+ ]
168
+ )
169
+ ```
170
+
159
171
  ### Add your own headers to a message (Queue or Topic)
160
172
 
161
173
  If you want you can add in your own headers to the send message
@@ -1,3 +1,4 @@
1
+ require 'active_support/core_ext/array/wrap'
1
2
  require 'active_support/core_ext/module/delegation'
2
3
  require 'active_support/core_ext/hash'
3
4
  require 'yaml'
@@ -40,6 +40,7 @@ module ManageIQ
40
40
  def self.open(options)
41
41
  protocol = options[:protocol] || :Stomp
42
42
  client = Object.const_get("ManageIQ::Messaging::#{protocol}::Client").new(options)
43
+
43
44
  return client unless block_given?
44
45
 
45
46
  begin
@@ -160,7 +161,8 @@ module ManageIQ
160
161
  end
161
162
 
162
163
  # Publish a message as a topic. All subscribers will receive a copy of the message.
163
- # Expected keys in +options+ are:
164
+ # +messages+ can be either a hash or an array of hashes.
165
+ # Expected keys are:
164
166
  # * :service (service is used to determine the topic address)
165
167
  # * :event (event name)
166
168
  # * :payload (message body, a string or an user object that can be serialized)
@@ -168,10 +170,11 @@ module ManageIQ
168
170
  # * :headers (optional, additional headers to add to the message)
169
171
  # Other options are underlying messaging system specific.
170
172
  #
171
- def publish_topic(options)
172
- assert_options(options, [:event, :service])
173
+ def publish_topic(messages)
174
+ messages = Array.wrap(messages)
175
+ messages.each { |msg| assert_options(msg, [:event, :service]) }
173
176
 
174
- publish_topic_impl(options)
177
+ publish_topic_impl(messages)
175
178
  end
176
179
 
177
180
  # Subscribe to receive topic type messages.
@@ -18,7 +18,7 @@ module ManageIQ
18
18
 
19
19
  def decode_body(headers, raw_body)
20
20
  return raw_body unless headers.kind_of?(Hash)
21
- case headers["encoding"]
21
+ case headers["encoding"] || headers[:encoding]
22
22
  when "json"
23
23
  JSON.parse(raw_body)
24
24
  when "yaml"
@@ -26,6 +26,10 @@ module ManageIQ
26
26
  else
27
27
  raw_body
28
28
  end
29
+ rescue => e # JSON or YAML parsing error
30
+ logger.error("Error decoding message body: #{e.message}")
31
+ logger.error(e.backtrace.join("\n"))
32
+ raw_body
29
33
  end
30
34
 
31
35
  def payload_log(payload)
@@ -10,16 +10,11 @@ module ManageIQ
10
10
  # * :hosts (Array of Kafka cluster hosts, or)
11
11
  # * :host (Single host name)
12
12
  # * :port (host port number)
13
- # * :ssl_ca_cert (security options)
14
- # * :ssl_client_cert
15
- # * :ssl_client_cert_key
16
- # * :sasl_gssapi_principal
17
- # * :sasl_gssapi_keytab
18
- # * :sasl_plain_username
19
- # * :sasl_plain_password
20
- # * :sasl_scram_username
21
- # * :sasl_scram_password
22
- # * :sasl_scram_mechanism
13
+ #
14
+ # For additional security options, please refer to
15
+ # https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka and
16
+ # https://github.com/edenhill/librdkafka/wiki/Using-SASL-with-librdkafka
17
+ #
23
18
  #
24
19
  # Kafka specific +publish_message+ options:
25
20
  # * :group_name (Used as Kafka partition_key)
@@ -42,7 +37,7 @@ module ManageIQ
42
37
  #
43
38
  # +subscribe_background_job+ is currently not implemented.
44
39
  class Client < ManageIQ::Messaging::Client
45
- require 'kafka'
40
+ require 'rdkafka'
46
41
  require 'manageiq/messaging/kafka/common'
47
42
  require 'manageiq/messaging/kafka/queue'
48
43
  require 'manageiq/messaging/kafka/background_job'
@@ -53,27 +48,27 @@ module ManageIQ
53
48
  include BackgroundJob
54
49
  include Topic
55
50
 
56
- private *delegate(:subscribe, :unsubscribe, :publish, :to => :kafka_client)
57
- delegate :close, :to => :kafka_client
58
-
59
51
  attr_accessor :encoding
60
52
 
61
53
  def ack(ack_ref)
62
- @queue_consumer.try(:mark_message_as_processed, ack_ref)
63
- @topic_consumer.try(:mark_message_as_processed, ack_ref)
54
+ ack_ref.commit
55
+ rescue Rdkafka::RdkafkaError => e
56
+ logger.warn("ack failed with error #{e.message}")
57
+ raise unless e.message =~ /no_offset/
64
58
  end
65
59
 
66
60
  def close
67
- @topic_consumer.try(:stop)
68
- @topic_consumer = nil
69
- @queue_consumer.try(:stop)
70
- @queue_consumer = nil
71
-
72
- @producer.try(:shutdown)
61
+ @producer&.close
73
62
  @producer = nil
74
63
 
75
- kafka_client.close
76
- @kafka_client = nil
64
+ @consumer&.close
65
+ @consumer = nil
66
+ end
67
+
68
+ # list all topics
69
+ def topics
70
+ native_kafka = producer.instance_variable_get(:@native_kafka)
71
+ Rdkafka::Metadata.new(native_kafka).topics.collect { |topic| topic[:topic_name] }
77
72
  end
78
73
 
79
74
  private
@@ -81,18 +76,23 @@ module ManageIQ
81
76
  attr_reader :kafka_client
82
77
 
83
78
  def initialize(options)
84
- hosts = Array(options[:hosts] || options[:host])
85
- hosts.collect! { |host| "#{host}:#{options[:port]}" }
86
-
87
79
  @encoding = options[:encoding] || 'yaml'
88
80
  require "json" if @encoding == "json"
89
81
 
90
- connection_opts = {}
91
- connection_opts[:client_id] = options[:client_ref] if options[:client_ref]
82
+ ::Rdkafka::Config.logger = logger
83
+ @kafka_client = ::Rdkafka::Config.new(rdkafka_connection_opts(options))
84
+ end
85
+
86
+ def rdkafka_connection_opts(options)
87
+ hosts = Array(options[:hosts] || options[:host])
88
+ hosts.collect! { |host| "#{host}:#{options[:port]}" }
92
89
 
93
- connection_opts.merge!(options.slice(:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :sasl_gssapi_principal, :sasl_gssapi_keytab, :sasl_plain_username, :sasl_plain_password, :sasl_scram_username, :sasl_scram_password, :sasl_scram_mechanism))
90
+ result = {:"bootstrap.servers" => hosts.join(',')}
91
+ result[:"client.id"] = options[:client_ref] if options[:client_ref]
92
+ result[:"sasl.username"] = options[:username] if options[:username]
93
+ result[:"sasl.password"] = options[:password] if options[:password]
94
94
 
95
- @kafka_client = ::Kafka.new(hosts, connection_opts)
95
+ result.merge(options.except(:port, :host, :hosts, :encoding, :protocol, :client_ref, :username, :password))
96
96
  end
97
97
  end
98
98
  end
@@ -5,48 +5,25 @@ module ManageIQ
5
5
  require 'manageiq/messaging/common'
6
6
  include ManageIQ::Messaging::Common
7
7
 
8
- GROUP_FOR_QUEUE_MESSAGES = 'manageiq_messaging_queue_group_'.freeze
9
-
10
8
  private
11
9
 
12
10
  def producer
13
11
  @producer ||= kafka_client.producer
14
12
  end
15
13
 
16
- def topic_consumer(persist_ref, session_timeout = nil)
17
- # persist_ref enables consumer to receive messages sent when consumer is temporarily offline
18
- # it also enables consumers to do load balancing when multiple consumers join the with the same ref.
19
- @topic_consumer.try(:stop) unless @persist_ref == persist_ref
20
- @persist_ref = persist_ref
21
-
22
- consumer_opts = {:group_id => persist_ref}
23
- consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
24
-
25
- @topic_consumer ||= kafka_client.consumer(consumer_opts)
26
- end
27
-
28
- def queue_consumer(topic, session_timeout = nil)
29
- # all queue consumers join the same group so that each message can be processed by one and only one consumer
30
- @queue_consumer.try(:stop) unless @queue_topic == topic
31
- @queue_topic = topic
32
-
33
- consumer_opts = {:group_id => GROUP_FOR_QUEUE_MESSAGES + topic}
34
- consumer_opts[:session_timeout] = session_timeout if session_timeout.present?
35
-
36
- @queue_consumer ||= kafka_client.consumer(consumer_opts)
37
- end
38
-
39
- trap("TERM") do
40
- @topic_consumer.try(:stop)
41
- @topic_consumer = nil
42
- @queue_consumer.try(:stop)
43
- @queue_consumer = nil
14
+ def consumer(beginning, options)
15
+ @consumer&.close
16
+ kafka_client[:"group.id"] = options[:persist_ref]
17
+ kafka_client[:"auto.offset.reset"] = beginning ? 'smallest' : 'largest'
18
+ kafka_client[:"enable.auto.commit"] = !!auto_ack?(options)
19
+ kafka_client[:"session.timeout.ms"] = options[:session_timeout] * 1000 if options[:session_timeout].present?
20
+ kafka_client[:"group.instance.id"] = options[:group_instance_id] if options[:group_instance_id].present?
21
+ @consumer = kafka_client.consumer
44
22
  end
45
23
 
46
- def raw_publish(commit, body, options)
47
- producer.produce(encode_body(options[:headers], body), options)
48
- producer.deliver_messages if commit
49
- logger.info("Published to topic(#{options[:topic]}), msg(#{payload_log(body.inspect)})")
24
+ def raw_publish(body, options)
25
+ options[:payload] = encode_body(options[:headers], body)
26
+ producer.produce(options)
50
27
  end
51
28
 
52
29
  def queue_for_publish(options)
@@ -68,8 +45,8 @@ module ManageIQ
68
45
 
69
46
  def for_publish(options)
70
47
  kafka_opts = {:topic => address(options)}
71
- kafka_opts[:partition_key] = options[:group_name] if options[:group_name]
72
- kafka_opts[:headers] = {}
48
+ kafka_opts[:partition_key] = options[:group_name] if options[:group_name]
49
+ kafka_opts[:headers] = {}
73
50
  kafka_opts[:headers][:sender] = options[:sender] if options[:sender]
74
51
 
75
52
  body = options[:payload] || ''
@@ -85,23 +62,30 @@ module ManageIQ
85
62
  end
86
63
  end
87
64
 
88
- def process_queue_message(queue, message)
89
- payload = decode_body(message.headers, message.value)
90
- sender, message_type, class_name = parse_message_headers(message.headers)
91
- client_headers = message.headers.except(*message_header_keys)
65
+ def process_queue_message(queue_consumer, queue, message)
66
+ begin
67
+ payload = decode_body(message.headers, message.payload)
68
+ sender, message_type, _class_name = parse_message_headers(message.headers)
69
+ client_headers = message.headers.except(*message_header_keys).with_indifferent_access
92
70
 
93
- logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
94
- [sender, message_type, class_name, payload, client_headers]
71
+ logger.info("Message received: queue(#{queue}), message(#{payload_log(payload)}), sender(#{sender}), type(#{message_type})")
72
+ yield [ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, client_headers, queue_consumer, self)]
73
+ logger.info("Messsage processed")
74
+ rescue StandardError => e
75
+ logger.error("Message processing error: #{e.message}")
76
+ logger.error(e.backtrace.join("\n"))
77
+ raise
78
+ end
95
79
  end
96
80
 
97
- def process_topic_message(topic, message)
81
+ def process_topic_message(topic_consumer, topic, message)
98
82
  begin
99
- payload = decode_body(message.headers, message.value)
83
+ payload = decode_body(message.headers, message.payload)
100
84
  sender, event_type = parse_event_headers(message.headers)
101
- client_headers = message.headers.except(*event_header_keys)
85
+ client_headers = message.headers.except(*event_header_keys).with_indifferent_access
102
86
 
103
87
  logger.info("Event received: topic(#{topic}), event(#{payload_log(payload)}), sender(#{sender}), type(#{event_type})")
104
- yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers, message, self)
88
+ yield ManageIQ::Messaging::ReceivedMessage.new(sender, event_type, payload, client_headers, topic_consumer, self)
105
89
  logger.info("Event processed")
106
90
  rescue StandardError => e
107
91
  logger.error("Event processing error: #{e.message}")
@@ -111,7 +95,7 @@ module ManageIQ
111
95
  end
112
96
 
113
97
  def message_header_keys
114
- ['sender', 'message_type', 'class_name']
98
+ [:sender, :message_type, :class_name]
115
99
  end
116
100
 
117
101
  def parse_message_headers(headers)
@@ -120,12 +104,12 @@ module ManageIQ
120
104
  end
121
105
 
122
106
  def event_header_keys
123
- ['sender', 'event_type']
107
+ [:sender, :event_type]
124
108
  end
125
109
 
126
110
  def parse_event_headers(headers)
127
111
  return [nil, nil] unless headers.kind_of?(Hash)
128
- headers.values_at('sender', 'event_type')
112
+ headers.values_at(*event_header_keys)
129
113
  end
130
114
  end
131
115
  end
@@ -2,42 +2,28 @@ module ManageIQ
2
2
  module Messaging
3
3
  module Kafka
4
4
  module Queue
5
+ GROUP_FOR_QUEUE_MESSAGES = ENV['QUEUE_MESSAGES_GROUP_PREFIX'].freeze || 'manageiq_messaging_queue_group_'.freeze
6
+
5
7
  private
6
8
 
7
9
  def publish_message_impl(options)
8
10
  raise ArgumentError, "Kafka messaging implementation does not take a block" if block_given?
9
- raw_publish(true, *queue_for_publish(options))
11
+ raw_publish(*queue_for_publish(options)).wait
10
12
  end
11
13
 
12
14
  def publish_messages_impl(messages)
13
- messages.each { |msg_options| raw_publish(false, *queue_for_publish(msg_options)) }
14
- producer.deliver_messages
15
+ handles = messages.collect { |msg_options| raw_publish(*queue_for_publish(msg_options)) }
16
+ handles.each(&:wait)
15
17
  end
16
18
 
17
- def subscribe_messages_impl(options)
19
+ def subscribe_messages_impl(options, &block)
18
20
  topic = address(options)
19
- session_timeout = options[:session_timeout]
20
-
21
- batch_options = {}
22
- batch_options[:automatically_mark_as_processed] = auto_ack?(options)
23
- batch_options[:max_bytes] = options[:max_bytes] if options.key?(:max_bytes)
24
-
25
- consumer = queue_consumer(topic, session_timeout)
26
- consumer.subscribe(topic)
27
- consumer.each_batch(batch_options) do |batch|
28
- logger.info("Batch message received: queue(#{topic})")
29
- begin
30
- messages = batch.messages.collect do |message|
31
- sender, message_type, _class_name, payload = process_queue_message(topic, message)
32
- ManageIQ::Messaging::ReceivedMessage.new(sender, message_type, payload, headers, message, self)
33
- end
21
+ options[:persist_ref] = GROUP_FOR_QUEUE_MESSAGES + topic
34
22
 
35
- yield messages
36
- rescue StandardError => e
37
- logger.error("Event processing error: #{e.message}")
38
- logger.error(e.backtrace.join("\n"))
39
- end
40
- logger.info("Batch message processed")
23
+ queue_consumer = consumer(true, options)
24
+ queue_consumer.subscribe(topic)
25
+ queue_consumer.each do |message|
26
+ process_queue_message(queue_consumer, topic, message, &block)
41
27
  end
42
28
  end
43
29
  end
@@ -1,28 +1,26 @@
1
+ require 'socket'
2
+
1
3
  module ManageIQ
2
4
  module Messaging
3
5
  module Kafka
4
6
  module Topic
7
+ GROUP_FOR_ADHOC_LISTENERS = Socket.gethostname.freeze
8
+
5
9
  private
6
10
 
7
- def publish_topic_impl(options)
8
- raw_publish(true, *topic_for_publish(options))
11
+ def publish_topic_impl(messages)
12
+ handles = messages.collect { |message| raw_publish(*topic_for_publish(message)) }
13
+ handles.each(&:wait)
9
14
  end
10
15
 
11
16
  def subscribe_topic_impl(options, &block)
12
17
  topic = address(options)
13
- persist_ref = options[:persist_ref]
14
- session_timeout = options[:session_timeout]
15
18
 
16
- if persist_ref
17
- consumer = topic_consumer(persist_ref, session_timeout)
18
- consumer.subscribe(topic, :start_from_beginning => false)
19
- consumer.each_message(:automatically_mark_as_processed => auto_ack?(options)) do |message|
20
- process_topic_message(topic, message, &block)
21
- end
22
- else
23
- kafka_client.each_message(:topic => topic, :start_from_beginning => false) do |message|
24
- process_topic_message(topic, message, &block)
25
- end
19
+ options[:persist_ref] = "#{GROUP_FOR_ADHOC_LISTENERS}_#{Time.now.to_i}" unless options[:persist_ref]
20
+ topic_consumer = consumer(false, options)
21
+ topic_consumer.subscribe(topic)
22
+ topic_consumer.each do |message|
23
+ process_topic_message(topic_consumer, topic, message, &block)
26
24
  end
27
25
  end
28
26
  end
@@ -52,6 +52,7 @@ module ManageIQ
52
52
  rescue => e
53
53
  logger.error("Message processing error: #{e.message}")
54
54
  logger.error(e.backtrace.join("\n"))
55
+ raise
55
56
  end
56
57
  end
57
58
  end
@@ -4,7 +4,7 @@ module ManageIQ
4
4
  module Topic
5
5
  private
6
6
 
7
- def publish_topic_impl(options)
7
+ def publish_topic_single(options)
8
8
  address, headers = topic_for_publish(options)
9
9
  headers[:sender] = options[:sender] if options[:sender]
10
10
  headers[:event_type] = options[:event] if options[:event]
@@ -12,6 +12,10 @@ module ManageIQ
12
12
  raw_publish(address, options[:payload], headers)
13
13
  end
14
14
 
15
+ def publish_topic_impl(messages)
16
+ messages.each { |message| publish_topic_single(message) }
17
+ end
18
+
15
19
  def subscribe_topic_impl(options)
16
20
  queue_name, headers = topic_for_subscribe(options)
17
21
 
@@ -30,6 +34,7 @@ module ManageIQ
30
34
  rescue => e
31
35
  logger.error("Event processing error: #{e.message}")
32
36
  logger.error(e.backtrace.join("\n"))
37
+ raise
33
38
  end
34
39
  end
35
40
  end
@@ -1,5 +1,5 @@
1
1
  module ManageIQ
2
2
  module Messaging
3
- VERSION = "0.1.5"
3
+ VERSION = "1.0.3"
4
4
  end
5
5
  end
@@ -4,9 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
  require 'manageiq/messaging/version'
5
5
 
6
6
  Gem::Specification.new do |spec|
7
- spec.name = "manageiq-messaging"
8
- spec.version = ManageIQ::Messaging::VERSION
9
- spec.authors = ["ManageIQ Authors"]
7
+ spec.name = "manageiq-messaging"
8
+ spec.version = ManageIQ::Messaging::VERSION
9
+ spec.required_ruby_version = '>= 2.4'
10
+ spec.authors = ["ManageIQ Authors"]
10
11
 
11
12
  spec.summary = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
12
13
  spec.description = 'Client library for ManageIQ components to exchange messages through its internal message bus.'
@@ -20,15 +21,14 @@ Gem::Specification.new do |spec|
20
21
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
22
  spec.require_paths = ["lib"]
22
23
 
23
- spec.add_dependency 'activesupport', '>= 4.2.2'
24
- spec.add_dependency 'ruby-kafka', '~> 0.7.0'
24
+ spec.add_dependency 'activesupport', '>= 5.2.4.3', "< 6.1"
25
+ spec.add_dependency 'rdkafka', '~> 0.8'
25
26
  spec.add_dependency 'stomp', '~> 1.4.4'
26
27
 
27
28
  spec.add_development_dependency "bundler"
28
29
  spec.add_development_dependency "codeclimate-test-reporter", "~> 1.0.0"
29
- spec.add_development_dependency "rake", "~> 10.0"
30
+ spec.add_development_dependency "manageiq-style"
31
+ spec.add_development_dependency "rake", ">= 12.3.3"
30
32
  spec.add_development_dependency "rspec", "~> 3.0"
31
- spec.add_development_dependency "rubocop"
32
- spec.add_development_dependency "rubocop-performance"
33
33
  spec.add_development_dependency "simplecov"
34
34
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: manageiq-messaging
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 1.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - ManageIQ Authors
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-06-12 00:00:00.000000000 Z
11
+ date: 2021-05-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -16,28 +16,34 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 4.2.2
19
+ version: 5.2.4.3
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '6.1'
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
27
  - - ">="
25
28
  - !ruby/object:Gem::Version
26
- version: 4.2.2
29
+ version: 5.2.4.3
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '6.1'
27
33
  - !ruby/object:Gem::Dependency
28
- name: ruby-kafka
34
+ name: rdkafka
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
31
37
  - - "~>"
32
38
  - !ruby/object:Gem::Version
33
- version: 0.7.0
39
+ version: '0.8'
34
40
  type: :runtime
35
41
  prerelease: false
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
44
  - - "~>"
39
45
  - !ruby/object:Gem::Version
40
- version: 0.7.0
46
+ version: '0.8'
41
47
  - !ruby/object:Gem::Dependency
42
48
  name: stomp
43
49
  requirement: !ruby/object:Gem::Requirement
@@ -81,61 +87,47 @@ dependencies:
81
87
  - !ruby/object:Gem::Version
82
88
  version: 1.0.0
83
89
  - !ruby/object:Gem::Dependency
84
- name: rake
85
- requirement: !ruby/object:Gem::Requirement
86
- requirements:
87
- - - "~>"
88
- - !ruby/object:Gem::Version
89
- version: '10.0'
90
- type: :development
91
- prerelease: false
92
- version_requirements: !ruby/object:Gem::Requirement
93
- requirements:
94
- - - "~>"
95
- - !ruby/object:Gem::Version
96
- version: '10.0'
97
- - !ruby/object:Gem::Dependency
98
- name: rspec
90
+ name: manageiq-style
99
91
  requirement: !ruby/object:Gem::Requirement
100
92
  requirements:
101
- - - "~>"
93
+ - - ">="
102
94
  - !ruby/object:Gem::Version
103
- version: '3.0'
95
+ version: '0'
104
96
  type: :development
105
97
  prerelease: false
106
98
  version_requirements: !ruby/object:Gem::Requirement
107
99
  requirements:
108
- - - "~>"
100
+ - - ">="
109
101
  - !ruby/object:Gem::Version
110
- version: '3.0'
102
+ version: '0'
111
103
  - !ruby/object:Gem::Dependency
112
- name: rubocop
104
+ name: rake
113
105
  requirement: !ruby/object:Gem::Requirement
114
106
  requirements:
115
107
  - - ">="
116
108
  - !ruby/object:Gem::Version
117
- version: '0'
109
+ version: 12.3.3
118
110
  type: :development
119
111
  prerelease: false
120
112
  version_requirements: !ruby/object:Gem::Requirement
121
113
  requirements:
122
114
  - - ">="
123
115
  - !ruby/object:Gem::Version
124
- version: '0'
116
+ version: 12.3.3
125
117
  - !ruby/object:Gem::Dependency
126
- name: rubocop-performance
118
+ name: rspec
127
119
  requirement: !ruby/object:Gem::Requirement
128
120
  requirements:
129
- - - ">="
121
+ - - "~>"
130
122
  - !ruby/object:Gem::Version
131
- version: '0'
123
+ version: '3.0'
132
124
  type: :development
133
125
  prerelease: false
134
126
  version_requirements: !ruby/object:Gem::Requirement
135
127
  requirements:
136
- - - ">="
128
+ - - "~>"
137
129
  - !ruby/object:Gem::Version
138
- version: '0'
130
+ version: '3.0'
139
131
  - !ruby/object:Gem::Dependency
140
132
  name: simplecov
141
133
  requirement: !ruby/object:Gem::Requirement
@@ -152,7 +144,7 @@ dependencies:
152
144
  version: '0'
153
145
  description: Client library for ManageIQ components to exchange messages through its
154
146
  internal message bus.
155
- email:
147
+ email:
156
148
  executables: []
157
149
  extensions: []
158
150
  extra_rdoc_files: []
@@ -200,7 +192,7 @@ homepage: http://github.com/ManageIQ/manageiq-messaging
200
192
  licenses:
201
193
  - MIT
202
194
  metadata: {}
203
- post_install_message:
195
+ post_install_message:
204
196
  rdoc_options: []
205
197
  require_paths:
206
198
  - lib
@@ -208,16 +200,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
208
200
  requirements:
209
201
  - - ">="
210
202
  - !ruby/object:Gem::Version
211
- version: '0'
203
+ version: '2.4'
212
204
  required_rubygems_version: !ruby/object:Gem::Requirement
213
205
  requirements:
214
206
  - - ">="
215
207
  - !ruby/object:Gem::Version
216
208
  version: '0'
217
209
  requirements: []
218
- rubyforge_project:
219
- rubygems_version: 2.7.6.2
220
- signing_key:
210
+ rubygems_version: 3.2.5
211
+ signing_key:
221
212
  specification_version: 4
222
213
  summary: Client library for ManageIQ components to exchange messages through its internal
223
214
  message bus.