receptor_controller-client 0.0.2 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1545985069380f37a6e78393b779ec86c3d189065c2a7086098128f5ff1f7f42
4
- data.tar.gz: 0a8a95476224fa7ed5f0f7cbca7b1b034b6ad2b0f8ae89a07ff987e80ffbdd4a
3
+ metadata.gz: ab72f29ace5c188996000048957bb58020940d3b8ce32b1a0b720602f4ff7e64
4
+ data.tar.gz: 4b68d648d59809f80e108a1d998af10d09fee233db085e02b736c691db05b8d5
5
5
  SHA512:
6
- metadata.gz: d78e3987b4436a930fa9bc8e5c6a727480202060ce3d9bd48b7e1b7bbe3b60874ea57191e0cd4610257a5c0b4ae1c747e01b662ca356b7050176f4727c06680f
7
- data.tar.gz: c7a7934d2aacf9fd83ce41fb38d5e536f380378e51857a3b1dc0a6ca0192a0073ff07dd7dccb8091fff65850ff2cc6d2f60aafa4308d4e78b7f69efa9e30fad4
6
+ metadata.gz: 32b8c1c4a2173e38ec229bad3b4f0c6747c9f9cc50ce3be86746217a6682b60e4c820e9677adc38798548d81bb62db858994a7019cb85fc9b58c149eb0357c2e
7
+ data.tar.gz: abfdf8a4c9924f4ba225310f5439b97f003d4ac01eedd14a822328d7750289a60cc9ec49a086ac125f446adbd0c84840afc02562a84c3c02c19f6402b201a469
@@ -5,6 +5,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5
5
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
6
 
7
7
  ## [Unreleased]
8
+ ## [0.0.7]
9
+ - rdkafka changes #32
10
+
11
+ ## [0.0.6]
12
+ - Waiting for response-worker kafka init #20
13
+ - Fix href_slug in directives log #29
14
+
15
+ ## [0.0.5] - 2020-08-11
16
+ ### Added
17
+ - Unique Persist-ref for every response worker #18
18
+
19
+ ## [0.0.4] - 2020-08-04
20
+ ### Added
21
+ - Default Kafka ACK to true
22
+
23
+ ## [0.0.3] - 2020-08-03
24
+ ### Added
25
+ - Do not send EOF message until all messages have been received
8
26
 
9
27
  ## [0.0.2] - 2020-07-31
10
28
  ### Initial release to rubygems.org
@@ -12,5 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
12
30
  ### Added
13
31
  - Added extra logging for messages
14
32
 
15
- [Unreleased]: https://github.com/RedHatInsights/receptor_controller-client-ruby/compare/v0.0.2...HEAD
33
+ [Unreleased]: https://github.com/RedHatInsights/receptor_controller-client-ruby/compare/v0.0.7...HEAD
34
+ [0.0.6]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.7
35
+ [0.0.6]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.6
36
+ [0.0.5]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.5
37
+ [0.0.4]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.4
38
+ [0.0.3]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.3
16
39
  [0.0.2]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.2
@@ -39,10 +39,10 @@ module ReceptorController
39
39
  @job_path = '/job'
40
40
  @pre_shared_key = nil
41
41
 
42
- @queue_auto_ack = false
42
+ @queue_auto_ack = true
43
43
  @queue_host = nil
44
44
  @queue_max_bytes = nil
45
- @queue_persist_ref = nil
45
+ @queue_persist_ref = "receptor-client.#{SecureRandom.uuid}"
46
46
  @queue_port = nil
47
47
  @queue_topic = 'platform.receptor-controller.responses'
48
48
 
@@ -20,7 +20,7 @@ module ReceptorController
20
20
 
21
21
  msg_id = JSON.parse(response.body)['id']
22
22
 
23
- logger.debug("Receptor response: registering message #{msg_id}".tap { |msg| msg << " req: #{body["href_slug"]}" if ENV["LOG_ALL_RECEPTOR_MESSAGES"]&.to_i != 0 })
23
+ logger.debug("Receptor response [#{ReceptorController::Client::Configuration.default.queue_persist_ref}]: registering message #{msg_id}, href_slug: #{log_message_common}")
24
24
  # registers message id for kafka responses
25
25
  response_worker.register_message(msg_id, self)
26
26
  wait_for_response(msg_id)
@@ -48,6 +48,7 @@ module ReceptorController
48
48
 
49
49
  # registers message id for kafka responses
50
50
  response_worker.register_message(msg_id, self)
51
+ logger.debug("Receptor response [#{ReceptorController::Client::Configuration.default.queue_persist_ref}]: registering message #{msg_id}, href_slug: #{log_message_common}")
51
52
 
52
53
  msg_id
53
54
  else
@@ -1,3 +1,4 @@
1
+ require 'active_support/notifications'
1
2
  require 'base64'
2
3
  require "concurrent"
3
4
  require 'stringio'
@@ -18,6 +19,10 @@ module ReceptorController
18
19
  #
19
20
  # Use "start" and "stop" methods to start/stop listening on Kafka
20
21
  class Client::ResponseWorker
22
+ EOF = "eof".freeze
23
+ RESPONSE = "response".freeze
24
+ INITIALIZATION_TIMEOUT = 20
25
+
21
26
  attr_reader :started
22
27
  alias started? started
23
28
 
@@ -36,13 +41,19 @@ module ReceptorController
36
41
 
37
42
  # Start listening on Kafka
38
43
  def start
44
+ init_lock, init_wait = Mutex.new, ConditionVariable.new
45
+
39
46
  lock.synchronize do
40
47
  return if started.value
41
48
 
49
+ default_messaging_opts # Thread-safe init
50
+
42
51
  started.value = true
43
52
  workers[:maintenance] = Thread.new { check_timeouts while started.value }
44
53
  workers[:listener] = Thread.new { listen while started.value }
45
54
  end
55
+
56
+ logger.info("Receptor Response worker started...")
46
57
  end
47
58
 
48
59
  # Stop listener
@@ -81,12 +92,11 @@ module ReceptorController
81
92
  # Open a connection to the messaging service
82
93
  client = ManageIQ::Messaging::Client.open(default_messaging_opts)
83
94
 
84
- logger.info("Receptor Response worker started...")
85
95
  client.subscribe_topic(queue_opts) do |message|
86
96
  process_message(message)
87
97
  end
88
98
  rescue => err
89
- logger.error("Exception in kafka listener: #{err}\n#{err.backtrace.join("\n")}")
99
+ logger.error(response_log("Exception in kafka listener: #{err}\n#{err.backtrace.join("\n")}"))
90
100
  ensure
91
101
  client&.close
92
102
  end
@@ -95,7 +105,7 @@ module ReceptorController
95
105
  response = JSON.parse(message.payload)
96
106
 
97
107
  if (message_id = response['in_response_to'])
98
- logger.debug("Receptor response: Received message_id: #{message_id}")
108
+ logger.debug(response_log("Received message #{message_id}: serial: #{response["serial"]}, type: #{response['message_type']}, payload: #{response['payload'] || "n/a"}"))
99
109
  if (callbacks = registered_messages[message_id]).present?
100
110
  # Reset last_checked_at to avoid timeout in multi-response messages
101
111
  reset_last_checked_at(callbacks)
@@ -105,35 +115,52 @@ module ReceptorController
105
115
  # Response OK
106
116
  #
107
117
  message_type = response['message_type'] # "response" (with data) or "eof" (without data)
108
- registered_messages.delete(message_id) if message_type == 'eof'
109
-
110
118
  payload = response['payload']
111
- payload = unpack_payload(payload) if message_type == 'response' && payload.kind_of?(String)
112
-
113
- logger.debug("Receptor response: OK | message #{message_id} (#{payload})")
114
-
115
- callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
119
+ callbacks[:received_msgs] ? callbacks[:received_msgs] += 1 : callbacks[:received_msgs] = 1
120
+
121
+ case message_type
122
+ when EOF
123
+ # Store how many messages are needed to be received for this request
124
+ callbacks[:total_msgs] = response["serial"]
125
+ when RESPONSE
126
+ payload = unpack_payload(payload) if payload.kind_of?(String)
127
+ callbacks[:msg_size] ? callbacks[:msg_size] += payload.size : callbacks[:msg_size] = payload.size
128
+ callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
129
+ else
130
+ # Send the callback to release the thread.
131
+ logger.warn(response_log("Unexpected type | message #{message_id}, type: #{message_type}"))
132
+ callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
133
+ end
134
+
135
+ # We received all the messages, complete the message.
136
+ if callbacks[:received_msgs] == callbacks[:total_msgs]
137
+ registered_messages.delete(message_id)
138
+ callbacks[:receiver].send(callbacks[:response_callback], message_id, EOF, payload)
139
+ logger.debug(response_log("Message #{message_id} complete, total bytes: #{callbacks[:msg_size]}"))
140
+ end
141
+
142
+ logger.debug(response_log("OK | message: #{message_id}, serial: #{response["serial"]}, type: #{message_type}, payload: #{payload || "n/a"}"))
116
143
  else
117
144
  #
118
145
  # Response Error
119
146
  #
120
147
  registered_messages.delete(message_id)
121
148
 
122
- logger.error("Receptor response: ERROR | message #{message_id} (#{response})")
149
+ logger.error(response_log("ERROR | message #{message_id} (#{response})"))
123
150
 
124
151
  callbacks[:receiver].send(callbacks[:error_callback], message_id, response['code'], response['payload'])
125
152
  end
126
153
  elsif ENV["LOG_ALL_RECEPTOR_MESSAGES"]&.to_i != 0
127
154
  # noop, it's not error if not registered, can be processed by another pod
128
- logger.debug("Receptor response unhandled: #{message_id} (#{response['code']})")
155
+ logger.debug(response_log("NOT REGISTERED | #{message_id} (#{response['code']})"))
129
156
  end
130
157
  else
131
- logger.error("Receptor response: Message id (in_response_to) not received! #{response}")
158
+ logger.error(response_log("MISSING | Message id (in_response_to) not received! #{response}"))
132
159
  end
133
160
  rescue JSON::ParserError => e
134
- logger.error("Receptor response: Failed to parse Kafka response (#{e.message})\n#{message.payload}")
161
+ logger.error(response_log("Failed to parse Kafka response (#{e.message})\n#{message.payload}"))
135
162
  rescue => e
136
- logger.error("Receptor response: #{e}\n#{e.backtrace.join("\n")}")
163
+ logger.error(response_log("#{e}\n#{e.backtrace.join("\n")}"))
137
164
  ensure
138
165
  message.ack unless config.queue_auto_ack
139
166
  end
@@ -194,21 +221,30 @@ module ReceptorController
194
221
  end
195
222
 
196
223
  # No persist_ref here, because all instances (pods) needs to receive kafka message
224
+ # TODO: temporary changed to unique persist_ref
197
225
  def queue_opts
198
- opts = {:service => config.queue_topic,
199
- :auto_ack => config.queue_auto_ack}
200
- opts[:max_bytes] = config.queue_max_bytes if config.queue_max_bytes
201
- opts[:persist_ref] = config.queue_persist_ref if config.queue_persist_ref
202
- opts
226
+ return @queue_opts if @queue_opts
227
+
228
+ @queue_opts = {:service => config.queue_topic,
229
+ :auto_ack => config.queue_auto_ack}
230
+ @queue_opts[:max_bytes] = config.queue_max_bytes if config.queue_max_bytes
231
+ @queue_opts[:persist_ref] = ENV['HOSTNAME']
232
+ @queue_opts
203
233
  end
204
234
 
205
235
  def default_messaging_opts
206
- {
236
+ return @default_messaging_opts if @default_messaging_opts
237
+
238
+ @default_messaging_opts = {
207
239
  :host => config.queue_host,
208
240
  :port => config.queue_port,
209
241
  :protocol => :Kafka,
210
242
  :client_ref => "receptor_client-responses-#{Time.now.to_i}", # A reference string to identify the client
211
243
  }
212
244
  end
245
+
246
+ def response_log(message)
247
+ "Receptor Response [#{queue_opts[:persist_ref]}]: #{message}"
248
+ end
213
249
  end
214
250
  end
@@ -1,5 +1,5 @@
1
1
  module ReceptorController
2
2
  class Client
3
- VERSION = "0.0.2".freeze
3
+ VERSION = "0.0.7".freeze
4
4
  end
5
5
  end
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
24
24
  s.add_runtime_dependency 'faraday', '~> 1.0'
25
25
  s.add_runtime_dependency 'json', '~> 2.3', '>= 2.3.0'
26
26
  s.add_runtime_dependency 'manageiq-loggers', '~> 0.5.0'
27
- s.add_runtime_dependency 'manageiq-messaging', '~> 0.1.5'
27
+ s.add_runtime_dependency 'manageiq-messaging', '~> 1.0.0'
28
28
 
29
29
  s.add_development_dependency 'bundler', '~> 2.0'
30
30
  s.add_development_dependency 'rake', '>= 12.3.3'
@@ -32,6 +32,10 @@ RSpec.describe ReceptorController::Client::DirectiveBlocking do
32
32
 
33
33
  subject { described_class.new(:name => directive, :account => external_tenant, :node_id => receptor_node, :payload => payload, :client => receptor_client) }
34
34
 
35
+ before do
36
+ allow(subject.response_worker).to receive_messages(:init_notifications => nil, :wait_for_notifications => nil)
37
+ end
38
+
35
39
  describe "#call" do
36
40
  it "makes POST /job request to receptor and registers received message ID" do
37
41
  allow(subject).to receive(:wait_for_response)
@@ -106,7 +110,8 @@ RSpec.describe ReceptorController::Client::DirectiveBlocking do
106
110
  response_message = {'code' => 0,
107
111
  'in_response_to' => http_response['id'],
108
112
  'message_type' => subject.class::MESSAGE_TYPE_EOF,
109
- 'payload' => 'Unimportant'}
113
+ 'payload' => 'Unimportant',
114
+ 'serial' => 1}
110
115
 
111
116
  allow(kafka_response).to receive(:payload).and_return(response_message.to_json)
112
117
 
@@ -101,13 +101,13 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
101
101
  # Kafka response - 'response' type
102
102
  expect(subject).to receive(:response_success).twice.and_call_original
103
103
  message_type, payload = 'response', 'Testing payload'
104
- response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload}
104
+ response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload, :serial => 1}
105
105
  allow(response).to receive(:payload).and_return(response_payload.to_json)
106
106
  subject.response_worker.send(:process_message, response)
107
107
 
108
108
  # Kafka response - 'eof' type
109
109
  message_type = 'eof'
110
- response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil}
110
+ response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil, :serial => 2}
111
111
  allow(response).to receive(:payload).and_return(response_payload.to_json)
112
112
  subject.response_worker.send(:process_message, response)
113
113
 
@@ -142,7 +142,7 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
142
142
 
143
143
  # Kafka response - 'response' type - success
144
144
  message_type, payload = 'response', 'Testing payload'
145
- response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload}
145
+ response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload, :serial => 1}
146
146
  allow(response).to receive(:payload).and_return(response_payload.to_json)
147
147
  success_calls.times do
148
148
  subject.response_worker.send(:process_message, response)
@@ -150,7 +150,7 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
150
150
 
151
151
  # Kafka response - 'eof' type
152
152
  message_type = 'eof'
153
- response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil}
153
+ response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil, :serial => 4}
154
154
  allow(response).to receive(:payload).and_return(response_payload.to_json)
155
155
  subject.response_worker.send(:process_message, response)
156
156
 
@@ -9,7 +9,13 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
9
9
 
10
10
  before do
11
11
  allow(logger).to receive_messages(%i[debug info warn error fatal])
12
- allow(config).to receive_messages(:queue_auto_ack => false, :response_timeout => 0, :response_timeout_poll_time => 0)
12
+ allow(config).to receive_messages(:queue_auto_ack => false,
13
+ :queue_max_bytes => nil,
14
+ :queue_persist_ref => 'consumer-group-1',
15
+ :queue_topic => 'receptor.kafka.topic',
16
+ :response_timeout => 0,
17
+ :response_timeout_poll_time => 0,
18
+ )
13
19
  end
14
20
 
15
21
  describe "#register_message" do
@@ -87,7 +93,7 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
87
93
  end
88
94
 
89
95
  it "logs error" do
90
- expect(logger).to receive(:error).with(/Receptor response: Failed to parse Kafka response/)
96
+ expect(logger).to receive(:error).with(/Failed to parse Kafka response/)
91
97
 
92
98
  subject.send(:process_message, message)
93
99
  end
@@ -97,7 +103,7 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
97
103
  let(:payload) { {'code' => 0, 'message_type' => 'response', 'payload' => response_body} }
98
104
 
99
105
  it "logs error" do
100
- expect(logger).to receive(:error).with(/Receptor response: Message id \(in_response_to\) not received!/)
106
+ expect(logger).to receive(:error).with(/Message id \(in_response_to\) not received!/)
101
107
 
102
108
  subject.send(:process_message, message)
103
109
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: receptor_controller-client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Martin Slemr
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-07-31 00:00:00.000000000 Z
11
+ date: 2020-10-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -98,14 +98,14 @@ dependencies:
98
98
  requirements:
99
99
  - - "~>"
100
100
  - !ruby/object:Gem::Version
101
- version: 0.1.5
101
+ version: 1.0.0
102
102
  type: :runtime
103
103
  prerelease: false
104
104
  version_requirements: !ruby/object:Gem::Requirement
105
105
  requirements:
106
106
  - - "~>"
107
107
  - !ruby/object:Gem::Version
108
- version: 0.1.5
108
+ version: 1.0.0
109
109
  - !ruby/object:Gem::Dependency
110
110
  name: bundler
111
111
  requirement: !ruby/object:Gem::Requirement
@@ -261,8 +261,8 @@ signing_key:
261
261
  specification_version: 4
262
262
  summary: Client for communication with Platform Receptor Controller - Gem
263
263
  test_files:
264
- - spec/receptor_controller/directive_blocking_spec.rb
264
+ - spec/receptor_controller/directive_non_blocking_spec.rb
265
265
  - spec/receptor_controller/response_worker_spec.rb
266
+ - spec/receptor_controller/directive_blocking_spec.rb
266
267
  - spec/receptor_controller/client_spec.rb
267
- - spec/receptor_controller/directive_non_blocking_spec.rb
268
268
  - spec/spec_helper.rb