receptor_controller-client 0.0.2 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -1
- data/lib/receptor_controller/client/configuration.rb +2 -2
- data/lib/receptor_controller/client/directive_blocking.rb +1 -1
- data/lib/receptor_controller/client/directive_non_blocking.rb +1 -0
- data/lib/receptor_controller/client/response_worker.rb +57 -21
- data/lib/receptor_controller/client/version.rb +1 -1
- data/receptor_controller-client.gemspec +1 -1
- data/spec/receptor_controller/directive_blocking_spec.rb +6 -1
- data/spec/receptor_controller/directive_non_blocking_spec.rb +4 -4
- data/spec/receptor_controller/response_worker_spec.rb +9 -3
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ab72f29ace5c188996000048957bb58020940d3b8ce32b1a0b720602f4ff7e64
|
4
|
+
data.tar.gz: 4b68d648d59809f80e108a1d998af10d09fee233db085e02b736c691db05b8d5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 32b8c1c4a2173e38ec229bad3b4f0c6747c9f9cc50ce3be86746217a6682b60e4c820e9677adc38798548d81bb62db858994a7019cb85fc9b58c149eb0357c2e
|
7
|
+
data.tar.gz: abfdf8a4c9924f4ba225310f5439b97f003d4ac01eedd14a822328d7750289a60cc9ec49a086ac125f446adbd0c84840afc02562a84c3c02c19f6402b201a469
|
data/CHANGELOG.md
CHANGED
@@ -5,6 +5,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
5
5
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
|
+
## [0.0.7]
|
9
|
+
- rdkafka changes #32
|
10
|
+
|
11
|
+
## [0.0.6]
|
12
|
+
- Waiting for response-worker kafka init #20
|
13
|
+
- Fix href_slug in directives log #29
|
14
|
+
|
15
|
+
## [0.0.5] - 2020-08-11
|
16
|
+
### Added
|
17
|
+
- Unique Persist-ref for every response worker #18
|
18
|
+
|
19
|
+
## [0.0.4] - 2020-08-04
|
20
|
+
### Added
|
21
|
+
- Default Kafka ACK to true
|
22
|
+
|
23
|
+
## [0.0.3] - 2020-08-03
|
24
|
+
### Added
|
25
|
+
- Do not send EOF message until all messages have been received
|
8
26
|
|
9
27
|
## [0.0.2] - 2020-07-31
|
10
28
|
### Initial release to rubygems.org
|
@@ -12,5 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
12
30
|
### Added
|
13
31
|
- Added extra logging for messages
|
14
32
|
|
15
|
-
[Unreleased]: https://github.com/RedHatInsights/receptor_controller-client-ruby/compare/v0.0.
|
33
|
+
[Unreleased]: https://github.com/RedHatInsights/receptor_controller-client-ruby/compare/v0.0.7...HEAD
|
34
|
+
[0.0.6]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.7
|
35
|
+
[0.0.6]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.6
|
36
|
+
[0.0.5]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.5
|
37
|
+
[0.0.4]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.4
|
38
|
+
[0.0.3]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.3
|
16
39
|
[0.0.2]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.2
|
@@ -39,10 +39,10 @@ module ReceptorController
|
|
39
39
|
@job_path = '/job'
|
40
40
|
@pre_shared_key = nil
|
41
41
|
|
42
|
-
@queue_auto_ack =
|
42
|
+
@queue_auto_ack = true
|
43
43
|
@queue_host = nil
|
44
44
|
@queue_max_bytes = nil
|
45
|
-
@queue_persist_ref =
|
45
|
+
@queue_persist_ref = "receptor-client.#{SecureRandom.uuid}"
|
46
46
|
@queue_port = nil
|
47
47
|
@queue_topic = 'platform.receptor-controller.responses'
|
48
48
|
|
@@ -20,7 +20,7 @@ module ReceptorController
|
|
20
20
|
|
21
21
|
msg_id = JSON.parse(response.body)['id']
|
22
22
|
|
23
|
-
logger.debug("Receptor response: registering message #{msg_id}
|
23
|
+
logger.debug("Receptor response [#{ReceptorController::Client::Configuration.default.queue_persist_ref}]: registering message #{msg_id}, href_slug: #{log_message_common}")
|
24
24
|
# registers message id for kafka responses
|
25
25
|
response_worker.register_message(msg_id, self)
|
26
26
|
wait_for_response(msg_id)
|
@@ -48,6 +48,7 @@ module ReceptorController
|
|
48
48
|
|
49
49
|
# registers message id for kafka responses
|
50
50
|
response_worker.register_message(msg_id, self)
|
51
|
+
logger.debug("Receptor response [#{ReceptorController::Client::Configuration.default.queue_persist_ref}]: registering message #{msg_id}, href_slug: #{log_message_common}")
|
51
52
|
|
52
53
|
msg_id
|
53
54
|
else
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'active_support/notifications'
|
1
2
|
require 'base64'
|
2
3
|
require "concurrent"
|
3
4
|
require 'stringio'
|
@@ -18,6 +19,10 @@ module ReceptorController
|
|
18
19
|
#
|
19
20
|
# Use "start" and "stop" methods to start/stop listening on Kafka
|
20
21
|
class Client::ResponseWorker
|
22
|
+
EOF = "eof".freeze
|
23
|
+
RESPONSE = "response".freeze
|
24
|
+
INITIALIZATION_TIMEOUT = 20
|
25
|
+
|
21
26
|
attr_reader :started
|
22
27
|
alias started? started
|
23
28
|
|
@@ -36,13 +41,19 @@ module ReceptorController
|
|
36
41
|
|
37
42
|
# Start listening on Kafka
|
38
43
|
def start
|
44
|
+
init_lock, init_wait = Mutex.new, ConditionVariable.new
|
45
|
+
|
39
46
|
lock.synchronize do
|
40
47
|
return if started.value
|
41
48
|
|
49
|
+
default_messaging_opts # Thread-safe init
|
50
|
+
|
42
51
|
started.value = true
|
43
52
|
workers[:maintenance] = Thread.new { check_timeouts while started.value }
|
44
53
|
workers[:listener] = Thread.new { listen while started.value }
|
45
54
|
end
|
55
|
+
|
56
|
+
logger.info("Receptor Response worker started...")
|
46
57
|
end
|
47
58
|
|
48
59
|
# Stop listener
|
@@ -81,12 +92,11 @@ module ReceptorController
|
|
81
92
|
# Open a connection to the messaging service
|
82
93
|
client = ManageIQ::Messaging::Client.open(default_messaging_opts)
|
83
94
|
|
84
|
-
logger.info("Receptor Response worker started...")
|
85
95
|
client.subscribe_topic(queue_opts) do |message|
|
86
96
|
process_message(message)
|
87
97
|
end
|
88
98
|
rescue => err
|
89
|
-
logger.error("Exception in kafka listener: #{err}\n#{err.backtrace.join("\n")}")
|
99
|
+
logger.error(response_log("Exception in kafka listener: #{err}\n#{err.backtrace.join("\n")}"))
|
90
100
|
ensure
|
91
101
|
client&.close
|
92
102
|
end
|
@@ -95,7 +105,7 @@ module ReceptorController
|
|
95
105
|
response = JSON.parse(message.payload)
|
96
106
|
|
97
107
|
if (message_id = response['in_response_to'])
|
98
|
-
logger.debug("
|
108
|
+
logger.debug(response_log("Received message #{message_id}: serial: #{response["serial"]}, type: #{response['message_type']}, payload: #{response['payload'] || "n/a"}"))
|
99
109
|
if (callbacks = registered_messages[message_id]).present?
|
100
110
|
# Reset last_checked_at to avoid timeout in multi-response messages
|
101
111
|
reset_last_checked_at(callbacks)
|
@@ -105,35 +115,52 @@ module ReceptorController
|
|
105
115
|
# Response OK
|
106
116
|
#
|
107
117
|
message_type = response['message_type'] # "response" (with data) or "eof" (without data)
|
108
|
-
registered_messages.delete(message_id) if message_type == 'eof'
|
109
|
-
|
110
118
|
payload = response['payload']
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
119
|
+
callbacks[:received_msgs] ? callbacks[:received_msgs] += 1 : callbacks[:received_msgs] = 1
|
120
|
+
|
121
|
+
case message_type
|
122
|
+
when EOF
|
123
|
+
# Store how many messages are needed to be received for this request
|
124
|
+
callbacks[:total_msgs] = response["serial"]
|
125
|
+
when RESPONSE
|
126
|
+
payload = unpack_payload(payload) if payload.kind_of?(String)
|
127
|
+
callbacks[:msg_size] ? callbacks[:msg_size] += payload.size : callbacks[:msg_size] = payload.size
|
128
|
+
callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
|
129
|
+
else
|
130
|
+
# Send the callback to release the thread.
|
131
|
+
logger.warn(response_log("Unexpected type | message #{message_id}, type: #{message_type}"))
|
132
|
+
callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
|
133
|
+
end
|
134
|
+
|
135
|
+
# We received all the messages, complete the message.
|
136
|
+
if callbacks[:received_msgs] == callbacks[:total_msgs]
|
137
|
+
registered_messages.delete(message_id)
|
138
|
+
callbacks[:receiver].send(callbacks[:response_callback], message_id, EOF, payload)
|
139
|
+
logger.debug(response_log("Message #{message_id} complete, total bytes: #{callbacks[:msg_size]}"))
|
140
|
+
end
|
141
|
+
|
142
|
+
logger.debug(response_log("OK | message: #{message_id}, serial: #{response["serial"]}, type: #{message_type}, payload: #{payload || "n/a"}"))
|
116
143
|
else
|
117
144
|
#
|
118
145
|
# Response Error
|
119
146
|
#
|
120
147
|
registered_messages.delete(message_id)
|
121
148
|
|
122
|
-
logger.error("
|
149
|
+
logger.error(response_log("ERROR | message #{message_id} (#{response})"))
|
123
150
|
|
124
151
|
callbacks[:receiver].send(callbacks[:error_callback], message_id, response['code'], response['payload'])
|
125
152
|
end
|
126
153
|
elsif ENV["LOG_ALL_RECEPTOR_MESSAGES"]&.to_i != 0
|
127
154
|
# noop, it's not error if not registered, can be processed by another pod
|
128
|
-
logger.debug("
|
155
|
+
logger.debug(response_log("NOT REGISTERED | #{message_id} (#{response['code']})"))
|
129
156
|
end
|
130
157
|
else
|
131
|
-
logger.error("
|
158
|
+
logger.error(response_log("MISSING | Message id (in_response_to) not received! #{response}"))
|
132
159
|
end
|
133
160
|
rescue JSON::ParserError => e
|
134
|
-
logger.error("
|
161
|
+
logger.error(response_log("Failed to parse Kafka response (#{e.message})\n#{message.payload}"))
|
135
162
|
rescue => e
|
136
|
-
logger.error("
|
163
|
+
logger.error(response_log("#{e}\n#{e.backtrace.join("\n")}"))
|
137
164
|
ensure
|
138
165
|
message.ack unless config.queue_auto_ack
|
139
166
|
end
|
@@ -194,21 +221,30 @@ module ReceptorController
|
|
194
221
|
end
|
195
222
|
|
196
223
|
# No persist_ref here, because all instances (pods) needs to receive kafka message
|
224
|
+
# TODO: temporary changed to unique persist_ref
|
197
225
|
def queue_opts
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
226
|
+
return @queue_opts if @queue_opts
|
227
|
+
|
228
|
+
@queue_opts = {:service => config.queue_topic,
|
229
|
+
:auto_ack => config.queue_auto_ack}
|
230
|
+
@queue_opts[:max_bytes] = config.queue_max_bytes if config.queue_max_bytes
|
231
|
+
@queue_opts[:persist_ref] = ENV['HOSTNAME']
|
232
|
+
@queue_opts
|
203
233
|
end
|
204
234
|
|
205
235
|
def default_messaging_opts
|
206
|
-
|
236
|
+
return @default_messaging_opts if @default_messaging_opts
|
237
|
+
|
238
|
+
@default_messaging_opts = {
|
207
239
|
:host => config.queue_host,
|
208
240
|
:port => config.queue_port,
|
209
241
|
:protocol => :Kafka,
|
210
242
|
:client_ref => "receptor_client-responses-#{Time.now.to_i}", # A reference string to identify the client
|
211
243
|
}
|
212
244
|
end
|
245
|
+
|
246
|
+
def response_log(message)
|
247
|
+
"Receptor Response [#{queue_opts[:persist_ref]}]: #{message}"
|
248
|
+
end
|
213
249
|
end
|
214
250
|
end
|
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
|
|
24
24
|
s.add_runtime_dependency 'faraday', '~> 1.0'
|
25
25
|
s.add_runtime_dependency 'json', '~> 2.3', '>= 2.3.0'
|
26
26
|
s.add_runtime_dependency 'manageiq-loggers', '~> 0.5.0'
|
27
|
-
s.add_runtime_dependency 'manageiq-messaging', '~> 0.
|
27
|
+
s.add_runtime_dependency 'manageiq-messaging', '~> 1.0.0'
|
28
28
|
|
29
29
|
s.add_development_dependency 'bundler', '~> 2.0'
|
30
30
|
s.add_development_dependency 'rake', '>= 12.3.3'
|
@@ -32,6 +32,10 @@ RSpec.describe ReceptorController::Client::DirectiveBlocking do
|
|
32
32
|
|
33
33
|
subject { described_class.new(:name => directive, :account => external_tenant, :node_id => receptor_node, :payload => payload, :client => receptor_client) }
|
34
34
|
|
35
|
+
before do
|
36
|
+
allow(subject.response_worker).to receive_messages(:init_notifications => nil, :wait_for_notifications => nil)
|
37
|
+
end
|
38
|
+
|
35
39
|
describe "#call" do
|
36
40
|
it "makes POST /job request to receptor and registers received message ID" do
|
37
41
|
allow(subject).to receive(:wait_for_response)
|
@@ -106,7 +110,8 @@ RSpec.describe ReceptorController::Client::DirectiveBlocking do
|
|
106
110
|
response_message = {'code' => 0,
|
107
111
|
'in_response_to' => http_response['id'],
|
108
112
|
'message_type' => subject.class::MESSAGE_TYPE_EOF,
|
109
|
-
'payload' => 'Unimportant'
|
113
|
+
'payload' => 'Unimportant',
|
114
|
+
'serial' => 1}
|
110
115
|
|
111
116
|
allow(kafka_response).to receive(:payload).and_return(response_message.to_json)
|
112
117
|
|
@@ -101,13 +101,13 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
|
|
101
101
|
# Kafka response - 'response' type
|
102
102
|
expect(subject).to receive(:response_success).twice.and_call_original
|
103
103
|
message_type, payload = 'response', 'Testing payload'
|
104
|
-
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload}
|
104
|
+
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload, :serial => 1}
|
105
105
|
allow(response).to receive(:payload).and_return(response_payload.to_json)
|
106
106
|
subject.response_worker.send(:process_message, response)
|
107
107
|
|
108
108
|
# Kafka response - 'eof' type
|
109
109
|
message_type = 'eof'
|
110
|
-
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil}
|
110
|
+
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil, :serial => 2}
|
111
111
|
allow(response).to receive(:payload).and_return(response_payload.to_json)
|
112
112
|
subject.response_worker.send(:process_message, response)
|
113
113
|
|
@@ -142,7 +142,7 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
|
|
142
142
|
|
143
143
|
# Kafka response - 'response' type - success
|
144
144
|
message_type, payload = 'response', 'Testing payload'
|
145
|
-
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload}
|
145
|
+
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload, :serial => 1}
|
146
146
|
allow(response).to receive(:payload).and_return(response_payload.to_json)
|
147
147
|
success_calls.times do
|
148
148
|
subject.response_worker.send(:process_message, response)
|
@@ -150,7 +150,7 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
|
|
150
150
|
|
151
151
|
# Kafka response - 'eof' type
|
152
152
|
message_type = 'eof'
|
153
|
-
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil}
|
153
|
+
response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil, :serial => 4}
|
154
154
|
allow(response).to receive(:payload).and_return(response_payload.to_json)
|
155
155
|
subject.response_worker.send(:process_message, response)
|
156
156
|
|
@@ -9,7 +9,13 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
|
|
9
9
|
|
10
10
|
before do
|
11
11
|
allow(logger).to receive_messages(%i[debug info warn error fatal])
|
12
|
-
allow(config).to receive_messages(:queue_auto_ack => false,
|
12
|
+
allow(config).to receive_messages(:queue_auto_ack => false,
|
13
|
+
:queue_max_bytes => nil,
|
14
|
+
:queue_persist_ref => 'consumer-group-1',
|
15
|
+
:queue_topic => 'receptor.kafka.topic',
|
16
|
+
:response_timeout => 0,
|
17
|
+
:response_timeout_poll_time => 0,
|
18
|
+
)
|
13
19
|
end
|
14
20
|
|
15
21
|
describe "#register_message" do
|
@@ -87,7 +93,7 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
|
|
87
93
|
end
|
88
94
|
|
89
95
|
it "logs error" do
|
90
|
-
expect(logger).to receive(:error).with(/
|
96
|
+
expect(logger).to receive(:error).with(/Failed to parse Kafka response/)
|
91
97
|
|
92
98
|
subject.send(:process_message, message)
|
93
99
|
end
|
@@ -97,7 +103,7 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
|
|
97
103
|
let(:payload) { {'code' => 0, 'message_type' => 'response', 'payload' => response_body} }
|
98
104
|
|
99
105
|
it "logs error" do
|
100
|
-
expect(logger).to receive(:error).with(/
|
106
|
+
expect(logger).to receive(:error).with(/Message id \(in_response_to\) not received!/)
|
101
107
|
|
102
108
|
subject.send(:process_message, message)
|
103
109
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: receptor_controller-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Martin Slemr
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -98,14 +98,14 @@ dependencies:
|
|
98
98
|
requirements:
|
99
99
|
- - "~>"
|
100
100
|
- !ruby/object:Gem::Version
|
101
|
-
version: 0.
|
101
|
+
version: 1.0.0
|
102
102
|
type: :runtime
|
103
103
|
prerelease: false
|
104
104
|
version_requirements: !ruby/object:Gem::Requirement
|
105
105
|
requirements:
|
106
106
|
- - "~>"
|
107
107
|
- !ruby/object:Gem::Version
|
108
|
-
version: 0.
|
108
|
+
version: 1.0.0
|
109
109
|
- !ruby/object:Gem::Dependency
|
110
110
|
name: bundler
|
111
111
|
requirement: !ruby/object:Gem::Requirement
|
@@ -261,8 +261,8 @@ signing_key:
|
|
261
261
|
specification_version: 4
|
262
262
|
summary: Client for communication with Platform Receptor Controller - Gem
|
263
263
|
test_files:
|
264
|
-
- spec/receptor_controller/
|
264
|
+
- spec/receptor_controller/directive_non_blocking_spec.rb
|
265
265
|
- spec/receptor_controller/response_worker_spec.rb
|
266
|
+
- spec/receptor_controller/directive_blocking_spec.rb
|
266
267
|
- spec/receptor_controller/client_spec.rb
|
267
|
-
- spec/receptor_controller/directive_non_blocking_spec.rb
|
268
268
|
- spec/spec_helper.rb
|