topological_inventory-providers-common 1.0.12 → 2.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.codeclimate.yml +31 -0
- data/.rubocop.yml +1 -1
- data/.yamllint +8 -0
- data/CHANGELOG.md +30 -5
- data/README.md +1 -1
- data/lib/topological_inventory/providers/common.rb +1 -2
- data/lib/topological_inventory/providers/common/logging.rb +6 -3
- data/lib/topological_inventory/providers/common/messaging_client.rb +40 -0
- data/lib/topological_inventory/providers/common/metrics.rb +84 -0
- data/lib/topological_inventory/providers/common/mixins/sources_api.rb +61 -0
- data/lib/topological_inventory/providers/common/mixins/statuses.rb +19 -0
- data/lib/topological_inventory/providers/common/mixins/topology_api.rb +26 -0
- data/lib/topological_inventory/providers/common/mixins/x_rh_headers.rb +24 -0
- data/lib/topological_inventory/providers/common/operations/async_worker.rb +12 -7
- data/lib/topological_inventory/providers/common/operations/processor.rb +46 -104
- data/lib/topological_inventory/providers/common/operations/source.rb +183 -144
- data/lib/topological_inventory/providers/common/sources_api_client.rb +92 -0
- data/lib/topological_inventory/providers/common/topology_api_client.rb +43 -0
- data/lib/topological_inventory/providers/common/version.rb +1 -1
- data/spec/support/shared/availability_check.rb +254 -90
- data/spec/topological_inventory/providers/common/operations/async_worker_spec.rb +14 -5
- data/spec/topological_inventory/providers/common/operations/processor_spec.rb +52 -83
- data/topological_inventory-providers-common.gemspec +14 -10
- metadata +72 -9
- data/lib/topological_inventory/providers/common/operations/endpoint_client.rb +0 -65
- data/lib/topological_inventory/providers/common/operations/sources_api_client.rb +0 -94
- data/lib/topological_inventory/providers/common/operations/topology_api_client.rb +0 -28
@@ -1,4 +1,6 @@
|
|
1
|
-
require "topological_inventory/providers/common/
|
1
|
+
require "topological_inventory/providers/common/logging"
|
2
|
+
require "topological_inventory/providers/common/mixins/statuses"
|
3
|
+
require "topological_inventory/providers/common/mixins/topology_api"
|
2
4
|
|
3
5
|
module TopologicalInventory
|
4
6
|
module Providers
|
@@ -6,127 +8,67 @@ module TopologicalInventory
|
|
6
8
|
module Operations
|
7
9
|
class Processor
|
8
10
|
include Logging
|
9
|
-
include
|
11
|
+
include Mixins::Statuses
|
12
|
+
include Mixins::TopologyApi
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
def self.process!(message)
|
15
|
-
model, method = message.headers['message_type'].to_s.split(".")
|
16
|
-
new(model, method, message.payload).process
|
14
|
+
def self.process!(message, metrics)
|
15
|
+
new(message, metrics).process
|
17
16
|
end
|
18
17
|
|
19
|
-
|
20
|
-
|
21
|
-
self.
|
22
|
-
self.method
|
23
|
-
|
24
|
-
self.
|
25
|
-
self.
|
18
|
+
def initialize(message, metrics)
|
19
|
+
self.message = message
|
20
|
+
self.metrics = metrics
|
21
|
+
self.model, self.method = message.message.split(".")
|
22
|
+
|
23
|
+
self.params = message.payload["params"]
|
24
|
+
self.identity = message.payload["request_context"]
|
26
25
|
end
|
27
26
|
|
28
27
|
def process
|
29
|
-
logger.info(
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
28
|
+
logger.info(status_log_msg)
|
29
|
+
impl = operation_class&.new(params, identity, metrics)
|
30
|
+
if impl&.respond_to?(method)
|
31
|
+
with_time_measure do
|
32
|
+
result = impl.send(method)
|
33
|
+
|
34
|
+
logger.info(status_log_msg("Complete"))
|
35
|
+
result
|
36
|
+
end
|
37
|
+
else
|
38
|
+
logger.warn(status_log_msg("Not Implemented!"))
|
39
|
+
complete_task("not implemented") if params["task_id"]
|
40
|
+
operation_status[:not_implemented]
|
41
|
+
end
|
42
|
+
rescue StandardError, NotImplementedError => e
|
43
|
+
complete_task(e.message) if params["task_id"]
|
44
|
+
raise
|
34
45
|
end
|
35
46
|
|
36
47
|
private
|
37
48
|
|
38
|
-
attr_accessor :identity, :model, :method, :metrics, :params
|
39
|
-
|
40
|
-
def endpoint_client(source_id, task_id, identity)
|
41
|
-
raise NotImplementedError, "#{__method__} must be implemented in a subclass as kind of TopologicalInventory::Providers::Common::EndpointClient class"
|
42
|
-
end
|
43
|
-
|
44
|
-
def order_service(params)
|
45
|
-
task_id, service_offering_id, service_plan_id, order_params = params.values_at("task_id", "service_offering_id", "service_plan_id", "order_params")
|
46
|
-
|
47
|
-
service_plan = topology_api_client.show_service_plan(service_plan_id) if service_plan_id.present?
|
48
|
-
service_offering_id = service_plan.service_offering_id if service_offering_id.nil? && service_plan.present?
|
49
|
-
service_offering = topology_api_client.show_service_offering(service_offering_id)
|
50
|
-
|
51
|
-
source_id = service_offering.source_id
|
52
|
-
client = endpoint_client(source_id, task_id, identity)
|
53
|
-
|
54
|
-
logger.info("Ordering #{service_offering.name}...")
|
55
|
-
remote_service_instance = client.order_service(service_offering, service_plan.presence, order_params)
|
56
|
-
logger.info("Ordering #{service_offering.name}...Complete")
|
49
|
+
attr_accessor :message, :identity, :model, :method, :metrics, :params
|
57
50
|
|
58
|
-
|
59
|
-
|
60
|
-
metrics&.record_error
|
61
|
-
logger.error("[Task #{task_id}] Ordering error: #{err}\n#{err.backtrace.join("\n")}")
|
62
|
-
update_task(task_id, :state => "completed", :status => "error", :context => {:error => err.to_s})
|
51
|
+
def operation_class
|
52
|
+
raise NotImplementedError, "#{__method__} must be implemented in a subclass"
|
63
53
|
end
|
64
54
|
|
65
|
-
def
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
logger.error("[Task #{task_id}] Waiting for complete: #{err}\n#{err.backtrace.join("\n")}")
|
71
|
-
update_task(task_id, :state => "completed", :status => "warn", :context => {:error => err.to_s})
|
72
|
-
end
|
55
|
+
def with_time_measure
|
56
|
+
if metrics.present?
|
57
|
+
metrics.record_operation_time(message.message) { yield }
|
58
|
+
else
|
59
|
+
yield
|
73
60
|
end
|
74
61
|
end
|
75
62
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
:source_id => source_id,
|
82
|
-
:source_ref => client.source_ref_of(remote_svc_instance)
|
83
|
-
}
|
84
|
-
}
|
85
|
-
|
86
|
-
remote_svc_instance = client.wait_for_provision_complete(task_id, remote_svc_instance, context)
|
87
|
-
|
88
|
-
if client.provisioned_successfully?(remote_svc_instance)
|
89
|
-
if (service_instance = load_topological_svc_instance(source_id, client.source_ref_of(remote_svc_instance))).present?
|
90
|
-
context[:service_instance][:id] = service_instance.id
|
91
|
-
context[:service_instance][:url] = svc_instance_url(service_instance)
|
92
|
-
else
|
93
|
-
logger.warn("Failed to get service_instance API URL (endpoint's service instance: #{remote_svc_instance.inspect})")
|
94
|
-
end
|
95
|
-
end
|
96
|
-
update_task(task_id, :state => "completed", :status => client.task_status_for(remote_svc_instance), :context => context)
|
63
|
+
def complete_task(msg, status = "error")
|
64
|
+
update_task(params["task_id"],
|
65
|
+
:state => "completed",
|
66
|
+
:status => status,
|
67
|
+
:context => {:error => "#{model}##{method} - #{msg}"})
|
97
68
|
end
|
98
69
|
|
99
|
-
def
|
100
|
-
|
101
|
-
|
102
|
-
count = 0
|
103
|
-
timeout_count = POLL_TIMEOUT / SLEEP_POLL
|
104
|
-
|
105
|
-
header_params = { 'Accept' => api.select_header_accept(['application/json']) }
|
106
|
-
query_params = { :'source_id' => source_id, :'source_ref' => source_ref }
|
107
|
-
return_type = 'ServiceInstancesCollection'
|
108
|
-
|
109
|
-
service_instance = nil
|
110
|
-
loop do
|
111
|
-
data, _status_code, _headers = api.call_api(:GET, "/service_instances",
|
112
|
-
:header_params => header_params,
|
113
|
-
:query_params => query_params,
|
114
|
-
:auth_names => ['UserSecurity'],
|
115
|
-
:return_type => return_type)
|
116
|
-
|
117
|
-
service_instance = data.data&.first if data.meta.count > 0
|
118
|
-
break if service_instance.present?
|
119
|
-
|
120
|
-
break if (count += 1) >= timeout_count
|
121
|
-
|
122
|
-
sleep(SLEEP_POLL) # seconds
|
123
|
-
end
|
124
|
-
|
125
|
-
if service_instance.nil?
|
126
|
-
logger.error("Failed to find service_instance by source_id [#{source_id}] source_ref [#{source_ref}]")
|
127
|
-
end
|
128
|
-
|
129
|
-
service_instance
|
70
|
+
def status_log_msg(status = nil)
|
71
|
+
"Processing #{model}##{method} [#{params}]...#{status}"
|
130
72
|
end
|
131
73
|
end
|
132
74
|
end
|
@@ -1,194 +1,233 @@
|
|
1
1
|
require "topological_inventory/providers/common/logging"
|
2
2
|
require "active_support/core_ext/numeric/time"
|
3
|
-
require "topological_inventory/providers/common/
|
3
|
+
require "topological_inventory/providers/common/mixins/sources_api"
|
4
|
+
require "topological_inventory/providers/common/mixins/statuses"
|
5
|
+
require "topological_inventory/providers/common/mixins/x_rh_headers"
|
6
|
+
require "topological_inventory/providers/common/messaging_client"
|
4
7
|
|
5
8
|
module TopologicalInventory
|
6
9
|
module Providers
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
10
|
+
module Common
|
11
|
+
module Operations
|
12
|
+
class Source
|
13
|
+
include Logging
|
14
|
+
include Mixins::SourcesApi
|
15
|
+
include Mixins::XRhHeaders
|
16
|
+
include Mixins::Statuses
|
17
|
+
|
18
|
+
STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
|
19
|
+
EVENT_AVAILABILITY_STATUS = "availability_status".freeze
|
20
|
+
SERVICE_NAME = "platform.sources.status".freeze
|
21
|
+
|
22
|
+
ERROR_MESSAGES = {
|
23
|
+
:authentication_not_found => "Authentication not found in Sources API",
|
24
|
+
:endpoint_or_application_not_found => "Endpoint or Application not found in Sources API",
|
25
|
+
}.freeze
|
26
|
+
|
27
|
+
LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
|
28
|
+
AUTH_NOT_NECESSARY = "n/a".freeze
|
29
|
+
|
30
|
+
attr_accessor :identity, :operation, :params, :request_context, :source_id, :account_number
|
31
|
+
|
32
|
+
def initialize(params = {}, request_context = nil, metrics = nil)
|
33
|
+
self.metrics = metrics
|
34
|
+
self.operation = 'Source'
|
35
|
+
self.params = params
|
36
|
+
self.request_context = request_context
|
37
|
+
self.source_id = params['source_id']
|
38
|
+
self.account_number = params['external_tenant']
|
39
|
+
self.updates_via_kafka = ENV['UPDATE_SOURCES_VIA_API'].blank?
|
40
|
+
self.identity = identity_by_account_number(account_number)
|
41
|
+
end
|
18
42
|
|
19
|
-
|
20
|
-
|
43
|
+
def availability_check
|
44
|
+
self.operation += '#availability_check'
|
21
45
|
|
22
|
-
|
46
|
+
return operation_status[:error] if params_missing?
|
23
47
|
|
24
|
-
|
25
|
-
self.params = params
|
26
|
-
self.request_context = request_context
|
27
|
-
self.source_id = params['source_id']
|
28
|
-
self.account_number = params['external_tenant']
|
29
|
-
end
|
48
|
+
return operation_status[:skipped] if checked_recently?
|
30
49
|
|
31
|
-
|
32
|
-
return if params_missing?
|
50
|
+
status, error_message = connection_status
|
33
51
|
|
34
|
-
|
52
|
+
update_source_and_subresources(status, error_message)
|
35
53
|
|
36
|
-
|
54
|
+
logger.availability_check("Completed: Source #{source_id} is #{status}")
|
37
55
|
|
38
|
-
|
56
|
+
operation_status[:success]
|
57
|
+
end
|
39
58
|
|
40
|
-
|
41
|
-
end
|
59
|
+
private
|
42
60
|
|
43
|
-
|
61
|
+
attr_accessor :metrics, :updates_via_kafka
|
44
62
|
|
45
|
-
|
46
|
-
|
47
|
-
|
63
|
+
def required_params
|
64
|
+
%w[source_id]
|
65
|
+
end
|
48
66
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
67
|
+
def params_missing?
|
68
|
+
is_missing = false
|
69
|
+
required_params.each do |attr|
|
70
|
+
if (is_missing = params[attr].blank?)
|
71
|
+
logger.availability_check("Missing #{attr} for the availability_check request [Source ID: #{source_id}]", :error)
|
72
|
+
break
|
73
|
+
end
|
55
74
|
end
|
56
|
-
end
|
57
75
|
|
58
|
-
|
59
|
-
|
76
|
+
is_missing
|
77
|
+
end
|
60
78
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
79
|
+
def checked_recently?
|
80
|
+
checked_recently = if endpoint.present?
|
81
|
+
endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
|
82
|
+
elsif application.present?
|
83
|
+
application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
|
84
|
+
end
|
67
85
|
|
68
|
-
|
86
|
+
logger.availability_check("Skipping, last check at #{endpoint&.last_checked_at || application&.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
|
69
87
|
|
70
|
-
|
71
|
-
|
88
|
+
checked_recently
|
89
|
+
end
|
72
90
|
|
73
|
-
|
74
|
-
|
75
|
-
|
91
|
+
def connection_status
|
92
|
+
# we need either an endpoint or application to check the source.
|
93
|
+
return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_or_application_not_found]] unless endpoint || application
|
76
94
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
95
|
+
check_time
|
96
|
+
if endpoint
|
97
|
+
endpoint_connection_check
|
98
|
+
elsif application
|
99
|
+
application_connection_check
|
100
|
+
end
|
82
101
|
end
|
83
|
-
end
|
84
102
|
|
85
|
-
|
86
|
-
|
103
|
+
def endpoint_connection_check
|
104
|
+
return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
|
87
105
|
|
88
|
-
|
89
|
-
|
90
|
-
end
|
91
|
-
|
92
|
-
def application_connection_check
|
93
|
-
case application.availability_status
|
94
|
-
when "available"
|
95
|
-
[STATUS_AVAILABLE, nil]
|
96
|
-
when "unavailable"
|
97
|
-
[STATUS_UNAVAILABLE, "Application id #{application.id} unavailable"]
|
106
|
+
# call down into the operations pod implementation of `Source#connection_check`
|
107
|
+
connection_check
|
98
108
|
end
|
99
|
-
end
|
100
109
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
110
|
+
def application_connection_check
|
111
|
+
case application.availability_status
|
112
|
+
when "available"
|
113
|
+
[STATUS_AVAILABLE, nil]
|
114
|
+
when "unavailable"
|
115
|
+
[STATUS_UNAVAILABLE, "Application id #{application.id} unavailable"]
|
116
|
+
end
|
117
|
+
end
|
108
118
|
|
109
|
-
|
119
|
+
# @return [Array<String, String|nil] - STATUS_[UN]AVAILABLE, error message
|
120
|
+
def connection_check
|
121
|
+
raise NotImplementedError, "#{__method__} must be implemented in a subclass"
|
122
|
+
end
|
110
123
|
|
111
|
-
|
112
|
-
|
113
|
-
|
124
|
+
def update_source_and_subresources(status, error_message = nil)
|
125
|
+
logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
|
126
|
+
|
127
|
+
if updates_via_kafka
|
128
|
+
update_source_by_kafka(status)
|
129
|
+
update_endpoint_by_kafka(status, error_message) if endpoint
|
130
|
+
update_application_by_kafka(status) if application
|
131
|
+
else
|
132
|
+
update_source(status)
|
133
|
+
update_endpoint(status, error_message) if endpoint
|
134
|
+
update_application(status) if application
|
135
|
+
end
|
136
|
+
end
|
114
137
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
138
|
+
def update_source(status)
|
139
|
+
source = ::SourcesApiClient::Source.new
|
140
|
+
source.availability_status = status
|
141
|
+
source.last_checked_at = check_time
|
142
|
+
source.last_available_at = check_time if status == STATUS_AVAILABLE
|
120
143
|
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
144
|
+
sources_api.update_source(source_id, source)
|
145
|
+
rescue ::SourcesApiClient::ApiError => e
|
146
|
+
metrics&.record_error(:sources_api)
|
147
|
+
logger.availability_check("Failed to update Source id:#{source_id} - #{e.message}", :error)
|
148
|
+
end
|
125
149
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
150
|
+
def update_source_by_kafka(status)
|
151
|
+
availability_status_message(
|
152
|
+
:resource_type => "Source",
|
153
|
+
:resource_id => source_id,
|
154
|
+
:status => status
|
155
|
+
)
|
130
156
|
end
|
131
157
|
|
132
|
-
|
158
|
+
def update_endpoint(status, error_message)
|
159
|
+
if endpoint.nil?
|
160
|
+
logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
|
161
|
+
return
|
162
|
+
end
|
133
163
|
|
134
|
-
|
135
|
-
endpoint_update.availability_status_error = error_message.to_s
|
136
|
-
endpoint_update.last_checked_at = check_time
|
137
|
-
endpoint_update.last_available_at = check_time if status == STATUS_AVAILABLE
|
164
|
+
endpoint_update = ::SourcesApiClient::Endpoint.new
|
138
165
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
166
|
+
endpoint_update.availability_status = status
|
167
|
+
endpoint_update.availability_status_error = error_message.to_s
|
168
|
+
endpoint_update.last_checked_at = check_time
|
169
|
+
endpoint_update.last_available_at = check_time if status == STATUS_AVAILABLE
|
143
170
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
171
|
+
sources_api.update_endpoint(endpoint.id, endpoint_update)
|
172
|
+
rescue ::SourcesApiClient::ApiError => e
|
173
|
+
metrics&.record_error(:sources_api)
|
174
|
+
logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
|
175
|
+
end
|
148
176
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
177
|
+
def update_endpoint_by_kafka(status, error_message)
|
178
|
+
if endpoint.nil?
|
179
|
+
logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
|
180
|
+
return
|
181
|
+
end
|
153
182
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
183
|
+
availability_status_message(
|
184
|
+
:resource_type => "Endpoint",
|
185
|
+
:resource_id => endpoint.id,
|
186
|
+
:status => status,
|
187
|
+
:error => error_message
|
188
|
+
)
|
189
|
+
end
|
160
190
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
api_client.fetch_authentication(source_id, endpoint)
|
166
|
-
end
|
167
|
-
rescue => e
|
168
|
-
logger.availability_check("Failed to fetch Authentication for Source #{source_id}: #{e.message}", :error)
|
169
|
-
nil
|
170
|
-
end
|
191
|
+
def update_application(status)
|
192
|
+
application_update = ::SourcesApiClient::Application.new
|
193
|
+
application_update.last_checked_at = check_time
|
194
|
+
application_update.last_available_at = check_time if status == STATUS_AVAILABLE
|
171
195
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
end
|
196
|
+
sources_api.update_application(application.id, application_update)
|
197
|
+
rescue ::SourcesApiClient::ApiError => e
|
198
|
+
metrics&.record_error(:sources_api)
|
199
|
+
logger.availability_check("Failed to update Application id: #{application.id} - #{e.message}", :error)
|
200
|
+
end
|
178
201
|
|
179
|
-
|
180
|
-
|
181
|
-
|
202
|
+
def update_application_by_kafka(status)
|
203
|
+
availability_status_message(
|
204
|
+
:resource_type => "Application",
|
205
|
+
:resource_id => application.id,
|
206
|
+
:status => status
|
207
|
+
)
|
208
|
+
end
|
182
209
|
|
183
|
-
|
184
|
-
|
185
|
-
|
210
|
+
def availability_status_message(payload)
|
211
|
+
messaging_client.publish_topic(
|
212
|
+
:service => SERVICE_NAME,
|
213
|
+
:event => EVENT_AVAILABILITY_STATUS,
|
214
|
+
:payload => payload.to_json
|
215
|
+
)
|
216
|
+
rescue => err
|
217
|
+
logger.availability_check("Failed to update #{payload[:resource_type]} id: #{payload[:resource_id]} - #{err.message}", :error)
|
218
|
+
ensure
|
219
|
+
messaging_client&.close
|
220
|
+
end
|
221
|
+
|
222
|
+
def messaging_client
|
223
|
+
TopologicalInventory::Providers::Common::MessagingClient.default.client
|
224
|
+
end
|
186
225
|
|
187
|
-
|
188
|
-
|
226
|
+
def check_time
|
227
|
+
@check_time ||= Time.now.utc
|
228
|
+
end
|
189
229
|
end
|
190
230
|
end
|
191
231
|
end
|
192
232
|
end
|
193
|
-
end
|
194
233
|
end
|