topological_inventory-providers-common 1.0.5 → 1.0.6
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aafb37efd571f568a887c8c11b31bc87a6141a556ae06b970bf4acfcbdea3d08
|
4
|
+
data.tar.gz: 2d1024891611a96df454a56819a5f6f921e195605d3867ca05c595c336f9a699
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: daff9b2dc2388297dba831994c74e048d608256c9fd3aadc06446c16abfa4143f5fcbed2e6dd3eff66cb6fbb9e6637abe8f47c1eda452b99d5557c9424c20849
|
7
|
+
data.tar.gz: ece193799a908c7ca96aee73de80e9a6dc7cf17346b549ebf2d802cd3ed42b11aec7b46837a4078c1569fe608ef0241f7ac64a7daf43bd2905df1d53e21c8e01
|
data/CHANGELOG.md
CHANGED
@@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
5
5
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
6
|
|
7
|
-
## [1.0.
|
7
|
+
## [1.0.6] - 2020-07-06
|
8
|
+
Add some error handling if Sources does not have endpoints/authentications for a source #38
|
9
|
+
Specs for Collector #35
|
10
|
+
|
11
|
+
## [1.0.5] - 2020-06-18
|
8
12
|
Change release workflow to do everything manually #32
|
9
13
|
Add specs to released files #33
|
10
14
|
|
@@ -34,7 +38,8 @@ manageiq-loggers to >= 0.4.2 #20
|
|
34
38
|
## [1.0.0] - 2020-03-19
|
35
39
|
### Initial release to rubygems.org
|
36
40
|
|
37
|
-
[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.
|
41
|
+
[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...HEAD
|
42
|
+
[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
|
38
43
|
[1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
|
39
44
|
[1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
|
40
45
|
[1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3
|
@@ -116,10 +116,16 @@ module TopologicalInventory
|
|
116
116
|
|
117
117
|
def endpoint
|
118
118
|
@endpoint ||= api_client.fetch_default_endpoint(source_id)
|
119
|
+
rescue StandardError => e
|
120
|
+
logger.availability_check("Failed to fetch Endpoint for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
|
121
|
+
return
|
119
122
|
end
|
120
123
|
|
121
124
|
def authentication
|
122
125
|
@authentication ||= api_client.fetch_authentication(source_id, endpoint)
|
126
|
+
rescue StandardError => e
|
127
|
+
logger.availability_check("Failed to fetch Authentication for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
|
128
|
+
return
|
123
129
|
end
|
124
130
|
|
125
131
|
def check_time
|
@@ -0,0 +1,171 @@
|
|
1
|
+
RSpec.describe TopologicalInventory::Providers::Common::Collector do
|
2
|
+
let(:collector) do
|
3
|
+
collector = described_class.new(source)
|
4
|
+
|
5
|
+
allow(collector).to receive(:ingress_api_client).and_return(client)
|
6
|
+
allow(collector).to receive(:logger).and_return(logger)
|
7
|
+
allow(logger).to receive(:error)
|
8
|
+
|
9
|
+
collector
|
10
|
+
end
|
11
|
+
|
12
|
+
let(:parser) { TopologicalInventory::Providers::Common::Collector::Parser.new }
|
13
|
+
|
14
|
+
let(:source) { "source_uid" }
|
15
|
+
let(:client) { double }
|
16
|
+
let(:logger) { double }
|
17
|
+
let(:refresh_state_uuid) { SecureRandom.uuid }
|
18
|
+
let(:refresh_state_part_uuid) { SecureRandom.uuid }
|
19
|
+
# based on the default, we can tell how many chunks the saver will break the payload up into
|
20
|
+
let(:max_size) { TopologicalInventory::Providers::Common::SaveInventory::Saver::KAFKA_PAYLOAD_MAX_BYTES_DEFAULT }
|
21
|
+
let(:multiplier) { 0.75 }
|
22
|
+
|
23
|
+
context "#save_inventory" do
|
24
|
+
it "does nothing with empty collections" do
|
25
|
+
parts = collector.send(:save_inventory, [], collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
26
|
+
|
27
|
+
expect(parts).to eq 0
|
28
|
+
end
|
29
|
+
|
30
|
+
it "saves 1 part if it fits" do
|
31
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
32
|
+
|
33
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(0)
|
34
|
+
|
35
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
36
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
37
|
+
expect(parts).to eq 1
|
38
|
+
end
|
39
|
+
|
40
|
+
it "saves 2 parts if over limit with 1 collection" do
|
41
|
+
(multiplier * 2000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
42
|
+
|
43
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
44
|
+
|
45
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
|
46
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
47
|
+
expect(parts).to eq 2
|
48
|
+
end
|
49
|
+
|
50
|
+
it "saves 2 parts if over limit with 2 collections" do
|
51
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
52
|
+
(multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
|
53
|
+
|
54
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
55
|
+
|
56
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
|
57
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
58
|
+
expect(parts).to eq 2
|
59
|
+
end
|
60
|
+
|
61
|
+
it "saves many parts" do
|
62
|
+
(multiplier * 1500).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
63
|
+
(multiplier * 2000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
|
64
|
+
|
65
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(4).times
|
66
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
67
|
+
expect(parts).to eq 4
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'raises exception when entity to save is too big' do
|
71
|
+
parser.collections.container_groups.build(:source_ref => "a" * (1_000_000 * multiplier))
|
72
|
+
|
73
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
74
|
+
# in this case, we first save empty inventory, then the size check fails saving the rest of data
|
75
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
76
|
+
|
77
|
+
expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
|
78
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
79
|
+
)
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'raises exception when entity of second collection is too big' do
|
83
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
84
|
+
parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
|
85
|
+
|
86
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
87
|
+
# We save the first collection then it fails on saving the second collection
|
88
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
89
|
+
|
90
|
+
expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
|
91
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
92
|
+
)
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'raises exception when entity of second collection is too big then continues with smaller' do
|
96
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
97
|
+
parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
|
98
|
+
(multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
|
99
|
+
|
100
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(2)
|
101
|
+
# We save the first collection then it fails on saving the second collection
|
102
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
103
|
+
|
104
|
+
expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
|
105
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
106
|
+
)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
context "#sweep_inventory" do
|
111
|
+
it "with nil total parts" do
|
112
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
113
|
+
|
114
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, nil, [])
|
115
|
+
end
|
116
|
+
|
117
|
+
it "with empty scope " do
|
118
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
119
|
+
|
120
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [])
|
121
|
+
end
|
122
|
+
|
123
|
+
it "with normal scope " do
|
124
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
125
|
+
|
126
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [:container_groups])
|
127
|
+
end
|
128
|
+
|
129
|
+
it "with normal targeted scope " do
|
130
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
131
|
+
|
132
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, {:container_groups => [{:source_ref => "a"}]})
|
133
|
+
end
|
134
|
+
|
135
|
+
it "fails with scope entity too large " do
|
136
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
137
|
+
|
138
|
+
sweep_scope = {:container_groups => [{:source_ref => "a" * (1_000_002 * multiplier)}]}
|
139
|
+
|
140
|
+
expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
|
141
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
142
|
+
)
|
143
|
+
end
|
144
|
+
|
145
|
+
it "fails when scope is too big " do
|
146
|
+
# We should have also sweep scope chunking, that is if we'll do big targeted refresh and sweeping
|
147
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
148
|
+
|
149
|
+
sweep_scope = {:container_groups => (0..1001 * multiplier).map { {:source_ref => "a" * 1_000} } }
|
150
|
+
|
151
|
+
expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
|
152
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
153
|
+
)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def build_inventory(collections)
|
158
|
+
TopologicalInventoryIngressApiClient::Inventory.new(
|
159
|
+
:name => collector.send(:inventory_name),
|
160
|
+
:schema => TopologicalInventoryIngressApiClient::Schema.new(:name => collector.send(:schema_name)),
|
161
|
+
:source => source,
|
162
|
+
:collections => collections,
|
163
|
+
:refresh_state_uuid => refresh_state_uuid,
|
164
|
+
:refresh_state_part_uuid => refresh_state_part_uuid,
|
165
|
+
)
|
166
|
+
end
|
167
|
+
|
168
|
+
def inventory_size(collections)
|
169
|
+
JSON.generate(build_inventory(collections).to_hash).size
|
170
|
+
end
|
171
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: topological_inventory-providers-common
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Martin Slemr
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -257,6 +257,7 @@ files:
|
|
257
257
|
- spec/spec_helper.rb
|
258
258
|
- spec/support/inventory_helper.rb
|
259
259
|
- spec/support/shared/availability_check.rb
|
260
|
+
- spec/topological_inventory/providers/common/collector_spec.rb
|
260
261
|
- spec/topological_inventory/providers/common/collectors/inventory_collection_storage_spec.rb
|
261
262
|
- spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb
|
262
263
|
- spec/topological_inventory/providers/common/collectors_pool_spec.rb
|