topological_inventory-providers-common 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aafb37efd571f568a887c8c11b31bc87a6141a556ae06b970bf4acfcbdea3d08
|
4
|
+
data.tar.gz: 2d1024891611a96df454a56819a5f6f921e195605d3867ca05c595c336f9a699
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: daff9b2dc2388297dba831994c74e048d608256c9fd3aadc06446c16abfa4143f5fcbed2e6dd3eff66cb6fbb9e6637abe8f47c1eda452b99d5557c9424c20849
|
7
|
+
data.tar.gz: ece193799a908c7ca96aee73de80e9a6dc7cf17346b549ebf2d802cd3ed42b11aec7b46837a4078c1569fe608ef0241f7ac64a7daf43bd2905df1d53e21c8e01
|
data/CHANGELOG.md
CHANGED
@@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
5
5
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
6
|
|
7
|
-
## [1.0.
|
7
|
+
## [1.0.6] - 2020-07-06
|
8
|
+
Add some error handling if Sources does not have endpoints/authentications for a source #38
|
9
|
+
Specs for Collector #35
|
10
|
+
|
11
|
+
## [1.0.5] - 2020-06-18
|
8
12
|
Change release workflow to do everything manually #32
|
9
13
|
Add specs to released files #33
|
10
14
|
|
@@ -34,7 +38,8 @@ manageiq-loggers to >= 0.4.2 #20
|
|
34
38
|
## [1.0.0] - 2020-03-19
|
35
39
|
### Initial release to rubygems.org
|
36
40
|
|
37
|
-
[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.
|
41
|
+
[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...HEAD
|
42
|
+
[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
|
38
43
|
[1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
|
39
44
|
[1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
|
40
45
|
[1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3
|
@@ -116,10 +116,16 @@ module TopologicalInventory
|
|
116
116
|
|
117
117
|
def endpoint
|
118
118
|
@endpoint ||= api_client.fetch_default_endpoint(source_id)
|
119
|
+
rescue StandardError => e
|
120
|
+
logger.availability_check("Failed to fetch Endpoint for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
|
121
|
+
return
|
119
122
|
end
|
120
123
|
|
121
124
|
def authentication
|
122
125
|
@authentication ||= api_client.fetch_authentication(source_id, endpoint)
|
126
|
+
rescue StandardError => e
|
127
|
+
logger.availability_check("Failed to fetch Authentication for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
|
128
|
+
return
|
123
129
|
end
|
124
130
|
|
125
131
|
def check_time
|
@@ -0,0 +1,171 @@
|
|
1
|
+
RSpec.describe TopologicalInventory::Providers::Common::Collector do
|
2
|
+
let(:collector) do
|
3
|
+
collector = described_class.new(source)
|
4
|
+
|
5
|
+
allow(collector).to receive(:ingress_api_client).and_return(client)
|
6
|
+
allow(collector).to receive(:logger).and_return(logger)
|
7
|
+
allow(logger).to receive(:error)
|
8
|
+
|
9
|
+
collector
|
10
|
+
end
|
11
|
+
|
12
|
+
let(:parser) { TopologicalInventory::Providers::Common::Collector::Parser.new }
|
13
|
+
|
14
|
+
let(:source) { "source_uid" }
|
15
|
+
let(:client) { double }
|
16
|
+
let(:logger) { double }
|
17
|
+
let(:refresh_state_uuid) { SecureRandom.uuid }
|
18
|
+
let(:refresh_state_part_uuid) { SecureRandom.uuid }
|
19
|
+
# based on the default, we can tell how many chunks the saver will break the payload up into
|
20
|
+
let(:max_size) { TopologicalInventory::Providers::Common::SaveInventory::Saver::KAFKA_PAYLOAD_MAX_BYTES_DEFAULT }
|
21
|
+
let(:multiplier) { 0.75 }
|
22
|
+
|
23
|
+
context "#save_inventory" do
|
24
|
+
it "does nothing with empty collections" do
|
25
|
+
parts = collector.send(:save_inventory, [], collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
26
|
+
|
27
|
+
expect(parts).to eq 0
|
28
|
+
end
|
29
|
+
|
30
|
+
it "saves 1 part if it fits" do
|
31
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
32
|
+
|
33
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(0)
|
34
|
+
|
35
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
36
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
37
|
+
expect(parts).to eq 1
|
38
|
+
end
|
39
|
+
|
40
|
+
it "saves 2 parts if over limit with 1 collection" do
|
41
|
+
(multiplier * 2000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
42
|
+
|
43
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
44
|
+
|
45
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
|
46
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
47
|
+
expect(parts).to eq 2
|
48
|
+
end
|
49
|
+
|
50
|
+
it "saves 2 parts if over limit with 2 collections" do
|
51
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
52
|
+
(multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
|
53
|
+
|
54
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
55
|
+
|
56
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
|
57
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
58
|
+
expect(parts).to eq 2
|
59
|
+
end
|
60
|
+
|
61
|
+
it "saves many parts" do
|
62
|
+
(multiplier * 1500).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
63
|
+
(multiplier * 2000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
|
64
|
+
|
65
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(4).times
|
66
|
+
parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
|
67
|
+
expect(parts).to eq 4
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'raises exception when entity to save is too big' do
|
71
|
+
parser.collections.container_groups.build(:source_ref => "a" * (1_000_000 * multiplier))
|
72
|
+
|
73
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
74
|
+
# in this case, we first save empty inventory, then the size check fails saving the rest of data
|
75
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
76
|
+
|
77
|
+
expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
|
78
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
79
|
+
)
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'raises exception when entity of second collection is too big' do
|
83
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
84
|
+
parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
|
85
|
+
|
86
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(1)
|
87
|
+
# We save the first collection then it fails on saving the second collection
|
88
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
89
|
+
|
90
|
+
expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
|
91
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
92
|
+
)
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'raises exception when entity of second collection is too big then continues with smaller' do
|
96
|
+
(multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
|
97
|
+
parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
|
98
|
+
(multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
|
99
|
+
|
100
|
+
expect(inventory_size(parser.collections.values) / max_size).to eq(2)
|
101
|
+
# We save the first collection then it fails on saving the second collection
|
102
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
103
|
+
|
104
|
+
expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
|
105
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
106
|
+
)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
context "#sweep_inventory" do
|
111
|
+
it "with nil total parts" do
|
112
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
113
|
+
|
114
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, nil, [])
|
115
|
+
end
|
116
|
+
|
117
|
+
it "with empty scope " do
|
118
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
119
|
+
|
120
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [])
|
121
|
+
end
|
122
|
+
|
123
|
+
it "with normal scope " do
|
124
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
125
|
+
|
126
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [:container_groups])
|
127
|
+
end
|
128
|
+
|
129
|
+
it "with normal targeted scope " do
|
130
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
|
131
|
+
|
132
|
+
collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, {:container_groups => [{:source_ref => "a"}]})
|
133
|
+
end
|
134
|
+
|
135
|
+
it "fails with scope entity too large " do
|
136
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
137
|
+
|
138
|
+
sweep_scope = {:container_groups => [{:source_ref => "a" * (1_000_002 * multiplier)}]}
|
139
|
+
|
140
|
+
expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
|
141
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
142
|
+
)
|
143
|
+
end
|
144
|
+
|
145
|
+
it "fails when scope is too big " do
|
146
|
+
# We should have also sweep scope chunking, that is if we'll do big targeted refresh and sweeping
|
147
|
+
expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
|
148
|
+
|
149
|
+
sweep_scope = {:container_groups => (0..1001 * multiplier).map { {:source_ref => "a" * 1_000} } }
|
150
|
+
|
151
|
+
expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
|
152
|
+
raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
|
153
|
+
)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def build_inventory(collections)
|
158
|
+
TopologicalInventoryIngressApiClient::Inventory.new(
|
159
|
+
:name => collector.send(:inventory_name),
|
160
|
+
:schema => TopologicalInventoryIngressApiClient::Schema.new(:name => collector.send(:schema_name)),
|
161
|
+
:source => source,
|
162
|
+
:collections => collections,
|
163
|
+
:refresh_state_uuid => refresh_state_uuid,
|
164
|
+
:refresh_state_part_uuid => refresh_state_part_uuid,
|
165
|
+
)
|
166
|
+
end
|
167
|
+
|
168
|
+
def inventory_size(collections)
|
169
|
+
JSON.generate(build_inventory(collections).to_hash).size
|
170
|
+
end
|
171
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: topological_inventory-providers-common
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Martin Slemr
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -257,6 +257,7 @@ files:
|
|
257
257
|
- spec/spec_helper.rb
|
258
258
|
- spec/support/inventory_helper.rb
|
259
259
|
- spec/support/shared/availability_check.rb
|
260
|
+
- spec/topological_inventory/providers/common/collector_spec.rb
|
260
261
|
- spec/topological_inventory/providers/common/collectors/inventory_collection_storage_spec.rb
|
261
262
|
- spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb
|
262
263
|
- spec/topological_inventory/providers/common/collectors_pool_spec.rb
|