topological_inventory-providers-common 1.0.5 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e289728fe42a0bdf12142485a444c851948e9db8ff215c6444f8a7b7bd98c409
4
- data.tar.gz: 54241e05b5d38f449a905b4edecd725b5110eaba96c213098ade88643601de76
3
+ metadata.gz: aafb37efd571f568a887c8c11b31bc87a6141a556ae06b970bf4acfcbdea3d08
4
+ data.tar.gz: 2d1024891611a96df454a56819a5f6f921e195605d3867ca05c595c336f9a699
5
5
  SHA512:
6
- metadata.gz: 14d64a1f0c4b208a43692905a3e843f6c255145cda9cc1a928b8db0a1da02a387cca940146d8044cdcdfbad15bee694e71660d318a2184cf7b1f920f1d6e86fe
7
- data.tar.gz: 5f6d8af6dff4fe080e223a9fb2e0ae81ac8d4c3789f5ef832043ad3b08f52ba2d72590963096a02eb9efc9ce31d6a0e36929a1d89c0ad54a2985aa7871921a74
6
+ metadata.gz: daff9b2dc2388297dba831994c74e048d608256c9fd3aadc06446c16abfa4143f5fcbed2e6dd3eff66cb6fbb9e6637abe8f47c1eda452b99d5557c9424c20849
7
+ data.tar.gz: ece193799a908c7ca96aee73de80e9a6dc7cf17346b549ebf2d802cd3ed42b11aec7b46837a4078c1569fe608ef0241f7ac64a7daf43bd2905df1d53e21c8e01
@@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file.
4
4
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5
5
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
6
 
7
- ## [1.0.5]
7
+ ## [1.0.6] - 2020-07-06
8
+ Add some error handling if Sources does not have endpoints/authentications for a source #38
9
+ Specs for Collector #35
10
+
11
+ ## [1.0.5] - 2020-06-18
8
12
  Change release workflow to do everything manually #32
9
13
  Add specs to released files #33
10
14
 
@@ -34,7 +38,8 @@ manageiq-loggers to >= 0.4.2 #20
34
38
  ## [1.0.0] - 2020-03-19
35
39
  ### Initial release to rubygems.org
36
40
 
37
- [Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...HEAD
41
+ [Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...HEAD
42
+ [1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
38
43
  [1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
39
44
  [1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
40
45
  [1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3
@@ -116,10 +116,16 @@ module TopologicalInventory
116
116
 
117
117
  def endpoint
118
118
  @endpoint ||= api_client.fetch_default_endpoint(source_id)
119
+ rescue StandardError => e
120
+ logger.availability_check("Failed to fetch Endpoint for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
121
+ return
119
122
  end
120
123
 
121
124
  def authentication
122
125
  @authentication ||= api_client.fetch_authentication(source_id, endpoint)
126
+ rescue StandardError => e
127
+ logger.availability_check("Failed to fetch Authentication for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
128
+ return
123
129
  end
124
130
 
125
131
  def check_time
@@ -1,7 +1,7 @@
1
1
  module TopologicalInventory
2
2
  module Providers
3
3
  module Common
4
- VERSION = "1.0.5"
4
+ VERSION = "1.0.6"
5
5
  end
6
6
  end
7
7
  end
@@ -0,0 +1,171 @@
1
+ RSpec.describe TopologicalInventory::Providers::Common::Collector do
2
+ let(:collector) do
3
+ collector = described_class.new(source)
4
+
5
+ allow(collector).to receive(:ingress_api_client).and_return(client)
6
+ allow(collector).to receive(:logger).and_return(logger)
7
+ allow(logger).to receive(:error)
8
+
9
+ collector
10
+ end
11
+
12
+ let(:parser) { TopologicalInventory::Providers::Common::Collector::Parser.new }
13
+
14
+ let(:source) { "source_uid" }
15
+ let(:client) { double }
16
+ let(:logger) { double }
17
+ let(:refresh_state_uuid) { SecureRandom.uuid }
18
+ let(:refresh_state_part_uuid) { SecureRandom.uuid }
19
+ # based on the default, we can tell how many chunks the saver will break the payload up into
20
+ let(:max_size) { TopologicalInventory::Providers::Common::SaveInventory::Saver::KAFKA_PAYLOAD_MAX_BYTES_DEFAULT }
21
+ let(:multiplier) { 0.75 }
22
+
23
+ context "#save_inventory" do
24
+ it "does nothing with empty collections" do
25
+ parts = collector.send(:save_inventory, [], collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
26
+
27
+ expect(parts).to eq 0
28
+ end
29
+
30
+ it "saves 1 part if it fits" do
31
+ (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
32
+
33
+ expect(inventory_size(parser.collections.values) / max_size).to eq(0)
34
+
35
+ expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
36
+ parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
37
+ expect(parts).to eq 1
38
+ end
39
+
40
+ it "saves 2 parts if over limit with 1 collection" do
41
+ (multiplier * 2000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
42
+
43
+ expect(inventory_size(parser.collections.values) / max_size).to eq(1)
44
+
45
+ expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
46
+ parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
47
+ expect(parts).to eq 2
48
+ end
49
+
50
+ it "saves 2 parts if over limit with 2 collections" do
51
+ (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
52
+ (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
53
+
54
+ expect(inventory_size(parser.collections.values) / max_size).to eq(1)
55
+
56
+ expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
57
+ parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
58
+ expect(parts).to eq 2
59
+ end
60
+
61
+ it "saves many parts" do
62
+ (multiplier * 1500).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
63
+ (multiplier * 2000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
64
+
65
+ expect(client).to receive(:save_inventory_with_http_info).exactly(4).times
66
+ parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
67
+ expect(parts).to eq 4
68
+ end
69
+
70
+ it 'raises exception when entity to save is too big' do
71
+ parser.collections.container_groups.build(:source_ref => "a" * (1_000_000 * multiplier))
72
+
73
+ expect(inventory_size(parser.collections.values) / max_size).to eq(1)
74
+ # in this case, we first save empty inventory, then the size check fails saving the rest of data
75
+ expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
76
+
77
+ expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
78
+ raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
79
+ )
80
+ end
81
+
82
+ it 'raises exception when entity of second collection is too big' do
83
+ (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
84
+ parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
85
+
86
+ expect(inventory_size(parser.collections.values) / max_size).to eq(1)
87
+ # We save the first collection then it fails on saving the second collection
88
+ expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
89
+
90
+ expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
91
+ raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
92
+ )
93
+ end
94
+
95
+ it 'raises exception when entity of second collection is too big then continues with smaller' do
96
+ (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
97
+ parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
98
+ (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
99
+
100
+ expect(inventory_size(parser.collections.values) / max_size).to eq(2)
101
+ # We save the first collection then it fails on saving the second collection
102
+ expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
103
+
104
+ expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
105
+ raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
106
+ )
107
+ end
108
+ end
109
+
110
+ context "#sweep_inventory" do
111
+ it "with nil total parts" do
112
+ expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
113
+
114
+ collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, nil, [])
115
+ end
116
+
117
+ it "with empty scope " do
118
+ expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
119
+
120
+ collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [])
121
+ end
122
+
123
+ it "with normal scope " do
124
+ expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
125
+
126
+ collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [:container_groups])
127
+ end
128
+
129
+ it "with normal targeted scope " do
130
+ expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
131
+
132
+ collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, {:container_groups => [{:source_ref => "a"}]})
133
+ end
134
+
135
+ it "fails with scope entity too large " do
136
+ expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
137
+
138
+ sweep_scope = {:container_groups => [{:source_ref => "a" * (1_000_002 * multiplier)}]}
139
+
140
+ expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
141
+ raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
142
+ )
143
+ end
144
+
145
+ it "fails when scope is too big " do
146
+ # We should have also sweep scope chunking, that is if we'll do big targeted refresh and sweeping
147
+ expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
148
+
149
+ sweep_scope = {:container_groups => (0..1001 * multiplier).map { {:source_ref => "a" * 1_000} } }
150
+
151
+ expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
152
+ raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
153
+ )
154
+ end
155
+ end
156
+
157
+ def build_inventory(collections)
158
+ TopologicalInventoryIngressApiClient::Inventory.new(
159
+ :name => collector.send(:inventory_name),
160
+ :schema => TopologicalInventoryIngressApiClient::Schema.new(:name => collector.send(:schema_name)),
161
+ :source => source,
162
+ :collections => collections,
163
+ :refresh_state_uuid => refresh_state_uuid,
164
+ :refresh_state_part_uuid => refresh_state_part_uuid,
165
+ )
166
+ end
167
+
168
+ def inventory_size(collections)
169
+ JSON.generate(build_inventory(collections).to_hash).size
170
+ end
171
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: topological_inventory-providers-common
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.5
4
+ version: 1.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Martin Slemr
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-06-18 00:00:00.000000000 Z
11
+ date: 2020-07-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -257,6 +257,7 @@ files:
257
257
  - spec/spec_helper.rb
258
258
  - spec/support/inventory_helper.rb
259
259
  - spec/support/shared/availability_check.rb
260
+ - spec/topological_inventory/providers/common/collector_spec.rb
260
261
  - spec/topological_inventory/providers/common/collectors/inventory_collection_storage_spec.rb
261
262
  - spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb
262
263
  - spec/topological_inventory/providers/common/collectors_pool_spec.rb