topological_inventory-providers-common 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e289728fe42a0bdf12142485a444c851948e9db8ff215c6444f8a7b7bd98c409
-  data.tar.gz: 54241e05b5d38f449a905b4edecd725b5110eaba96c213098ade88643601de76
+  metadata.gz: aafb37efd571f568a887c8c11b31bc87a6141a556ae06b970bf4acfcbdea3d08
+  data.tar.gz: 2d1024891611a96df454a56819a5f6f921e195605d3867ca05c595c336f9a699
 SHA512:
-  metadata.gz: 14d64a1f0c4b208a43692905a3e843f6c255145cda9cc1a928b8db0a1da02a387cca940146d8044cdcdfbad15bee694e71660d318a2184cf7b1f920f1d6e86fe
-  data.tar.gz: 5f6d8af6dff4fe080e223a9fb2e0ae81ac8d4c3789f5ef832043ad3b08f52ba2d72590963096a02eb9efc9ce31d6a0e36929a1d89c0ad54a2985aa7871921a74
+  metadata.gz: daff9b2dc2388297dba831994c74e048d608256c9fd3aadc06446c16abfa4143f5fcbed2e6dd3eff66cb6fbb9e6637abe8f47c1eda452b99d5557c9424c20849
+  data.tar.gz: ece193799a908c7ca96aee73de80e9a6dc7cf17346b549ebf2d802cd3ed42b11aec7b46837a4078c1569fe608ef0241f7ac64a7daf43bd2905df1d53e21c8e01

data/CHANGELOG.md

@@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.5]
+## [1.0.6] - 2020-07-06
+Add some error handling if Sources does not have endpoints/authentications for a source #38
+Specs for Collector #35
+
+## [1.0.5] - 2020-06-18
 Change release workflow to do everything manually #32
 Add specs to released files #33
 
@@ -34,7 +38,8 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...HEAD
+[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
 [1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
 [1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
 [1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -116,10 +116,16 @@ module TopologicalInventory
 
         def endpoint
           @endpoint ||= api_client.fetch_default_endpoint(source_id)
+        rescue StandardError => e
+          logger.availability_check("Failed to fetch Endpoint for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
+          return
         end
 
         def authentication
           @authentication ||= api_client.fetch_authentication(source_id, endpoint)
+        rescue StandardError => e
+          logger.availability_check("Failed to fetch Authentication for Source #{source_id}, Endpoint #{endpoint}: #{e.message}", :error)
+          return
         end
 
         def check_time

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "1.0.5"
+      VERSION = "1.0.6"
     end
   end
 end

data/spec/topological_inventory/providers/common/collector_spec.rb

@@ -0,0 +1,171 @@
+RSpec.describe TopologicalInventory::Providers::Common::Collector do
+  let(:collector) do
+    collector = described_class.new(source)
+
+    allow(collector).to receive(:ingress_api_client).and_return(client)
+    allow(collector).to receive(:logger).and_return(logger)
+    allow(logger).to receive(:error)
+
+    collector
+  end
+
+  let(:parser) { TopologicalInventory::Providers::Common::Collector::Parser.new }
+
+  let(:source)  { "source_uid" }
+  let(:client)  { double }
+  let(:logger)  { double }
+  let(:refresh_state_uuid) { SecureRandom.uuid }
+  let(:refresh_state_part_uuid) { SecureRandom.uuid }
+  # based on the default, we can tell how many chunks the saver will break the payload up into
+  let(:max_size) { TopologicalInventory::Providers::Common::SaveInventory::Saver::KAFKA_PAYLOAD_MAX_BYTES_DEFAULT }
+  let(:multiplier) { 0.75 }
+
+  context "#save_inventory" do
+    it "does nothing with empty collections" do
+      parts = collector.send(:save_inventory, [], collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+
+      expect(parts).to eq 0
+    end
+
+    it "saves 1 part if it fits" do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(0)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 1
+    end
+
+    it "saves 2 parts if over limit with 1 collection" do
+      (multiplier * 2000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 2
+    end
+
+    it "saves 2 parts if over limit with 2 collections" do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 2
+    end
+
+    it "saves many parts" do
+      (multiplier * 1500).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      (multiplier * 2000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(4).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 4
+    end
+
+    it 'raises exception when entity to save is too big' do
+      parser.collections.container_groups.build(:source_ref => "a" * (1_000_000 * multiplier))
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+      # in this case, we first save empty inventory, then the size check fails saving the rest of data
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it 'raises exception when entity of second collection is too big' do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+      # We save the first collection then it fails on saving the second collection
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it 'raises exception when entity of second collection is too big then continues with smaller' do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
+      (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(2)
+      # We save the first collection then it fails on saving the second collection
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+  end
+
+  context "#sweep_inventory" do
+    it "with nil total parts" do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, nil, [])
+    end
+
+    it "with empty scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [])
+    end
+
+    it "with normal scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [:container_groups])
+    end
+
+    it "with normal targeted scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, {:container_groups => [{:source_ref => "a"}]})
+    end
+
+    it "fails with scope entity too large " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      sweep_scope = {:container_groups => [{:source_ref => "a" * (1_000_002 * multiplier)}]}
+
+      expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it "fails when scope is too big " do
+      # We should have also sweep scope chunking, that is if we'll do big targeted refresh and sweeping
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      sweep_scope = {:container_groups => (0..1001 * multiplier).map { {:source_ref => "a" * 1_000} } }
+
+      expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+  end
+
+  def build_inventory(collections)
+    TopologicalInventoryIngressApiClient::Inventory.new(
+      :name                    => collector.send(:inventory_name),
+      :schema                  => TopologicalInventoryIngressApiClient::Schema.new(:name => collector.send(:schema_name)),
+      :source                  => source,
+      :collections             => collections,
+      :refresh_state_uuid      => refresh_state_uuid,
+      :refresh_state_part_uuid => refresh_state_part_uuid,
+    )
+  end
+
+  def inventory_size(collections)
+    JSON.generate(build_inventory(collections).to_hash).size
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-providers-common
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-18 00:00:00.000000000 Z
+date: 2020-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -257,6 +257,7 @@ files:
 - spec/spec_helper.rb
 - spec/support/inventory_helper.rb
 - spec/support/shared/availability_check.rb
+- spec/topological_inventory/providers/common/collector_spec.rb
 - spec/topological_inventory/providers/common/collectors/inventory_collection_storage_spec.rb
 - spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb
 - spec/topological_inventory/providers/common/collectors_pool_spec.rb