topological_inventory-providers-common 1.0.5 → 1.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e289728fe42a0bdf12142485a444c851948e9db8ff215c6444f8a7b7bd98c409
-  data.tar.gz: 54241e05b5d38f449a905b4edecd725b5110eaba96c213098ade88643601de76
+  metadata.gz: cd89d895ea401fd0692588fb0f7dc86d6df4a4ec189e73ec4453eafc2be13c7e
+  data.tar.gz: 933d448f102d927a62153b75eadd2fb835d1f6fa5a72496ddbaf830a040ff1cc
 SHA512:
-  metadata.gz: 14d64a1f0c4b208a43692905a3e843f6c255145cda9cc1a928b8db0a1da02a387cca940146d8044cdcdfbad15bee694e71660d318a2184cf7b1f920f1d6e86fe
-  data.tar.gz: 5f6d8af6dff4fe080e223a9fb2e0ae81ac8d4c3789f5ef832043ad3b08f52ba2d72590963096a02eb9efc9ce31d6a0e36929a1d89c0ad54a2985aa7871921a74
+  metadata.gz: 012ad9a5e0f75a7a9970c31b269fdd7f0ef740ec7626c585ff003ba7cc784ba32995208ebef39de05666e1e442b2104aa44685d2b0d2855c012744e7553c7ea1
+  data.tar.gz: 73590bf64ce08ae60a3693f0214a60f31a2f5b82780542180bf7d6a5e8b32d156b2e4fa2e85cea8088b8ef0178737ca365b5e932117b52becf8777cc5916acd1

data/CHANGELOG.md

@@ -4,7 +4,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.5]
+## [1.0.10]
+Add HealthCheck class for operations workers #48
+Set the LOG_LEVEL if present #50
+
+## [1.0.9]
+Added refresh-type to save and sweep inventory #45
+
+## [1.0.8] - 2020-08-12
+Add => to error messages that rubocop missed #44
+
+## [1.0.7] - 2020-07-27
+Update operations/source model for receptor-enabled availability checks #36
+Add check for Application subresource under a Source during Availability check #40
+Remove infinite loop in error messages #43
+
+## [1.0.6] - 2020-07-06
+Add some error handling if Sources does not have endpoints/authentications for a source #38
+Specs for Collector #35
+
+## [1.0.5] - 2020-06-18
 Change release workflow to do everything manually #32
 Add specs to released files #33
 
@@ -34,7 +53,12 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.10...HEAD
+[1.0.10]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.9...v1.0.10
+[1.0.9]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.8...v1.0.9
+[1.0.8]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.7...v1.0.8
+[1.0.7]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...v1.0.7
+[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
 [1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
 [1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
 [1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3

data/lib/topological_inventory/providers/common.rb

@@ -2,6 +2,7 @@ require "topological_inventory/providers/common/version"
 require "topological_inventory/providers/common/logging"
 require "topological_inventory/providers/common/operations/processor"
 require "topological_inventory/providers/common/operations/endpoint_client"
+require "topological_inventory/providers/common/operations/health_check"
 require "topological_inventory/providers/common/collectors_pool"
 require "topological_inventory/providers/common/collector"
 

data/lib/topological_inventory/providers/common/collector.rb

@@ -104,7 +104,8 @@ module TopologicalInventory
                            refresh_state_uuid = nil,
                            refresh_state_part_uuid = nil,
                            refresh_state_part_collected_at = nil,
-                           refresh_state_part_sent_at = Time.now.utc)
+                           refresh_state_part_sent_at = Time.now.utc,
+                           refresh_type = default_refresh_type)
           return 0 if collections.empty?
 
           SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
@@ -116,7 +117,8 @@ module TopologicalInventory
               :refresh_state_uuid              => refresh_state_uuid,
               :refresh_state_part_uuid         => refresh_state_part_uuid,
               :refresh_state_part_collected_at => refresh_state_part_collected_at,
-              :refresh_state_part_sent_at      => refresh_state_part_sent_at
+              :refresh_state_part_sent_at      => refresh_state_part_sent_at,
+              :refresh_type                    => refresh_type
             )
           )
         rescue => e
@@ -134,7 +136,8 @@ module TopologicalInventory
                             total_parts,
                             sweep_scope,
                             refresh_state_started_at = nil,
-                            refresh_state_sent_at = Time.now.utc)
+                            refresh_state_sent_at = Time.now.utc,
+                            refresh_type = default_refresh_type)
           return if !total_parts || sweep_scope.empty?
 
           SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
@@ -147,7 +150,8 @@ module TopologicalInventory
               :total_parts              => total_parts,
               :sweep_scope              => sweep_scope,
               :refresh_state_started_at => refresh_state_started_at,
-              :refresh_state_sent_at    => refresh_state_sent_at
+              :refresh_state_sent_at    => refresh_state_sent_at,
+              :refresh_type             => refresh_type
             )
           )
         rescue => e
@@ -165,6 +169,10 @@ module TopologicalInventory
           "Default"
         end
 
+        def default_refresh_type
+          'full-refresh'
+        end
+
         def ingress_api_client
           TopologicalInventoryIngressApiClient::DefaultApi.new
         end

data/lib/topological_inventory/providers/common/logging.rb

@@ -33,7 +33,10 @@ module TopologicalInventory
 
       class Logger < ManageIQ::Loggers::CloudWatch
         def self.new(*args)
-          super.tap { |logger| logger.extend(TopologicalInventory::Providers::Common::LoggingFunctions) }
+          super.tap do |logger|
+            logger.extend(TopologicalInventory::Providers::Common::LoggingFunctions)
+            logger.level = ENV['LOG_LEVEL'] if ENV['LOG_LEVEL']
+          end
         end
       end
 

data/lib/topological_inventory/providers/common/operations/health_check.rb

@@ -0,0 +1,15 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class HealthCheck
+          HEARTBEAT_FILE = '/tmp/healthy'.freeze
+
+          def self.touch_file
+            FileUtils.touch(HEARTBEAT_FILE)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -12,18 +12,20 @@ module TopologicalInventory
         STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
 
         ERROR_MESSAGES = {
-          :authentication_not_found => "Authentication not found in Sources API",
-          :endpoint_not_found       => "Endpoint not found in Sources API",
+          :authentication_not_found          => "Authentication not found in Sources API",
+          :endpoint_or_application_not_found => "Endpoint or Application not found in Sources API",
         }.freeze
 
         LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
+        AUTH_NOT_NECESSARY = "n/a".freeze
 
-        attr_accessor :params, :request_context, :source_id
+        attr_accessor :params, :request_context, :source_id, :account_number
 
         def initialize(params = {}, request_context = nil)
           self.params          = params
           self.request_context = request_context
           self.source_id       = params['source_id']
+          self.account_number  = params['external_tenant']
         end
 
         def availability_check
@@ -33,7 +35,7 @@ module TopologicalInventory
 
           status, error_message = connection_status
 
-          update_source_and_endpoint(status, error_message)
+          update_source_and_subresources(status, error_message)
 
           logger.availability_check("Completed: Source #{source_id} is #{status}")
         end
@@ -57,32 +59,57 @@ module TopologicalInventory
         end
 
         def checked_recently?
-          return false if endpoint.nil?
+          checked_recently = if endpoint.present?
+                               endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
+                             elsif application.present?
+                               application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
+                             end
 
-          checked_recently = endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
-          logger.availability_check("Skipping, last check at #{endpoint.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
+          logger.availability_check("Skipping, last check at #{endpoint.last_checked_at || application.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
 
           checked_recently
         end
 
         def connection_status
-          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_not_found]] unless endpoint
-          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
+          # we need either an endpoint or application to check the source.
+          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_or_application_not_found]] unless endpoint || application
 
           check_time
+          if endpoint
+            endpoint_connection_check
+          elsif application
+            application_connection_check
+          end
+        end
+
+        def endpoint_connection_check
+          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
+
+          # call down into the operations pod implementation of `Source#connection_check`
           connection_check
         end
 
+        def application_connection_check
+          case application.availability_status
+          when "available"
+            [STATUS_AVAILABLE, nil]
+          when "unavailable"
+            [STATUS_UNAVAILABLE, "Application id #{application.id} unavailable"]
+          end
+        end
+
         # @return [Array<String, String|nil] - STATUS_[UN]AVAILABLE, error message
         def connection_check
           raise NotImplementedError, "#{__method__} must be implemented in a subclass"
         end
 
-        def update_source_and_endpoint(status, error_message = nil)
+        def update_source_and_subresources(status, error_message = nil)
           logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
 
           update_source(status)
-          update_endpoint(status, error_message)
+
+          update_endpoint(status, error_message) if endpoint
+          update_application(status) if application
         end
 
         def update_source(status)
@@ -114,12 +141,39 @@ module TopologicalInventory
           logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
         end
 
+        def update_application(status)
+          application_update = ::SourcesApiClient::Application.new
+          application_update.last_checked_at     = check_time
+          application_update.last_available_at   = check_time if status == STATUS_AVAILABLE
+
+          api_client.update_application(application.id, application_update)
+        rescue ::SourcesApiClient::ApiError => e
+          logger.availability_check("Failed to update Application id: #{application.id} - #{e.message}", :error)
+        end
+
         def endpoint
           @endpoint ||= api_client.fetch_default_endpoint(source_id)
+        rescue => e
+          logger.availability_check("Failed to fetch Endpoint for Source #{source_id}: #{e.message}", :error)
+          nil
         end
 
         def authentication
-          @authentication ||= api_client.fetch_authentication(source_id, endpoint)
+          @authentication ||= if endpoint.receptor_node.present?
+                                AUTH_NOT_NECESSARY
+                              else
+                                api_client.fetch_authentication(source_id, endpoint)
+                              end
+        rescue => e
+          logger.availability_check("Failed to fetch Authentication for Source #{source_id}: #{e.message}", :error)
+          nil
+        end
+
+        def application
+          @application ||= api_client.fetch_application(source_id)
+        rescue => e
+          logger.availability_check("Failed to fetch Application for Source #{source_id}: #{e.message}", :error)
+          nil
         end
 
         def check_time
@@ -127,7 +181,7 @@ module TopologicalInventory
         end
 
         def identity
-          @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => params["external_tenant"], "user" => {"is_org_admin" => true}}}.to_json)}
+          @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => account_number, "user" => {"is_org_admin" => true}}}.to_json)}
         end
 
         def api_client

data/lib/topological_inventory/providers/common/operations/sources_api_client.rb

@@ -5,7 +5,7 @@ module TopologicalInventory
     module Common
       module Operations
         class SourcesApiClient < ::SourcesApiClient::ApiClient
-          delegate :update_source, :update_endpoint, :to => :api
+          delegate :update_source, :update_endpoint, :update_application, :to => :api
 
           INTERNAL_API_PATH = '//internal/v1.0'.freeze
 
@@ -25,6 +25,11 @@ module TopologicalInventory
             endpoints.find(&:default)
           end
 
+          def fetch_application(source_id)
+            applications = api.list_source_applications(source_id)&.data || []
+            applications.first
+          end
+
           def fetch_authentication(source_id, default_endpoint = nil, authtype = nil)
             endpoint = default_endpoint || fetch_default_endpoint(source_id)
             return if endpoint.nil?

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "1.0.5"
+      VERSION = "1.0.10"
     end
   end
 end

data/spec/support/shared/availability_check.rb

@@ -11,6 +11,7 @@ RSpec.shared_examples "availability_check" do
   let(:headers) { {'Content-Type' => 'application/json'}.merge(identity) }
   let(:source_id) { '123' }
   let(:endpoint_id) { '234' }
+  let(:application_id) { '345' }
   let(:authentication_id) { '345' }
   let(:payload) do
     {
@@ -26,6 +27,8 @@ RSpec.shared_examples "availability_check" do
   let(:list_endpoint_authentications_response) { "{\"data\":[{\"authtype\":\"username_password\",\"id\":\"#{authentication_id}\",\"resource_id\":\"#{endpoint_id}\",\"resource_type\":\"Endpoint\",\"username\":\"admin\",\"tenant\":\"#{external_tenant}\"}]}" }
   let(:list_endpoint_authentications_response_empty) { "{\"data\":[]}" }
   let(:internal_api_authentication_response) { "{\"authtype\":\"username_password\",\"id\":\"#{authentication_id}\",\"resource_id\":\"#{endpoint_id}\",\"resource_type\":\"Endpoint\",\"username\":\"admin\",\"tenant\":\"#{external_tenant}\",\"password\":\"xxx\"}" }
+  let(:list_applications_response) { {:data => [{:id => "345", :availability_status => "available"}]}.to_json }
+  let(:list_applications_unavailable_response) { {:data => [{:id => "345", :availability_status => "unavailable"}]}.to_json }
 
   subject { described_class.new(payload["params"]) }
 
@@ -39,6 +42,7 @@ RSpec.shared_examples "availability_check" do
       stub_get(:endpoint, list_endpoints_response)
       stub_get(:authentication, list_endpoint_authentications_response)
       stub_get(:password, internal_api_authentication_response)
+      stub_not_found(:application)
 
       # PATCH
       source_patch_body   = {'availability_status' => described_class::STATUS_AVAILABLE, 'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
@@ -61,6 +65,7 @@ RSpec.shared_examples "availability_check" do
       stub_get(:endpoint, list_endpoints_response)
       stub_get(:authentication, list_endpoint_authentications_response)
       stub_get(:password, internal_api_authentication_response)
+      stub_not_found(:application)
 
       # PATCH
       connection_error_message = "Some connection error"
@@ -81,7 +86,8 @@ RSpec.shared_examples "availability_check" do
 
     it "updates only Source to 'unavailable' status if Endpoint not found" do
       # GET
-      stub_get(:endpoint, '')
+      stub_not_found(:endpoint)
+      stub_not_found(:application)
 
       # PATCH
       source_patch_body = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
@@ -100,6 +106,7 @@ RSpec.shared_examples "availability_check" do
       # GET
       stub_get(:endpoint, list_endpoints_response)
       stub_get(:authentication, list_endpoint_authentications_response_empty)
+      stub_not_found(:application)
 
       # PATCH
       source_patch_body   = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
@@ -131,6 +138,49 @@ RSpec.shared_examples "availability_check" do
     end
   end
 
+  context "when there is an application" do
+    context "when it is available" do
+      it "updates the availability status to available" do
+        # GET
+        stub_not_found(:endpoint)
+        stub_get(:application, list_applications_response)
+        # PATCH
+        application_patch_body = {'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
+        source_patch_body = {'availability_status' => described_class::STATUS_AVAILABLE, 'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
+
+        stub_patch(:source, source_patch_body)
+        stub_patch(:application, application_patch_body)
+
+        # Check
+        expect(subject).not_to receive(:connection_check)
+        subject.availability_check
+
+        assert_patch(:source, source_patch_body)
+        assert_patch(:application, application_patch_body)
+      end
+    end
+
+    context "when it is unavailable" do
+      it "updates the availability status to unavailable" do
+        # GET
+        stub_not_found(:endpoint)
+        stub_get(:application, list_applications_unavailable_response)
+        # PATCH
+        application_patch_body = {'last_checked_at' => subject.send(:check_time)}.to_json
+        source_patch_body = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
+
+        stub_patch(:source, source_patch_body)
+        stub_patch(:application, application_patch_body)
+
+        # Check
+        expect(subject).not_to receive(:connection_check)
+        subject.availability_check
+
+        assert_patch(:source, source_patch_body)
+        assert_patch(:application, application_patch_body)
+      end
+    end
+  end
 
   def stub_get(object_type, response)
     case object_type
@@ -146,6 +196,31 @@ RSpec.shared_examples "availability_check" do
       stub_request(:get, "#{host_url}#{sources_internal_api_path}/authentications/#{authentication_id}?expose_encrypted_attribute%5B%5D=password")
         .with(:headers => headers)
         .to_return(:status => 200, :body => response, :headers => {})
+    when :application
+      stub_request(:get, "#{sources_api_url}/sources/#{source_id}/applications")
+        .with(:headers => headers)
+        .to_return(:status => 200, :body => response, :headers => {})
+    end
+  end
+
+  def stub_not_found(object_type)
+    case object_type
+    when :endpoint
+      stub_request(:get, "#{sources_api_url}/sources/#{source_id}/endpoints")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
+    when :authentication
+      stub_request(:get, "#{sources_api_url}/endpoints/#{endpoint_id}/authentications")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
+    when :password
+      stub_request(:get, "#{host_url}#{sources_internal_api_path}/authentications/#{authentication_id}?expose_encrypted_attribute%5B%5D=password")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
+    when :application
+      stub_request(:get, "#{sources_api_url}/sources/#{source_id}/applications")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
     end
   end
 
@@ -159,6 +234,10 @@ RSpec.shared_examples "availability_check" do
       stub_request(:patch, "#{sources_api_url}/endpoints/#{endpoint_id}")
         .with(:body => data, :headers => headers)
         .to_return(:status => 200, :body => "", :headers => {})
+    when :application
+      stub_request(:patch, "#{sources_api_url}/applications/#{application_id}")
+        .with(:body => data, :headers => headers)
+        .to_return(:status => 200, :body => "", :headers => {})
     end
   end
 
@@ -166,10 +245,13 @@ RSpec.shared_examples "availability_check" do
     case object_type
     when :source
       expect(WebMock).to have_requested(:patch, "#{sources_api_url}/sources/#{source_id}")
-                           .with(:body => data, :headers => headers).once
+        .with(:body => data, :headers => headers).once
     when :endpoint
       expect(WebMock).to have_requested(:patch, "#{sources_api_url}/endpoints/#{endpoint_id}")
-                           .with(:body => data, :headers => headers).once
+        .with(:body => data, :headers => headers).once
+    when :application
+      expect(WebMock).to have_requested(:patch, "#{sources_api_url}/applications/#{application_id}")
+        .with(:body => data, :headers => headers).once
     end
   end
 end

data/spec/topological_inventory/providers/common/collector_spec.rb

@@ -0,0 +1,171 @@
+RSpec.describe TopologicalInventory::Providers::Common::Collector do
+  let(:collector) do
+    collector = described_class.new(source)
+
+    allow(collector).to receive(:ingress_api_client).and_return(client)
+    allow(collector).to receive(:logger).and_return(logger)
+    allow(logger).to receive(:error)
+
+    collector
+  end
+
+  let(:parser) { TopologicalInventory::Providers::Common::Collector::Parser.new }
+
+  let(:source)  { "source_uid" }
+  let(:client)  { double }
+  let(:logger)  { double }
+  let(:refresh_state_uuid) { SecureRandom.uuid }
+  let(:refresh_state_part_uuid) { SecureRandom.uuid }
+  # based on the default, we can tell how many chunks the saver will break the payload up into
+  let(:max_size) { TopologicalInventory::Providers::Common::SaveInventory::Saver::KAFKA_PAYLOAD_MAX_BYTES_DEFAULT }
+  let(:multiplier) { 0.75 }
+
+  context "#save_inventory" do
+    it "does nothing with empty collections" do
+      parts = collector.send(:save_inventory, [], collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+
+      expect(parts).to eq 0
+    end
+
+    it "saves 1 part if it fits" do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(0)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 1
+    end
+
+    it "saves 2 parts if over limit with 1 collection" do
+      (multiplier * 2000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 2
+    end
+
+    it "saves 2 parts if over limit with 2 collections" do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 2
+    end
+
+    it "saves many parts" do
+      (multiplier * 1500).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      (multiplier * 2000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(4).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 4
+    end
+
+    it 'raises exception when entity to save is too big' do
+      parser.collections.container_groups.build(:source_ref => "a" * (1_000_000 * multiplier))
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+      # in this case, we first save empty inventory, then the size check fails saving the rest of data
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it 'raises exception when entity of second collection is too big' do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+      # We save the first collection then it fails on saving the second collection
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it 'raises exception when entity of second collection is too big then continues with smaller' do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
+      (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(2)
+      # We save the first collection then it fails on saving the second collection
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+  end
+
+  context "#sweep_inventory" do
+    it "with nil total parts" do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, nil, [])
+    end
+
+    it "with empty scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [])
+    end
+
+    it "with normal scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [:container_groups])
+    end
+
+    it "with normal targeted scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, {:container_groups => [{:source_ref => "a"}]})
+    end
+
+    it "fails with scope entity too large " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      sweep_scope = {:container_groups => [{:source_ref => "a" * (1_000_002 * multiplier)}]}
+
+      expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it "fails when scope is too big " do
+      # We should have also sweep scope chunking, that is if we'll do big targeted refresh and sweeping
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      sweep_scope = {:container_groups => (0..1001 * multiplier).map { {:source_ref => "a" * 1_000} } }
+
+      expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+  end
+
+  def build_inventory(collections)
+    TopologicalInventoryIngressApiClient::Inventory.new(
+      :name                    => collector.send(:inventory_name),
+      :schema                  => TopologicalInventoryIngressApiClient::Schema.new(:name => collector.send(:schema_name)),
+      :source                  => source,
+      :collections             => collections,
+      :refresh_state_uuid      => refresh_state_uuid,
+      :refresh_state_part_uuid => refresh_state_part_uuid,
+    )
+  end
+
+  def inventory_size(collections)
+    JSON.generate(build_inventory(collections).to_hash).size
+  end
+end

data/topological_inventory-providers-common.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "manageiq-loggers", ">= 0.4.2"
   spec.add_runtime_dependency "sources-api-client", "~> 3.0"
   spec.add_runtime_dependency "topological_inventory-api-client", "~> 3.0", ">= 3.0.1"
-  spec.add_runtime_dependency "topological_inventory-ingress_api-client", "~> 1.0"
+  spec.add_runtime_dependency "topological_inventory-ingress_api-client", "~> 1.0", ">= 1.0.3"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", ">= 12.3.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-providers-common
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.10
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-18 00:00:00.000000000 Z
+date: 2020-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -113,6 +113,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -120,6 +123,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -247,6 +253,7 @@ files:
 - lib/topological_inventory/providers/common/collectors_pool.rb
 - lib/topological_inventory/providers/common/logging.rb
 - lib/topological_inventory/providers/common/operations/endpoint_client.rb
+- lib/topological_inventory/providers/common/operations/health_check.rb
 - lib/topological_inventory/providers/common/operations/processor.rb
 - lib/topological_inventory/providers/common/operations/source.rb
 - lib/topological_inventory/providers/common/operations/sources_api_client.rb
@@ -257,6 +264,7 @@ files:
 - spec/spec_helper.rb
 - spec/support/inventory_helper.rb
 - spec/support/shared/availability_check.rb
+- spec/topological_inventory/providers/common/collector_spec.rb
 - spec/topological_inventory/providers/common/collectors/inventory_collection_storage_spec.rb
 - spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb
 - spec/topological_inventory/providers/common/collectors_pool_spec.rb