topological_inventory-providers-common 1.0.2 → 1.0.7

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "1.0.2"
+      VERSION = "1.0.7"
     end
   end
 end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+if ENV['CI']
+  require 'simplecov'
+  SimpleCov.start
+end
+
+require "bundler/setup"
+require "topological_inventory/providers/common"
+require "webmock/rspec"
+
+Dir["./spec/support/**/*.rb"].each {|f| require f}
+
+spec_path = File.dirname(__FILE__)
+Dir[File.join(spec_path, "support/**/*.rb")].each { |f| require f }
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/spec/support/inventory_helper.rb

@@ -0,0 +1,14 @@
+module InventorySpecHelper
+  def self.big_inventory(size, chunk_size)
+    {
+      :collections => [
+        :name => SecureRandom.uuid,
+        :data => data_chunks(size, chunk_size)
+      ]
+    }
+  end
+
+  def self.data_chunks(size, chunk)
+    Array.new(size / chunk) { "a" * chunk }
+  end
+end

data/spec/support/shared/availability_check.rb

@@ -0,0 +1,236 @@
+require "topological_inventory/providers/common/operations/source"
+
+RSpec.shared_examples "availability_check" do
+  let(:host_url) { 'https://cloud.redhat.com' }
+  let(:sources_api_path) { '/api/sources/v3.0' }
+  let(:sources_internal_api_path) { '/internal/v1.0' }
+  let(:sources_api_url) { "#{host_url}#{sources_api_path}" }
+
+  let(:external_tenant) { '11001' }
+  let(:identity) { {'x-rh-identity' => Base64.strict_encode64({'identity' => {'account_number' => external_tenant, 'user' => {'is_org_admin' => true}}}.to_json)} }
+  let(:headers) { {'Content-Type' => 'application/json'}.merge(identity) }
+  let(:source_id) { '123' }
+  let(:endpoint_id) { '234' }
+  let(:application_id) { '345' }
+  let(:authentication_id) { '345' }
+  let(:payload) do
+    {
+      'params' => {
+        'source_id'       => source_id,
+        'external_tenant' => external_tenant,
+        'timestamp'       => Time.now.utc
+      }
+    }
+  end
+
+  let(:list_endpoints_response) { "{\"data\":[{\"default\":true,\"host\":\"10.0.0.1\",\"id\":\"#{endpoint_id}\",\"path\":\"/\",\"role\":\"ansible\",\"scheme\":\"https\",\"source_id\":\"#{source_id}\",\"tenant\":\"#{external_tenant}\"}]}" }
+  let(:list_endpoint_authentications_response) { "{\"data\":[{\"authtype\":\"username_password\",\"id\":\"#{authentication_id}\",\"resource_id\":\"#{endpoint_id}\",\"resource_type\":\"Endpoint\",\"username\":\"admin\",\"tenant\":\"#{external_tenant}\"}]}" }
+  let(:list_endpoint_authentications_response_empty) { "{\"data\":[]}" }
+  let(:internal_api_authentication_response) { "{\"authtype\":\"username_password\",\"id\":\"#{authentication_id}\",\"resource_id\":\"#{endpoint_id}\",\"resource_type\":\"Endpoint\",\"username\":\"admin\",\"tenant\":\"#{external_tenant}\",\"password\":\"xxx\"}" }
+  let(:list_applications_response) { {:data => [{:id => "345", :availability_status => "available"}]}.to_json }
+  let(:list_applications_unavailable_response) { {:data => [{:id => "345", :availability_status => "unavailable"}]}.to_json }
+
+  subject { described_class.new(payload["params"]) }
+
+  context "when not checked recently" do
+    before do
+      allow(subject).to receive(:checked_recently?).and_return(false)
+    end
+
+    it "updates Source and Endpoint when available" do
+      # GET
+      stub_get(:endpoint, list_endpoints_response)
+      stub_get(:authentication, list_endpoint_authentications_response)
+      stub_get(:password, internal_api_authentication_response)
+      stub_get(:application, "[]")
+
+      # PATCH
+      source_patch_body   = {'availability_status' => described_class::STATUS_AVAILABLE, 'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
+      endpoint_patch_body = {'availability_status' => described_class::STATUS_AVAILABLE, 'availability_status_error' => '', 'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
+
+      stub_patch(:source, source_patch_body)
+      stub_patch(:endpoint, endpoint_patch_body)
+
+      # Check ---
+      expect(subject).to receive(:connection_check).and_return([described_class::STATUS_AVAILABLE, nil])
+
+      subject.availability_check
+
+      assert_patch(:source, source_patch_body)
+      assert_patch(:endpoint, endpoint_patch_body)
+    end
+
+    it "updates Source and Endpoint when unavailable" do
+      # GET
+      stub_get(:endpoint, list_endpoints_response)
+      stub_get(:authentication, list_endpoint_authentications_response)
+      stub_get(:password, internal_api_authentication_response)
+      stub_get(:application, "[]")
+
+      # PATCH
+      connection_error_message = "Some connection error"
+      source_patch_body        = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
+      endpoint_patch_body      = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'availability_status_error' => connection_error_message, 'last_checked_at' => subject.send(:check_time)}.to_json
+
+      stub_patch(:source, source_patch_body)
+      stub_patch(:endpoint, endpoint_patch_body)
+
+      # Check ---
+      expect(subject).to receive(:connection_check).and_return([described_class::STATUS_UNAVAILABLE, connection_error_message])
+
+      subject.availability_check
+
+      assert_patch(:source, source_patch_body)
+      assert_patch(:endpoint, endpoint_patch_body)
+    end
+
+    it "updates only Source to 'unavailable' status if Endpoint not found" do
+      # GET
+      stub_get(:endpoint, '')
+      stub_get(:application, "[]")
+
+      # PATCH
+      source_patch_body = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
+      stub_patch(:source, source_patch_body)
+
+      # Check
+      api_client = subject.send(:api_client)
+      expect(api_client).not_to receive(:update_endpoint)
+
+      subject.availability_check
+
+      assert_patch(:source, source_patch_body)
+    end
+
+    it "updates Source and Endpoint to 'unavailable' if Authentication not found" do
+      # GET
+      stub_get(:endpoint, list_endpoints_response)
+      stub_get(:authentication, list_endpoint_authentications_response_empty)
+      stub_get(:application, "[]")
+
+      # PATCH
+      source_patch_body   = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
+      endpoint_patch_body = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'availability_status_error' => described_class::ERROR_MESSAGES[:authentication_not_found], 'last_checked_at' => subject.send(:check_time)}.to_json
+
+      stub_patch(:source, source_patch_body)
+      stub_patch(:endpoint, endpoint_patch_body)
+
+      # Check
+      expect(subject).not_to receive(:connection_check)
+      subject.availability_check
+
+      assert_patch(:source, source_patch_body)
+      assert_patch(:endpoint, endpoint_patch_body)
+    end
+  end
+
+  context "when checked recently" do
+    before do
+      allow(subject).to receive(:checked_recently?).and_return(true)
+    end
+
+    it "doesn't do connection check" do
+      expect(subject).not_to receive(:connection_check)
+      expect(WebMock).not_to have_requested(:patch, "#{sources_api_url}/sources/#{source_id}")
+      expect(WebMock).not_to have_requested(:patch, "#{sources_api_url}/endpoints/#{endpoint_id}")
+
+      subject.availability_check
+    end
+  end
+
+  context "when there is an application" do
+    context "when it is available" do
+      it "updates the availability status to available" do
+        # GET
+        stub_get(:endpoint, "[]")
+        stub_get(:application, list_applications_response)
+        # PATCH
+        application_patch_body = {'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
+        source_patch_body = {'availability_status' => described_class::STATUS_AVAILABLE, 'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
+
+        stub_patch(:source, source_patch_body)
+        stub_patch(:application, application_patch_body)
+
+        # Check
+        expect(subject).not_to receive(:connection_check)
+        subject.availability_check
+
+        assert_patch(:source, source_patch_body)
+        assert_patch(:application, application_patch_body)
+      end
+    end
+
+    context "when it is unavailable" do
+      it "updates the availability status to unavailable" do
+        # GET
+        stub_get(:endpoint, "[]")
+        stub_get(:application, list_applications_unavailable_response)
+        # PATCH
+        application_patch_body = {'last_checked_at' => subject.send(:check_time)}.to_json
+        source_patch_body = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
+
+        stub_patch(:source, source_patch_body)
+        stub_patch(:application, application_patch_body)
+
+        # Check
+        expect(subject).not_to receive(:connection_check)
+        subject.availability_check
+
+        assert_patch(:source, source_patch_body)
+        assert_patch(:application, application_patch_body)
+      end
+    end
+  end
+
+  def stub_get(object_type, response)
+    case object_type
+    when :endpoint
+      stub_request(:get, "#{sources_api_url}/sources/#{source_id}/endpoints")
+        .with(:headers => headers)
+        .to_return(:status => 200, :body => response, :headers => {})
+    when :authentication
+      stub_request(:get, "#{sources_api_url}/endpoints/#{endpoint_id}/authentications")
+        .with(:headers => headers)
+        .to_return(:status => 200, :body => response, :headers => {})
+    when :password
+      stub_request(:get, "#{host_url}#{sources_internal_api_path}/authentications/#{authentication_id}?expose_encrypted_attribute%5B%5D=password")
+        .with(:headers => headers)
+        .to_return(:status => 200, :body => response, :headers => {})
+    when :application
+      stub_request(:get, "#{sources_api_url}/sources/#{source_id}/applications")
+        .with(:headers => headers)
+        .to_return(:status => 200, :body => response, :headers => {})
+    end
+  end
+
+  def stub_patch(object_type, data)
+    case object_type
+    when :source
+      stub_request(:patch, "#{sources_api_url}/sources/#{source_id}")
+        .with(:body => data, :headers => headers)
+        .to_return(:status => 200, :body => "", :headers => {})
+    when :endpoint
+      stub_request(:patch, "#{sources_api_url}/endpoints/#{endpoint_id}")
+        .with(:body => data, :headers => headers)
+        .to_return(:status => 200, :body => "", :headers => {})
+    when :application
+      stub_request(:patch, "#{sources_api_url}/applications/#{application_id}")
+        .with(:body => data, :headers => headers)
+        .to_return(:status => 200, :body => "", :headers => {})
+    end
+  end
+
+  def assert_patch(object_type, data)
+    case object_type
+    when :source
+      expect(WebMock).to have_requested(:patch, "#{sources_api_url}/sources/#{source_id}")
+        .with(:body => data, :headers => headers).once
+    when :endpoint
+      expect(WebMock).to have_requested(:patch, "#{sources_api_url}/endpoints/#{endpoint_id}")
+        .with(:body => data, :headers => headers).once
+    when :application
+      expect(WebMock).to have_requested(:patch, "#{sources_api_url}/applications/#{application_id}")
+        .with(:body => data, :headers => headers).once
+    end
+  end
+end

data/spec/topological_inventory/providers/common/collector_spec.rb

@@ -0,0 +1,171 @@
+RSpec.describe TopologicalInventory::Providers::Common::Collector do
+  let(:collector) do
+    collector = described_class.new(source)
+
+    allow(collector).to receive(:ingress_api_client).and_return(client)
+    allow(collector).to receive(:logger).and_return(logger)
+    allow(logger).to receive(:error)
+
+    collector
+  end
+
+  let(:parser) { TopologicalInventory::Providers::Common::Collector::Parser.new }
+
+  let(:source)  { "source_uid" }
+  let(:client)  { double }
+  let(:logger)  { double }
+  let(:refresh_state_uuid) { SecureRandom.uuid }
+  let(:refresh_state_part_uuid) { SecureRandom.uuid }
+  # based on the default, we can tell how many chunks the saver will break the payload up into
+  let(:max_size) { TopologicalInventory::Providers::Common::SaveInventory::Saver::KAFKA_PAYLOAD_MAX_BYTES_DEFAULT }
+  let(:multiplier) { 0.75 }
+
+  context "#save_inventory" do
+    it "does nothing with empty collections" do
+      parts = collector.send(:save_inventory, [], collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+
+      expect(parts).to eq 0
+    end
+
+    it "saves 1 part if it fits" do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(0)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 1
+    end
+
+    it "saves 2 parts if over limit with 1 collection" do
+      (multiplier * 2000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 2
+    end
+
+    it "saves 2 parts if over limit with 2 collections" do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(2).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 2
+    end
+
+    it "saves many parts" do
+      (multiplier * 1500).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      (multiplier * 2000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(client).to receive(:save_inventory_with_http_info).exactly(4).times
+      parts = collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid)
+      expect(parts).to eq 4
+    end
+
+    it 'raises exception when entity to save is too big' do
+      parser.collections.container_groups.build(:source_ref => "a" * (1_000_000 * multiplier))
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+      # in this case, we first save empty inventory, then the size check fails saving the rest of data
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it 'raises exception when entity of second collection is too big' do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(1)
+      # We save the first collection then it fails on saving the second collection
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it 'raises exception when entity of second collection is too big then continues with smaller' do
+      (multiplier * 1000).floor.times { parser.collections.container_groups.build(:source_ref => "a" * 950) }
+      parser.collections.container_nodes.build(:source_ref => "a" * (1_000_000 * multiplier))
+      (multiplier * 1000).floor.times { parser.collections.container_nodes.build(:source_ref => "a" * 950) }
+
+      expect(inventory_size(parser.collections.values) / max_size).to eq(2)
+      # We save the first collection then it fails on saving the second collection
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      expect { collector.send(:save_inventory, parser.collections.values, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, refresh_state_part_uuid) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+  end
+
+  context "#sweep_inventory" do
+    it "with nil total parts" do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, nil, [])
+    end
+
+    it "with empty scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [])
+    end
+
+    it "with normal scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, [:container_groups])
+    end
+
+    it "with normal targeted scope " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(1).times
+
+      collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, {:container_groups => [{:source_ref => "a"}]})
+    end
+
+    it "fails with scope entity too large " do
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      sweep_scope = {:container_groups => [{:source_ref => "a" * (1_000_002 * multiplier)}]}
+
+      expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+
+    it "fails when scope is too big " do
+      # We should have also sweep scope chunking, that is if we'll do big targeted refresh and sweeping
+      expect(client).to receive(:save_inventory_with_http_info).exactly(0).times
+
+      sweep_scope = {:container_groups => (0..1001 * multiplier).map { {:source_ref => "a" * 1_000} } }
+
+      expect { collector.send(:sweep_inventory, collector.send(:inventory_name), collector.send(:schema_name), refresh_state_uuid, 1, sweep_scope) }.to(
+        raise_error(TopologicalInventory::Providers::Common::SaveInventory::Exception::EntityTooLarge)
+      )
+    end
+  end
+
+  def build_inventory(collections)
+    TopologicalInventoryIngressApiClient::Inventory.new(
+      :name                    => collector.send(:inventory_name),
+      :schema                  => TopologicalInventoryIngressApiClient::Schema.new(:name => collector.send(:schema_name)),
+      :source                  => source,
+      :collections             => collections,
+      :refresh_state_uuid      => refresh_state_uuid,
+      :refresh_state_part_uuid => refresh_state_part_uuid,
+    )
+  end
+
+  def inventory_size(collections)
+    JSON.generate(build_inventory(collections).to_hash).size
+  end
+end