topological_inventory-providers-common 1.0.2 → 1.0.7

data/spec/topological_inventory/providers/common/collectors/inventory_collection_storage_spec.rb

@@ -0,0 +1,44 @@
+describe TopologicalInventory::Providers::Common::Collector::InventoryCollectionStorage do
+  before do
+    @storage = described_class.new
+  end
+
+  it "should add collection to data" do
+    @storage.add_collection(:vms)
+
+    expect(@storage.data[:vms]).not_to be_nil
+  end
+
+  it "should access the same collection through brackets, method name and data" do
+    @storage.add_collection(:vms)
+
+    vm_name = "My VM"
+
+    @storage[:vms].build(:name => vm_name)
+
+    expect(@storage.vms.data[0].name).to eq(vm_name)
+    expect(@storage[:vms].data[0].name).to eq(vm_name)
+    expect(@storage.data[:vms].data[0].name).to eq(vm_name)
+  end
+
+  it "should create collection automatically when api object exists" do
+    expect("TopologicalInventoryIngressApiClient::Vm".safe_constantize).not_to be_nil
+
+    storage = described_class.new
+    expect(storage.vms).to be_kind_of(TopologicalInventory::Providers::Common::Collector::InventoryCollectionWrapper)
+
+    storage = described_class.new
+    expect(storage[:vms]).to be_kind_of(TopologicalInventory::Providers::Common::Collector::InventoryCollectionWrapper)
+  end
+
+  it "should raise NameError when api object doesn't exist" do
+    expect("TopologicalInventoryIngressApiClient::SomethingNonexisting".safe_constantize).to be_nil
+
+    storage = described_class.new
+
+    expect { storage.add_collection(:something_nonexisting) }.to raise_error(NameError)
+
+    expect { storage.something_nonexisting.build(:name => "Vm") }.to raise_error(NameError)
+  end
+
+end

data/spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb

@@ -0,0 +1,9 @@
+describe TopologicalInventory::Providers::Common::Collector::InventoryCollectionWrapper do
+  it "builds only existing ingress api client's object" do
+    ic = described_class.new(:name => :some_undefined_class_in_api_models)
+    ic_existing = described_class.new(:name => :vm)
+
+    expect { ic.build({}) }.to raise_error(NameError)
+    expect(ic_existing.build({})).to be_kind_of(TopologicalInventoryIngressApiClient::Vm)
+  end
+end

data/spec/topological_inventory/providers/common/collectors_pool_spec.rb

@@ -0,0 +1,150 @@
+require "tempfile"
+require "yaml"
+require "json"
+
+RSpec.describe TopologicalInventory::Providers::Common::CollectorsPool do
+  let(:source1) { {:source => '42b1893c-ebbd-44e9-89b1-5c29b5fe6e10', :schema => 'http', :host => 'cloud.redhat.com', :port => 80} }
+  let(:source2) { {:source => 'fe8bcaea-3670-42c7-bed9-71f6e0bceadd', :schema => 'https', :host => 'cloud.redhat.com', :port => 443} }
+  let(:source3) { {:source => '05838743-4285-404a-b4d6-294045c0d4be', :schema => 'xxx', :host => 'cloud.redhat.com', :port => 1234} }
+  let(:source4) { {:source => '5ed08a3c-3de4-4a90-8ce9-e0f724b2b2e6', :schema => 'xxx', :host => 'cloud.redhat.com', :port => 1234} }
+  let(:sources) { [source1, source2, source3] }
+
+  before do
+    clear_settings
+  end
+
+  subject { described_class.new(nil, nil, :thread_pool_size => 2) }
+
+  context "config reload" do
+    it "changes settings with different configs" do
+      settings = [{:sources => sources},
+                  {:sources => [ source2, source4 ]}]
+
+      2.times do |i|
+        config = Tempfile.new(["config#{i}", '.yml'])
+        begin
+          config.write(settings[i].to_yaml)
+          config.rewind
+
+          name, path = path_and_filename(config)
+          subject.send(:config_name=, name.split('.')[0])
+          allow(subject).to receive(:path_to_config).and_return(path)
+
+          subject.send(:reload_config)
+
+          expect(::Settings.sources.to_a.collect(&:to_hash)).to eq(settings[i][:sources])
+        ensure
+          config.close
+          config.unlink
+        end
+      end
+    end
+  end
+
+  context "secret reload" do
+    it "changes credentials with new secret" do
+      uuid = SecureRandom.uuid
+
+      secrets = [
+        {'updated_at' => Time.now.to_s, uuid => {'username' => 'admin1', 'password' => 'password1'}},
+        {'updated_at' => Time.now.to_s, uuid => {'username' => 'admin2', 'password' => 'password2'}},
+      ]
+
+      2.times do |i|
+        secret = Tempfile.new(["credentials#{i}"])
+        begin
+          secret.write(secrets[i].to_json)
+          secret.rewind
+
+          name, path = path_and_filename(secret)
+
+          allow(subject).to receive(:path_to_secrets).and_return(path)
+          stub_const("#{described_class}::SECRET_FILENAME", name)
+
+          subject.send(:reload_secrets)
+
+          expect(subject.send(:secrets)).to eq(secrets[i])
+        end
+      end
+    end
+  end
+
+  context "add or remove collector" do
+    before do
+      ::Config.load_and_set_settings('some-value-needed.txt')
+      @collector = double("collector")
+      allow(subject).to receive(:new_collector).and_return(@collector)
+    end
+
+    context "without secrets check" do
+      before do
+        allow(subject).to receive(:secrets_for_source).and_return({})
+      end
+
+      it "adds new collectors from settings" do
+        allow(@collector).to receive(:collect!).and_return(nil)
+        expect(@collector).to receive(:collect!).exactly(sources.size).times
+
+        sources.each do |source|
+          stub_settings_merge(:sources => ::Settings.sources.to_a + [source])
+
+          subject.send(:queue_collectors)
+        end
+
+        pool = subject.send(:thread_pool)
+        pool.shutdown
+        pool.wait_for_termination
+      end
+    end
+
+    context "with secrets check" do
+      let(:secrets) do
+        { 'updated_at' => Time.now.to_s,
+          source1[:source] => { 'username' => 'admin1', 'password' => 'password1' },
+          source2[:source] => { 'username' => 'admin2', 'password' => 'password2' },
+          'unknown' => { 'username' => 'admin3', 'password' => 'password3' }
+        }
+      end
+
+      before do
+        allow(@collector).to receive(:collect!).and_return(nil)
+      end
+
+      it "creates only collectors found in both secret and config" do
+        # 4 sources in yaml config
+        stub_settings_merge(:sources => sources + [source4])
+        # 3 sources in secret
+        allow(subject).to receive(:secrets).and_return(secrets)
+
+        # for each source in yaml secret is searched (4x)
+        expect(subject).to receive(:secrets_for_source).and_call_original.exactly(4).times
+        # only 2 corresponding
+        expect(@collector).to receive(:collect!).exactly(2).times
+
+        subject.send(:queue_collectors)
+
+        pool = subject.send(:thread_pool)
+        pool.shutdown
+        pool.wait_for_termination
+      end
+    end
+  end
+
+  def stub_settings_merge(hash)
+    if defined?(::Settings)
+      Settings.add_source!(hash)
+      Settings.reload!
+    end
+  end
+
+  def clear_settings
+    ::Settings.keys.dup.each { |k| ::Settings.delete_field(k) } if defined?(::Settings)
+  end
+
+  def path_and_filename(tempfile)
+    parts = tempfile.path.split('/')
+    name = parts[-1]
+    path = parts[0..-2].join('/')
+    [name, path]
+  end
+end

data/spec/topological_inventory/providers/common/logger_spec.rb

@@ -0,0 +1,38 @@
+RSpec.describe TopologicalInventory::Providers::Common::Logger do
+  let(:status) { :test }
+  let(:source) { '92844e11-17d5-4998-a33d-d886c3c7a80e' }
+  let(:entity_type) { 'test-entity' }
+  let(:refresh_state_uuid) { 'cd22ba1c-56f6-4fd4-a191-ec8eb8e993a8' }
+  let(:sweep_scope) { [entity_type] }
+  let(:total_parts) { 10 }
+
+  subject { described_class.new }
+
+  it 'receives collecting method' do
+    msg = "[#{status.to_s.upcase}] Collecting #{entity_type}"
+    msg += ", :total parts => #{total_parts}" if total_parts.present?
+    msg += ", :source_uid => #{source}, :refresh_state_uuid => #{refresh_state_uuid}"
+    expect(subject).to receive(:info).with(msg)
+
+    subject.collecting(status, source, entity_type, refresh_state_uuid, total_parts)
+  end
+
+  it 'receives sweeping method' do
+    msg = "[#{status.to_s.upcase}] Sweeping inactive records, :sweep_scope => #{sweep_scope}, :source_uid => #{source}, :refresh_state_uuid => #{refresh_state_uuid}"
+    expect(subject).to receive(:info).with(msg)
+
+    subject.sweeping(status, source, sweep_scope, refresh_state_uuid)
+  end
+
+  it 'receives collecting error method' do
+    begin
+      raise 'Test exception'
+    rescue => e
+      msg = "[ERROR] Collecting #{entity_type}, :source_uid => #{source}, :refresh_state_uuid => #{refresh_state_uuid}"
+      msg += ":message => #{e.message}\n#{e.backtrace.join("\n")}"
+      expect(subject).to receive(:error).with(msg)
+
+      subject.collecting_error(source, entity_type, refresh_state_uuid, e)
+    end
+  end
+end

data/spec/topological_inventory/providers/common/operations/processor_spec.rb

@@ -0,0 +1,102 @@
+require "topological_inventory/providers/common/operations/processor"
+
+RSpec.describe TopologicalInventory::Providers::Common::Operations::Processor do
+  let(:topology_api_client) { double }
+  let(:source_id) { 1 }
+  let(:source_ref) { 1000 }
+  let(:service_plan) { double("TopologicalInventoryApiClient::ServicePlan") }
+  let(:service_offering) { double("TopologicalInventoryApiClient::ServiceOffering") }
+
+  # Overriden in contexts
+  let(:payload) { {} }
+
+  before do
+    @processor = described_class.new(nil, nil, payload)
+    allow(@processor).to receive(:logger).and_return(double('null_object').as_null_object)
+
+    allow(service_plan).to receive(:service_offering_id).and_return(1)
+    allow(service_plan).to receive(:name).and_return(double)
+
+    allow(service_offering).to receive(:name).and_return(double)
+    allow(service_offering).to receive(:source_ref).and_return(source_ref)
+    allow(service_offering).to receive(:extra).and_return({:type => 'job_template'})
+    allow(service_offering).to receive(:source_id).and_return(source_id)
+
+    @endpoint_client = double
+    allow(@endpoint_client).to receive(:order_service)
+
+    allow(@processor).to receive(:endpoint_client).and_return(@endpoint_client)
+    allow(@processor).to receive(:topology_api_client).and_return(topology_api_client)
+    allow(topology_api_client).to receive(:update_task)
+    allow(topology_api_client).to receive(:show_service_plan).and_return(service_plan)
+    allow(topology_api_client).to receive(:show_service_offering).and_return(service_offering)
+  end
+
+  context "Order by ServicePlan" do
+    let(:payload) do
+      {
+        'request_context' => {"x-rh-identity" => 'abcd'},
+        'params'          => {
+          'order_params'    => {
+            'service_plan_id'             => 1,
+            'service_parameters'          => { :name   => "Job 1",
+                                               :param1 => "Test Topology",
+                                               :param2 => 50 },
+            'provider_control_parameters' => {}
+          },
+          'service_plan_id' => 1,
+          'task_id'         => 1 # in tp-inv api (Task)
+        }
+      }
+    end
+
+    describe "#order_service" do
+      it "orders job" do
+        allow(@processor).to receive(:poll_order_complete_thread).and_return(double)
+
+        expect(@endpoint_client).to receive(:order_service).with(service_offering, service_plan, payload['params']['order_params'])
+        @processor.send(:order_service, payload['params'])
+      end
+
+      it "updates task on error" do
+        err_message = "Sample error"
+
+        allow(@processor).to receive(:poll_order_complete_thread).and_return(double)
+        allow(@processor).to receive(:update_task).and_return(double)
+        allow(@endpoint_client).to receive(:order_service).and_raise(err_message)
+
+        expect(@processor).to receive(:update_task).with(payload['params']['task_id'], :state => "completed", :status => "error", :context => { :error => err_message })
+
+        @processor.send(:order_service, payload['params'])
+      end
+    end
+  end
+
+  context "Order by ServiceOffering" do
+    let(:payload) do
+      {
+        'request_context' => {"x-rh-identity" => 'abcd'},
+        'params'          => {
+          'order_params'    => {
+            'service_offering_id'             => 1,
+            'service_parameters'          => { :name   => "Job 1",
+                                               :param1 => "Test Topology",
+                                               :param2 => 50 },
+            'provider_control_parameters' => {}
+          },
+          'service_offering_id' => 1,
+          'task_id'         => 1 # in tp-inv api (Task)
+        }
+      }
+    end
+
+    describe "#order_service" do
+      it "orders job" do
+        allow(@processor).to receive(:poll_order_complete_thread).and_return(double)
+
+        expect(@endpoint_client).to receive(:order_service).with(service_offering, nil, payload['params']['order_params'])
+        @processor.send(:order_service, payload['params'])
+      end
+    end
+  end
+end

data/spec/topological_inventory/providers/common/operations/source_spec.rb

@@ -0,0 +1,5 @@
+require "topological_inventory/providers/common/operations/source"
+
+RSpec.describe TopologicalInventory::Providers::Common::Operations::Source do
+  it_behaves_like "availability_check"
+end

data/spec/topological_inventory/providers/common/save_inventory/saver_spec.rb

@@ -0,0 +1,65 @@
+require "topological_inventory/providers/common/save_inventory/saver"
+
+RSpec.describe TopologicalInventory::Providers::Common::SaveInventory::Saver do
+  let(:client) { instance_double(TopologicalInventoryIngressApiClient::DefaultApi) }
+  let(:logger) { double }
+  let(:base_args) { {client: client, logger: logger} }
+
+  let(:small_json) { {:test => ["values"]} }
+  let(:big_json) { InventorySpecHelper.big_inventory(80_000, 1_000) }
+
+  describe "#save" do
+    subject { described_class.new(args).save(:inventory => inventory) }
+
+    context "when the data size is less than max_bytes" do
+      let(:args) { base_args }
+      let(:inventory) { small_json }
+
+      before do
+        allow(client).to receive(:save_inventory_with_http_info).with(small_json.to_json)
+      end
+
+      it "returns that it saved one chunk" do
+        is_expected.to eq 1
+      end
+
+      it "does not split the payload into batches" do
+        expect(client).to receive(:save_inventory_with_http_info).with(small_json.to_json).once
+        subject
+      end
+    end
+
+    context "when the data size is greater than specified max_bytes" do
+      let(:args) { base_args.merge!(:max_bytes => 19_512) }
+      let(:inventory) { big_json }
+
+      before do
+        allow(client).to receive(:save_inventory_with_http_info)
+      end
+
+      it "returns that it saved five chunks" do
+        is_expected.to eq 5
+      end
+
+      it "splits the payload up into chunks" do
+        expect(client).to receive(:save_inventory_with_http_info).exactly(5).times
+        subject
+      end
+    end
+
+    context "when the KAFKA_PAYLOAD_MAX_BYTES ENV var is set" do
+      let(:args) { base_args }
+      let(:inventory) { big_json }
+
+      before do
+        allow(ENV).to receive(:[]).with("KAFKA_PAYLOAD_MAX_BYTES").and_return("9_512")
+        allow(client).to receive(:save_inventory_with_http_info)
+      end
+
+      it "splits the payload into smaller chunks" do
+        expect(client).to receive(:save_inventory_with_http_info).exactly(10).times
+        is_expected.to eq 10
+      end
+    end
+  end
+end

data/spec/topological_inventory/providers/common_spec.rb

@@ -0,0 +1,3 @@
+RSpec.describe TopologicalInventory::Providers::Common do
+
+end

data/topological_inventory-providers-common.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0")
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -27,9 +27,15 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'config', '~> 1.7', '>= 1.7.2'
   spec.add_runtime_dependency 'json', '~> 2.3'
   spec.add_runtime_dependency "manageiq-loggers", ">= 0.4.2"
+  spec.add_runtime_dependency "sources-api-client", "~> 3.0"
   spec.add_runtime_dependency "topological_inventory-api-client", "~> 3.0", ">= 3.0.1"
+  spec.add_runtime_dependency "topological_inventory-ingress_api-client", "~> 1.0"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'rubocop', '~>0.69.0'
+  spec.add_development_dependency 'rubocop-performance', '~>1.3'
+  spec.add_development_dependency "simplecov", "~> 0.17.1"
+  spec.add_development_dependency 'webmock'
 end