topological_inventory-providers-common 1.0.7 → 1.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 868c8f764cd11d9888ed99698a309cc199fdf92823963f4e5cae71208d506ff7
-  data.tar.gz: 39c352e8b1306bdb87675bf0ccc1fe08f02b0c13555b2466c4c38d7a76979aca
+  metadata.gz: b81c523169c3ca87e678e3e7d046b4daee863200cf4eadd695e9022d17a11e5a
+  data.tar.gz: 1fc2a47cedf6f8d3b41f0df728a5ac8ceceaea03b58d43ef67da90680861c0eb
 SHA512:
-  metadata.gz: c2a8af390114548a89c7dc5bb43f1a6e3784a5a59ebd2ce1c7451c15726015df1353e3dcbc3755d7637d696707b0d6db93e5d4907920a1c4b09d3f6410130a77
-  data.tar.gz: 46e6596954367336de4049f8024da8b5e63e5be4c60636c77f91f22c3e407229b38aca604137907d1f5cd6411201d9c3b57e4ad5bab2cfb749b2c04e9ca8eeab
+  metadata.gz: 9afb8ba326d632ba59245d93837f32bd8edbcb34b3d82e029c351f904bb75f196592eb4724116a48caf0b7df8fc7f4101f5b59bd0e23b65d6fca25ada3ac4cb6
+  data.tar.gz: '00924d04d1fc4623cd4e0f0a719c5471dbd39635820375b4957278db1c6e3ecbdf2b58c25758bca22e91e979511286708279922919f6303118401e58ac9cc5c4'

data/CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.12] - 2020-10-01
+Add Operations Async Worker class #55
+
+## [1.0.11]
+Make Collector Poll Time a parameter so we can tweak the collection interval #51
+
+## [1.0.10]
+Add HealthCheck class for operations workers #48
+Set the LOG_LEVEL if present #50
+
+## [1.0.9]
+Added refresh-type to save and sweep inventory #45
+
+## [1.0.8] - 2020-08-12
+Add => to error messages that rubocop missed #44
+
 ## [1.0.7] - 2020-07-27
 Update operations/source model for receptor-enabled availability checks #36
 Add check for Application subresource under a Source during Availability check #40
@@ -43,8 +59,13 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.7...HEAD
-[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...v1.0.7
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...HEAD
+[1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12
+[1.0.11]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.10...v1.0.11
+[1.0.10]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.9...v1.0.10
+[1.0.9]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.8...v1.0.9
+[1.0.8]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.7...v1.0.8
+[1.0.7]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...v1.0.7
 [1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
 [1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
 [1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4

data/lib/topological_inventory/providers/common.rb

@@ -2,6 +2,7 @@ require "topological_inventory/providers/common/version"
 require "topological_inventory/providers/common/logging"
 require "topological_inventory/providers/common/operations/processor"
 require "topological_inventory/providers/common/operations/endpoint_client"
+require "topological_inventory/providers/common/operations/health_check"
 require "topological_inventory/providers/common/collectors_pool"
 require "topological_inventory/providers/common/collector"
 

data/lib/topological_inventory/providers/common/collector.rb

@@ -104,7 +104,8 @@ module TopologicalInventory
                            refresh_state_uuid = nil,
                            refresh_state_part_uuid = nil,
                            refresh_state_part_collected_at = nil,
-                           refresh_state_part_sent_at = Time.now.utc)
+                           refresh_state_part_sent_at = Time.now.utc,
+                           refresh_type = default_refresh_type)
           return 0 if collections.empty?
 
           SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
@@ -116,7 +117,8 @@ module TopologicalInventory
               :refresh_state_uuid              => refresh_state_uuid,
               :refresh_state_part_uuid         => refresh_state_part_uuid,
               :refresh_state_part_collected_at => refresh_state_part_collected_at,
-              :refresh_state_part_sent_at      => refresh_state_part_sent_at
+              :refresh_state_part_sent_at      => refresh_state_part_sent_at,
+              :refresh_type                    => refresh_type
             )
           )
         rescue => e
@@ -134,7 +136,8 @@ module TopologicalInventory
                             total_parts,
                             sweep_scope,
                             refresh_state_started_at = nil,
-                            refresh_state_sent_at = Time.now.utc)
+                            refresh_state_sent_at = Time.now.utc,
+                            refresh_type = default_refresh_type)
           return if !total_parts || sweep_scope.empty?
 
           SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
@@ -147,7 +150,8 @@ module TopologicalInventory
               :total_parts              => total_parts,
               :sweep_scope              => sweep_scope,
               :refresh_state_started_at => refresh_state_started_at,
-              :refresh_state_sent_at    => refresh_state_sent_at
+              :refresh_state_sent_at    => refresh_state_sent_at,
+              :refresh_type             => refresh_type
             )
           )
         rescue => e
@@ -165,6 +169,10 @@ module TopologicalInventory
           "Default"
         end
 
+        def default_refresh_type
+          'full-refresh'
+        end
+
         def ingress_api_client
           TopologicalInventoryIngressApiClient::DefaultApi.new
         end

data/lib/topological_inventory/providers/common/collectors_pool.rb

@@ -5,8 +5,9 @@ module TopologicalInventory
     module Common
       class CollectorsPool
         SECRET_FILENAME = "credentials".freeze
+        COLLECTOR_POLL_TIME = ENV['COLLECTOR_POLL_TIME']&.to_i || 300
 
-        def initialize(config_name, metrics, collector_poll_time: 60, thread_pool_size: 2)
+        def initialize(config_name, metrics, collector_poll_time: COLLECTOR_POLL_TIME, thread_pool_size: 2)
           self.config_name         = config_name
           self.collector_status    = Concurrent::Map.new
           self.metrics             = metrics

data/lib/topological_inventory/providers/common/logging.rb

@@ -33,7 +33,10 @@ module TopologicalInventory
 
       class Logger < ManageIQ::Loggers::CloudWatch
         def self.new(*args)
-          super.tap { |logger| logger.extend(TopologicalInventory::Providers::Common::LoggingFunctions) }
+          super.tap do |logger|
+            logger.extend(TopologicalInventory::Providers::Common::LoggingFunctions)
+            logger.level = ENV['LOG_LEVEL'] if ENV['LOG_LEVEL']
+          end
         end
       end
 

data/lib/topological_inventory/providers/common/operations/async_worker.rb

@@ -0,0 +1,56 @@
+require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/operations/health_check"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class AsyncWorker
+          include Logging
+
+          def initialize(processor, queue = nil)
+            @processor = processor
+            @queue = queue || Queue.new
+          end
+
+          def start
+            return if thread.present?
+
+            @thread = Thread.new { listen }
+          end
+
+          def stop
+            thread&.exit
+          end
+
+          def enqueue(msg)
+            queue << msg
+          end
+
+          def listen
+            loop do
+              # the queue thread waits for a message to come during `Queue#pop`
+              msg = queue.pop
+              process_message(msg)
+            end
+          end
+
+          private
+
+          attr_reader :thread, :queue, :processor
+
+          def process_message(msg)
+            processor.process!(msg)
+          rescue => err
+            model, method = msg.message.to_s.split(".")
+            logger.error("#{model}##{method}: async worker failure: #{err.cause}\n#{err}\n#{err.backtrace.join("\n")}")
+          ensure
+            msg.ack
+            TopologicalInventory::Providers::Common::Operations::HealthCheck.touch_file
+            logger.debug("Operations::AsyncWorker queue length: #{queue.length}") if queue.length >= 20 && queue.length % 5 == 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/health_check.rb

@@ -0,0 +1,15 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class HealthCheck
+          HEARTBEAT_FILE = '/tmp/healthy'.freeze
+
+          def self.touch_file
+            FileUtils.touch(HEARTBEAT_FILE)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -153,8 +153,9 @@ module TopologicalInventory
 
         def endpoint
           @endpoint ||= api_client.fetch_default_endpoint(source_id)
-        rescue e
+        rescue => e
           logger.availability_check("Failed to fetch Endpoint for Source #{source_id}: #{e.message}", :error)
+          nil
         end
 
         def authentication
@@ -163,14 +164,16 @@ module TopologicalInventory
                               else
                                 api_client.fetch_authentication(source_id, endpoint)
                               end
-        rescue e
+        rescue => e
           logger.availability_check("Failed to fetch Authentication for Source #{source_id}: #{e.message}", :error)
+          nil
         end
 
         def application
           @application ||= api_client.fetch_application(source_id)
-        rescue e
+        rescue => e
           logger.availability_check("Failed to fetch Application for Source #{source_id}: #{e.message}", :error)
+          nil
         end
 
         def check_time

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "1.0.7"
+      VERSION = "1.0.12"
     end
   end
 end

data/spec/support/shared/availability_check.rb

@@ -42,7 +42,7 @@ RSpec.shared_examples "availability_check" do
       stub_get(:endpoint, list_endpoints_response)
       stub_get(:authentication, list_endpoint_authentications_response)
       stub_get(:password, internal_api_authentication_response)
-      stub_get(:application, "[]")
+      stub_not_found(:application)
 
       # PATCH
       source_patch_body   = {'availability_status' => described_class::STATUS_AVAILABLE, 'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
@@ -65,7 +65,7 @@ RSpec.shared_examples "availability_check" do
       stub_get(:endpoint, list_endpoints_response)
       stub_get(:authentication, list_endpoint_authentications_response)
       stub_get(:password, internal_api_authentication_response)
-      stub_get(:application, "[]")
+      stub_not_found(:application)
 
       # PATCH
       connection_error_message = "Some connection error"
@@ -86,8 +86,8 @@ RSpec.shared_examples "availability_check" do
 
     it "updates only Source to 'unavailable' status if Endpoint not found" do
       # GET
-      stub_get(:endpoint, '')
-      stub_get(:application, "[]")
+      stub_not_found(:endpoint)
+      stub_not_found(:application)
 
       # PATCH
       source_patch_body = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
@@ -106,7 +106,7 @@ RSpec.shared_examples "availability_check" do
       # GET
       stub_get(:endpoint, list_endpoints_response)
       stub_get(:authentication, list_endpoint_authentications_response_empty)
-      stub_get(:application, "[]")
+      stub_not_found(:application)
 
       # PATCH
       source_patch_body   = {'availability_status' => described_class::STATUS_UNAVAILABLE, 'last_checked_at' => subject.send(:check_time)}.to_json
@@ -142,7 +142,7 @@ RSpec.shared_examples "availability_check" do
     context "when it is available" do
       it "updates the availability status to available" do
         # GET
-        stub_get(:endpoint, "[]")
+        stub_not_found(:endpoint)
         stub_get(:application, list_applications_response)
         # PATCH
         application_patch_body = {'last_available_at' => subject.send(:check_time), 'last_checked_at' => subject.send(:check_time)}.to_json
@@ -163,7 +163,7 @@ RSpec.shared_examples "availability_check" do
     context "when it is unavailable" do
       it "updates the availability status to unavailable" do
         # GET
-        stub_get(:endpoint, "[]")
+        stub_not_found(:endpoint)
         stub_get(:application, list_applications_unavailable_response)
         # PATCH
         application_patch_body = {'last_checked_at' => subject.send(:check_time)}.to_json
@@ -203,6 +203,27 @@ RSpec.shared_examples "availability_check" do
     end
   end
 
+  def stub_not_found(object_type)
+    case object_type
+    when :endpoint
+      stub_request(:get, "#{sources_api_url}/sources/#{source_id}/endpoints")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
+    when :authentication
+      stub_request(:get, "#{sources_api_url}/endpoints/#{endpoint_id}/authentications")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
+    when :password
+      stub_request(:get, "#{host_url}#{sources_internal_api_path}/authentications/#{authentication_id}?expose_encrypted_attribute%5B%5D=password")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
+    when :application
+      stub_request(:get, "#{sources_api_url}/sources/#{source_id}/applications")
+        .with(:headers => headers)
+        .to_return(:status => 404, :body => {}.to_json, :headers => {})
+    end
+  end
+
   def stub_patch(object_type, data)
     case object_type
     when :source

data/spec/topological_inventory/providers/common/operations/async_worker_spec.rb

@@ -0,0 +1,36 @@
+require "topological_inventory/providers/common/operations/async_worker"
+
+describe TopologicalInventory::Providers::Common::Operations::AsyncWorker do
+  let(:queue) { double }
+  let(:impl) { double }
+  let(:msg) { double }
+  subject { described_class.new(impl, queue) }
+
+  before do
+    allow(queue).to receive(:length).and_return(0)
+    allow(msg).to receive(:message).and_return("Source.availability_check")
+  end
+
+  context "when the message is able to be processed" do
+    before do
+      allow(impl).to receive(:process!).with(msg)
+      allow(msg).to receive(:ack)
+    end
+
+    it "drains messages that are added to the queue" do
+      expect(impl).to receive(:process!).with(msg).once
+      subject.send(:process_message, msg)
+    end
+  end
+
+  context "when the message results in an error" do
+    before do
+      allow(impl).to receive(:process!).with(msg).and_raise(StandardError.new("boom!"))
+    end
+
+    it "ack's the message on failure" do
+      expect(msg).to receive(:ack).once
+      subject.send(:process_message, msg)
+    end
+  end
+end

data/topological_inventory-providers-common.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "manageiq-loggers", ">= 0.4.2"
   spec.add_runtime_dependency "sources-api-client", "~> 3.0"
   spec.add_runtime_dependency "topological_inventory-api-client", "~> 3.0", ">= 3.0.1"
-  spec.add_runtime_dependency "topological_inventory-ingress_api-client", "~> 1.0"
+  spec.add_runtime_dependency "topological_inventory-ingress_api-client", "~> 1.0", ">= 1.0.3"
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", ">= 12.3.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-providers-common
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.12
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-28 00:00:00.000000000 Z
+date: 2020-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -113,6 +113,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -120,6 +123,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -246,7 +252,9 @@ files:
 - lib/topological_inventory/providers/common/collector/parser.rb
 - lib/topological_inventory/providers/common/collectors_pool.rb
 - lib/topological_inventory/providers/common/logging.rb
+- lib/topological_inventory/providers/common/operations/async_worker.rb
 - lib/topological_inventory/providers/common/operations/endpoint_client.rb
+- lib/topological_inventory/providers/common/operations/health_check.rb
 - lib/topological_inventory/providers/common/operations/processor.rb
 - lib/topological_inventory/providers/common/operations/source.rb
 - lib/topological_inventory/providers/common/operations/sources_api_client.rb
@@ -262,6 +270,7 @@ files:
 - spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb
 - spec/topological_inventory/providers/common/collectors_pool_spec.rb
 - spec/topological_inventory/providers/common/logger_spec.rb
+- spec/topological_inventory/providers/common/operations/async_worker_spec.rb
 - spec/topological_inventory/providers/common/operations/processor_spec.rb
 - spec/topological_inventory/providers/common/operations/source_spec.rb
 - spec/topological_inventory/providers/common/save_inventory/saver_spec.rb