topological_inventory-providers-common 1.0.9 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 880ff945277cf130a2617a20278ff29298e8a7644b92e254e7c4dd32002a58bc
-  data.tar.gz: b3e65e3652f156fc0de8e2f6d97c1a9eac095d015035ae08d787c86f1da3b597
+  metadata.gz: aad209b97efea9bb19ce488782adb1dabfa8d650c12f8637e2b348ed4ec95a15
+  data.tar.gz: 69b9bbed6a2d725990206ab3bfc7b13ec25b10f7e0c1a979b48c61b67e53fd0d
 SHA512:
-  metadata.gz: 7ad7a27bd9ae06e1161917aa5fa55f20f87ed9a69d9e1936968e21c7c4bab634924317e544afeaaa8530de2998bd156f912435eef9c3151758e82f8f3f0f9c08
-  data.tar.gz: 37e3bdc21b29c71927a0c797a6cc56267a8be9ab1031a702ed6d6fa8f38c76c38ddb13baf32f1eeaccf3306328daed21edbcedeb0a9f1669c0df3ee9a685c029
+  metadata.gz: 45722b70f009a764d8cd804424157cc98b1a225b72326b8b8971fecd093fa6260446777db76bc3502e487d27dc82b2b4325959302f4a1abe13fbbf8c015c1d54
+  data.tar.gz: b1a6de8e2eb93b2b26d4b609f17a1a518cf4b318dbcf117c6225fe297bb3ec97f870b944e151786977ad90f42c16746eeeadf6a3d113b648921e278cb831aadd

data/.codeclimate.yml

@@ -0,0 +1,31 @@
+---
+version: '2'
+prepare:
+  fetch:
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
+    path: ".rubocop_base.yml"
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_cc_base.yml
+    path: ".rubocop_cc_base.yml"
+checks:
+  argument-count:
+    enabled: false
+  complex-logic:
+    enabled: false
+  file-lines:
+    enabled: false
+  method-complexity:
+    config:
+      threshold: 11
+  method-count:
+    enabled: false
+  method-lines:
+    enabled: false
+  nested-control-flow:
+    enabled: false
+  return-statements:
+    enabled: false
+plugins:
+  rubocop:
+    enabled: true
+    config: ".rubocop_cc.yml"
+    channel: rubocop-1-0

data/.rubocop.yml

@@ -1,3 +1,3 @@
 inherit_from:
-- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
+- https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
 - .rubocop_local.yml

data/.yamllint

@@ -0,0 +1,8 @@
+---
+ignore: |
+
+extends: relaxed
+
+rules:
+  line-length:
+    max: 120

data/CHANGELOG.md

@@ -4,7 +4,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.9]
+## [2.1.0]
+Add Availability checks for sources via Kafka #54
+Common Metrics exporter #57
+Rubocop rules from insights-api-common + yamllint #59
+
+## [2.0.0] - 2020-10-20
+Operations/API clients refactoring
+
+## [1.0.12] - 2020-10-01
+Add Operations Async Worker class #55
+
+## [1.0.11] - 2020-09-04
+Make Collector Poll Time a parameter so we can tweak the collection interval #51
+
+## [1.0.10] - 2020-08-26
+Add HealthCheck class for operations workers #48
+Set the LOG_LEVEL if present #50
+
+## [1.0.9] - 2020-08-17
 Added refresh-type to save and sweep inventory #45
 
 ## [1.0.8] - 2020-08-12
@@ -49,7 +67,12 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.9...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...HEAD
+[2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...v2.1.0
+[2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0
+[1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12
+[1.0.11]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.10...v1.0.11
+[1.0.10]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.9...v1.0.10
 [1.0.9]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.8...v1.0.9
 [1.0.8]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.7...v1.0.8
 [1.0.7]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...v1.0.7

data/lib/topological_inventory/providers/common.rb

@@ -1,9 +1,9 @@
 require "topological_inventory/providers/common/version"
 require "topological_inventory/providers/common/logging"
-require "topological_inventory/providers/common/operations/processor"
-require "topological_inventory/providers/common/operations/endpoint_client"
+require "topological_inventory/providers/common/operations/health_check"
 require "topological_inventory/providers/common/collectors_pool"
 require "topological_inventory/providers/common/collector"
+require "topological_inventory/providers/common/metrics"
 
 module TopologicalInventory
   module Providers

data/lib/topological_inventory/providers/common/collectors_pool.rb

@@ -5,8 +5,9 @@ module TopologicalInventory
     module Common
       class CollectorsPool
         SECRET_FILENAME = "credentials".freeze
+        COLLECTOR_POLL_TIME = ENV['COLLECTOR_POLL_TIME']&.to_i || 300
 
-        def initialize(config_name, metrics, collector_poll_time: 60, thread_pool_size: 2)
+        def initialize(config_name, metrics, collector_poll_time: COLLECTOR_POLL_TIME, thread_pool_size: 2)
           self.config_name         = config_name
           self.collector_status    = Concurrent::Map.new
           self.metrics             = metrics

data/lib/topological_inventory/providers/common/logging.rb

@@ -23,17 +23,23 @@ module TopologicalInventory
         end
 
         def availability_check(message, severity = :info)
-          log_with_prefix("Source#availability_check", message, severity)
+          send("#{severity}_ext", "Source#availability_check", message)
         end
 
-        def log_with_prefix(prefix, message, severity)
-          send(severity, "#{prefix} - #{message}") if respond_to?(severity)
+        %w[debug info warn error fatal].each do |severity|
+          define_method("#{severity}_ext".to_sym) do |prefix, message|
+            ext_message = [prefix, message].compact.join(' - ')
+            send(severity, ext_message)
+          end
         end
       end
 
       class Logger < ManageIQ::Loggers::CloudWatch
         def self.new(*args)
-          super.tap { |logger| logger.extend(TopologicalInventory::Providers::Common::LoggingFunctions) }
+          super.tap do |logger|
+            logger.extend(TopologicalInventory::Providers::Common::LoggingFunctions)
+            logger.level = ENV['LOG_LEVEL'] if ENV['LOG_LEVEL']
+          end
         end
       end
 

data/lib/topological_inventory/providers/common/messaging_client.rb

@@ -0,0 +1,40 @@
+require "more_core_extensions/core_ext/module/cache_with_timeout"
+require "manageiq-messaging"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class MessagingClient
+        # Kafka host name
+        attr_accessor :queue_host
+        # Kafka port
+        attr_accessor :queue_port
+
+        def initialize
+          @queue_host = ENV['QUEUE_HOST'] || 'localhost'
+          @queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
+        end
+
+        def self.default
+          @@default ||= new
+        end
+
+        def self.configure
+          if block_given?
+            yield(default)
+          else
+            default
+          end
+        end
+
+        cache_with_timeout(:client) do
+          ManageIQ::Messaging::Client.open(:protocol => :Kafka, :host => @queue_host, :port => @queue_port)
+        end
+
+        def client
+          self.class.client
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/metrics.rb

@@ -0,0 +1,84 @@
+require "benchmark"
+require "prometheus_exporter"
+require "prometheus_exporter/server"
+require "prometheus_exporter/client"
+require "prometheus_exporter/instrumentation"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Metrics
+        ERROR_COUNTER_MESSAGE = "total number of errors".freeze
+
+        def initialize(port = 9394)
+          return if port == 0
+
+          configure_server(port)
+          configure_metrics
+        end
+
+        def stop_server
+          @server&.stop
+        end
+
+        def record_error(type = :general)
+          @error_counter&.observe(1, :type => type.to_s)
+        end
+
+        def record_operation(name, labels = {})
+          @status_counter&.observe(1, (labels || {}).merge(:name => name))
+        end
+
+        def record_operation_time(name, labels = {}, &block)
+          record_time(@duration_seconds, (labels || {}).merge(:name => name), &block)
+        end
+
+        # Common method for gauge
+        def record_gauge(metric, opt, value: nil, labels: {})
+          case opt
+          when :set then
+            metric&.observe(value.to_i, labels)
+          when :add then
+            metric&.increment(labels)
+          when :remove then
+            metric&.decrement(labels)
+          end
+        end
+
+        # Common method for histogram
+        def record_time(metric, labels = {})
+          result = nil
+          time = Benchmark.realtime { result = yield }
+          metric&.observe(time, labels)
+          result
+        end
+
+        private
+
+        def configure_server(port)
+          @server = PrometheusExporter::Server::WebServer.new(:port => port)
+          @server.start
+
+          PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(:collector => @server.collector)
+        end
+
+        def configure_metrics
+          PrometheusExporter::Instrumentation::Process.start
+          PrometheusExporter::Metric::Base.default_prefix = default_prefix
+
+          @duration_seconds = PrometheusExporter::Metric::Histogram.new('duration_seconds', 'Duration of processed operation')
+          @error_counter = PrometheusExporter::Metric::Counter.new("error", ERROR_COUNTER_MESSAGE)
+          @status_counter = PrometheusExporter::Metric::Counter.new('status_counter', 'number of processed operations')
+
+          [@duration_seconds, @error_counter, @status_counter].each do |metric|
+            @server.collector.register_metric(metric)
+          end
+        end
+
+        def default_prefix
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/sources_api.rb

@@ -0,0 +1,61 @@
+require "topological_inventory/providers/common/sources_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module SourcesApi
+          AUTH_NOT_NECESSARY = "n/a".freeze
+
+          def sources_api
+            @sources_api ||= TopologicalInventory::Providers::Common::SourcesApiClient.new(identity)
+          end
+
+          def endpoint
+            @endpoint ||= sources_api.fetch_default_endpoint(source_id)
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Endpoint for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def authentication
+            @authentication ||= if endpoint.receptor_node.present?
+                                  AUTH_NOT_NECESSARY
+                                else
+                                  sources_api.fetch_authentication(source_id, endpoint)
+                                end
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Authentication for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def application
+            @application ||= sources_api.fetch_application(source_id)
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Application for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def on_premise?
+            @on_premise ||= endpoint&.receptor_node.to_s.strip.present?
+          end
+
+          def verify_ssl_mode
+            endpoint&.verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+          end
+
+          def full_hostname(endpoint)
+            if on_premise?
+              "receptor://#{endpoint.receptor_node}"
+            else
+              endpoint.host.tap { |host| host << ":#{endpoint.port}" if endpoint.port }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/statuses.rb

@@ -0,0 +1,19 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module Statuses
+          def operation_status
+            return @statuses if @statuses.present?
+
+            @statuses = {}
+            %i[success error skipped not_implemented].each do |status|
+              @statuses[status] = status.to_s
+            end
+            @statuses
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/topology_api.rb

@@ -0,0 +1,26 @@
+require "topological_inventory/providers/common/topology_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module TopologyApi
+          # @identity attr_reader is expected
+          def topology_api
+            @topology_api ||= TopologicalInventory::Providers::Common::TopologyApiClient.new(identity)
+          end
+
+          def update_task(task_id, source_id: nil, state:, status:, target_type: nil, target_source_ref: nil, context: nil)
+            topology_api.update_task(task_id,
+                                     :source_id         => source_id,
+                                     :state             => state,
+                                     :status            => status,
+                                     :target_type       => target_type,
+                                     :target_source_ref => target_source_ref,
+                                     :context           => context)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/x_rh_headers.rb

@@ -0,0 +1,24 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module XRhHeaders
+          def account_number_by_identity(identity)
+            return @account_number if @account_number
+            return if identity.try(:[], 'x-rh-identity').nil?
+
+            identity_hash = JSON.parse(Base64.decode64(identity['x-rh-identity']))
+            @account_number = identity_hash.dig('identity', 'account_number')
+          rescue JSON::ParserError => e
+            logger.error_ext(operation, "Failed to parse identity header: #{e.message}")
+            nil
+          end
+
+          def identity_by_account_number(account_number)
+            @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => account_number, "user" => {"is_org_admin" => true}}}.to_json)}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/async_worker.rb

@@ -0,0 +1,56 @@
+require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/operations/health_check"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class AsyncWorker
+          include Logging
+
+          def initialize(processor, queue = nil)
+            @processor = processor
+            @queue = queue || Queue.new
+          end
+
+          def start
+            return if thread.present?
+
+            @thread = Thread.new { listen }
+          end
+
+          def stop
+            thread&.exit
+          end
+
+          def enqueue(msg)
+            queue << msg
+          end
+
+          def listen
+            loop do
+              # the queue thread waits for a message to come during `Queue#pop`
+              msg = queue.pop
+              process_message(msg)
+            end
+          end
+
+          private
+
+          attr_reader :thread, :queue, :processor
+
+          def process_message(msg)
+            processor.process!(msg)
+          rescue => err
+            model, method = msg.message.to_s.split(".")
+            logger.error("#{model}##{method}: async worker failure: #{err.cause}\n#{err}\n#{err.backtrace.join("\n")}")
+          ensure
+            msg.ack
+            TopologicalInventory::Providers::Common::Operations::HealthCheck.touch_file
+            logger.debug("Operations::AsyncWorker queue length: #{queue.length}") if queue.length >= 20 && queue.length % 5 == 0
+          end
+        end
+      end
+    end
+  end
+end