topological_inventory-providers-common 1.0.12 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b81c523169c3ca87e678e3e7d046b4daee863200cf4eadd695e9022d17a11e5a
-  data.tar.gz: 1fc2a47cedf6f8d3b41f0df728a5ac8ceceaea03b58d43ef67da90680861c0eb
+  metadata.gz: 9848e297a81db270e52355b3caf500b38391f1c35e118ca31cadbec35c3f0c5e
+  data.tar.gz: ad23c8d4f9a46b6824b20e2d870611d4b3a9c570da00f00a262513934b2349ab
 SHA512:
-  metadata.gz: 9afb8ba326d632ba59245d93837f32bd8edbcb34b3d82e029c351f904bb75f196592eb4724116a48caf0b7df8fc7f4101f5b59bd0e23b65d6fca25ada3ac4cb6
-  data.tar.gz: '00924d04d1fc4623cd4e0f0a719c5471dbd39635820375b4957278db1c6e3ecbdf2b58c25758bca22e91e979511286708279922919f6303118401e58ac9cc5c4'
+  metadata.gz: b9e92c54114310f529eb78c8faa1ec64fd56350c82086ea2fbe126594afa596757e3a39e6f068929c093b0a607e60cf6ca46f2671dea867a5c87b1f59cae6de7
+  data.tar.gz: 567e5f95a9138dbc8ac7dd52e26a7fe923075552c89d24c3c4e6111f072c0b1b0fb61c5a1fd463bec86d87f00e359b9ca55cbe728fff5b4a0a2fc7a07e4853ba

data/.codeclimate.yml

@@ -0,0 +1,31 @@
+---
+version: '2'
+prepare:
+  fetch:
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
+    path: ".rubocop_base.yml"
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_cc_base.yml
+    path: ".rubocop_cc_base.yml"
+checks:
+  argument-count:
+    enabled: false
+  complex-logic:
+    enabled: false
+  file-lines:
+    enabled: false
+  method-complexity:
+    config:
+      threshold: 11
+  method-count:
+    enabled: false
+  method-lines:
+    enabled: false
+  nested-control-flow:
+    enabled: false
+  return-statements:
+    enabled: false
+plugins:
+  rubocop:
+    enabled: true
+    config: ".rubocop_cc.yml"
+    channel: rubocop-1-0

data/.rubocop.yml

@@ -1,3 +1,3 @@
 inherit_from:
-- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
+- https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
 - .rubocop_local.yml

data/.yamllint

@@ -0,0 +1,8 @@
+---
+ignore: |
+
+extends: relaxed
+
+rules:
+  line-length:
+    max: 120

data/CHANGELOG.md

@@ -4,17 +4,37 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.12] - 2020-10-01
+## [2.1.3] - 2020-12-14
+Update travis URL in README file #67
+Kafka availability_checks fix #68
+
+## [2.1.2] - 2020-11-24
+Custom Metrics for AsyncWorker #60
+Change error metric name to errors_total #64
+
+## [2.1.1] - 2020-11-04
+MessagingClient and Source fix #61
+Fix availability_check in app check #62
+
+## [2.1.0] - 2020-11-02
+Add Availability checks for sources via Kafka #54
+Common Metrics exporter #57
+Rubocop rules from insights-api-common + yamllint #59
+
+## [2.0.0] - 2020-10-20
+Operations/API clients refactoring
+
+## [1.0.12] - 2020-10-05
 Add Operations Async Worker class #55
 
-## [1.0.11]
+## [1.0.11] - 2020-09-04
 Make Collector Poll Time a parameter so we can tweak the collection interval #51
 
-## [1.0.10]
+## [1.0.10] - 2020-08-26
 Add HealthCheck class for operations workers #48
 Set the LOG_LEVEL if present #50
 
-## [1.0.9]
+## [1.0.9] - 2020-08-17
 Added refresh-type to save and sweep inventory #45
 
 ## [1.0.8] - 2020-08-12
@@ -59,7 +79,12 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.3...HEAD
+[2.1.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.2...v2.1.3
+[2.1.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.1...v2.1.2
+[2.1.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...v2.1.1
+[2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...v2.1.0
+[2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0
 [1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12
 [1.0.11]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.10...v1.0.11
 [1.0.10]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.9...v1.0.10

data/README.md

@@ -1,6 +1,6 @@
 # TopologicalInventory::Providers::Common
 
-[![Build Status](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common)
+[![Build Status](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common)
 [![Maintainability](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/maintainability)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/test_coverage)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/test_coverage)
 [![security](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master.svg)](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master)

data/lib/topological_inventory/providers/common.rb

@@ -1,10 +1,9 @@
 require "topological_inventory/providers/common/version"
 require "topological_inventory/providers/common/logging"
-require "topological_inventory/providers/common/operations/processor"
-require "topological_inventory/providers/common/operations/endpoint_client"
 require "topological_inventory/providers/common/operations/health_check"
 require "topological_inventory/providers/common/collectors_pool"
 require "topological_inventory/providers/common/collector"
+require "topological_inventory/providers/common/metrics"
 
 module TopologicalInventory
   module Providers

data/lib/topological_inventory/providers/common/logging.rb

@@ -23,11 +23,14 @@ module TopologicalInventory
         end
 
         def availability_check(message, severity = :info)
-          log_with_prefix("Source#availability_check", message, severity)
+          send("#{severity}_ext", "Source#availability_check", message)
         end
 
-        def log_with_prefix(prefix, message, severity)
-          send(severity, "#{prefix} - #{message}") if respond_to?(severity)
+        %w[debug info warn error fatal].each do |severity|
+          define_method("#{severity}_ext".to_sym) do |prefix, message|
+            ext_message = [prefix, message].compact.join(' - ')
+            send(severity, ext_message)
+          end
         end
       end
 

data/lib/topological_inventory/providers/common/messaging_client.rb

@@ -0,0 +1,40 @@
+require "more_core_extensions/core_ext/module/cache_with_timeout"
+require "manageiq-messaging"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class MessagingClient
+        # Kafka host name
+        attr_accessor :queue_host
+        # Kafka port
+        attr_accessor :queue_port
+
+        def initialize
+          self.queue_host = ENV['QUEUE_HOST'] || 'localhost'
+          self.queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
+        end
+
+        def self.default
+          @@default ||= new
+        end
+
+        def self.configure
+          if block_given?
+            yield(default)
+          else
+            default
+          end
+        end
+
+        cache_with_timeout(:client) do
+          ManageIQ::Messaging::Client.open(:protocol => :Kafka, :host => default.queue_host, :port => default.queue_port)
+        end
+
+        def client
+          self.class.client
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/metrics.rb

@@ -0,0 +1,84 @@
+require "benchmark"
+require "prometheus_exporter"
+require "prometheus_exporter/server"
+require "prometheus_exporter/client"
+require "prometheus_exporter/instrumentation"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Metrics
+        ERROR_COUNTER_MESSAGE = "total number of errors".freeze
+
+        def initialize(port = 9394)
+          return if port == 0
+
+          configure_server(port)
+          configure_metrics
+        end
+
+        def stop_server
+          @server&.stop
+        end
+
+        def record_error(type = :general)
+          @error_counter&.observe(1, :type => type.to_s)
+        end
+
+        def record_operation(name, labels = {})
+          @status_counter&.observe(1, (labels || {}).merge(:name => name))
+        end
+
+        def record_operation_time(name, labels = {}, &block)
+          record_time(@duration_seconds, (labels || {}).merge(:name => name), &block)
+        end
+
+        # Common method for gauge
+        def record_gauge(metric, opt, value: nil, labels: {})
+          case opt
+          when :set then
+            metric&.observe(value.to_i, labels)
+          when :add then
+            metric&.increment(labels)
+          when :remove then
+            metric&.decrement(labels)
+          end
+        end
+
+        # Common method for histogram
+        def record_time(metric, labels = {})
+          result = nil
+          time = Benchmark.realtime { result = yield }
+          metric&.observe(time, labels)
+          result
+        end
+
+        private
+
+        def configure_server(port)
+          @server = PrometheusExporter::Server::WebServer.new(:port => port)
+          @server.start
+
+          PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(:collector => @server.collector)
+        end
+
+        def configure_metrics
+          PrometheusExporter::Instrumentation::Process.start
+          PrometheusExporter::Metric::Base.default_prefix = default_prefix
+
+          @duration_seconds = PrometheusExporter::Metric::Histogram.new('duration_seconds', 'Duration of processed operation')
+          @error_counter = PrometheusExporter::Metric::Counter.new('errors_total', ERROR_COUNTER_MESSAGE)
+          @status_counter = PrometheusExporter::Metric::Counter.new('status_counter', 'number of processed operations')
+
+          [@duration_seconds, @error_counter, @status_counter].each do |metric|
+            @server.collector.register_metric(metric)
+          end
+        end
+
+        def default_prefix
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/sources_api.rb

@@ -0,0 +1,61 @@
+require "topological_inventory/providers/common/sources_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module SourcesApi
+          AUTH_NOT_NECESSARY = "n/a".freeze
+
+          def sources_api
+            @sources_api ||= TopologicalInventory::Providers::Common::SourcesApiClient.new(identity)
+          end
+
+          def endpoint
+            @endpoint ||= sources_api.fetch_default_endpoint(source_id)
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Endpoint for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def authentication
+            @authentication ||= if endpoint.receptor_node.present?
+                                  AUTH_NOT_NECESSARY
+                                else
+                                  sources_api.fetch_authentication(source_id, endpoint)
+                                end
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Authentication for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def application
+            @application ||= sources_api.fetch_application(source_id)
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Application for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def on_premise?
+            @on_premise ||= endpoint&.receptor_node.to_s.strip.present?
+          end
+
+          def verify_ssl_mode
+            endpoint&.verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+          end
+
+          def full_hostname(endpoint)
+            if on_premise?
+              "receptor://#{endpoint.receptor_node}"
+            else
+              endpoint.host.tap { |host| host << ":#{endpoint.port}" if endpoint.port }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/statuses.rb

@@ -0,0 +1,19 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module Statuses
+          def operation_status
+            return @statuses if @statuses.present?
+
+            @statuses = {}
+            %i[success error skipped not_implemented].each do |status|
+              @statuses[status] = status.to_s
+            end
+            @statuses
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/topology_api.rb

@@ -0,0 +1,26 @@
+require "topological_inventory/providers/common/topology_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module TopologyApi
+          # @identity attr_reader is expected
+          def topology_api
+            @topology_api ||= TopologicalInventory::Providers::Common::TopologyApiClient.new(identity)
+          end
+
+          def update_task(task_id, source_id: nil, state:, status:, target_type: nil, target_source_ref: nil, context: nil)
+            topology_api.update_task(task_id,
+                                     :source_id         => source_id,
+                                     :state             => state,
+                                     :status            => status,
+                                     :target_type       => target_type,
+                                     :target_source_ref => target_source_ref,
+                                     :context           => context)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/x_rh_headers.rb

@@ -0,0 +1,24 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module XRhHeaders
+          def account_number_by_identity(identity)
+            return @account_number if @account_number
+            return if identity.try(:[], 'x-rh-identity').nil?
+
+            identity_hash = JSON.parse(Base64.decode64(identity['x-rh-identity']))
+            @account_number = identity_hash.dig('identity', 'account_number')
+          rescue JSON::ParserError => e
+            logger.error_ext(operation, "Failed to parse identity header: #{e.message}")
+            nil
+          end
+
+          def identity_by_account_number(account_number)
+            @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => account_number, "user" => {"is_org_admin" => true}}}.to_json)}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/async_worker.rb

@@ -1,4 +1,5 @@
 require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/mixins/statuses"
 require "topological_inventory/providers/common/operations/health_check"
 
 module TopologicalInventory
@@ -7,10 +8,12 @@ module TopologicalInventory
       module Operations
         class AsyncWorker
           include Logging
+          include TopologicalInventory::Providers::Common::Mixins::Statuses
 
-          def initialize(processor, queue = nil)
+          def initialize(processor, queue: nil, metrics: nil)
             @processor = processor
-            @queue = queue || Queue.new
+            @queue     = queue || Queue.new
+            @metrics   = metrics
           end
 
           def start
@@ -37,15 +40,17 @@ module TopologicalInventory
 
           private
 
-          attr_reader :thread, :queue, :processor
+          attr_reader :thread, :queue, :processor, :metrics
 
-          def process_message(msg)
-            processor.process!(msg)
+          def process_message(message)
+            result = processor.process!(message, metrics)
+            metrics&.record_operation(message.message, :status => result)
           rescue => err
-            model, method = msg.message.to_s.split(".")
+            model, method = message.message.to_s.split(".")
             logger.error("#{model}##{method}: async worker failure: #{err.cause}\n#{err}\n#{err.backtrace.join("\n")}")
+            metrics&.record_operation(message.message, :status => operation_status[:error])
           ensure
-            msg.ack
+            message.ack
             TopologicalInventory::Providers::Common::Operations::HealthCheck.touch_file
             logger.debug("Operations::AsyncWorker queue length: #{queue.length}") if queue.length >= 20 && queue.length % 5 == 0
           end