topological_inventory-providers-common 1.0.11 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2593d36aee528450e29282d903ddcc2d74c6593e9579c69ec2a510c9fcc15ab
-  data.tar.gz: 47b0675fcab7eaab9c2b3899837b597295a0e7c6c737f1ce39ca49c4a7968cc5
+  metadata.gz: 661072299fccb0117c3dcbe1fba36e4836a82ac1a6d98abbaab1808ea437e6d7
+  data.tar.gz: a1a45e527dc14cb4b09004d2df98f075818fb35d616d84d954839c289dac5c8a
 SHA512:
-  metadata.gz: 03f98f6334065f62a4462e30633c6a6d852552f49414be2293b4864d3971372ce66ab52d47e23b4a7ad2f8c0277d9d2352f38519e2621a80f6c9b2b011a7cdd4
-  data.tar.gz: b801478114b42cc897329a12eca067cc707306f488ed7e84eddf7eddd91c0098dbbdf5d6d3fcb116734b39cd08d25dcabdd6731b60057fc900ca2c39a5008bd8
+  metadata.gz: ba101d97cca75814a073eb70f8528e3debb0f4270fb4cfb98cc9db55f53ff790efcd99380158e8c808b317603a9ce1a7a30d13f791e03741b699e06f0556965e
+  data.tar.gz: 3e4340ad6743693f104d0c847995bb866860ef2911c57865ce042389cb3872badf05f2b46119664650214285f509495f656dc5cf82f77cec9cf5e2a253a0f2cc

data/.codeclimate.yml

@@ -0,0 +1,31 @@
+---
+version: '2'
+prepare:
+  fetch:
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
+    path: ".rubocop_base.yml"
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_cc_base.yml
+    path: ".rubocop_cc_base.yml"
+checks:
+  argument-count:
+    enabled: false
+  complex-logic:
+    enabled: false
+  file-lines:
+    enabled: false
+  method-complexity:
+    config:
+      threshold: 11
+  method-count:
+    enabled: false
+  method-lines:
+    enabled: false
+  nested-control-flow:
+    enabled: false
+  return-statements:
+    enabled: false
+plugins:
+  rubocop:
+    enabled: true
+    config: ".rubocop_cc.yml"
+    channel: rubocop-1-0

data/.rubocop.yml

@@ -1,3 +1,3 @@
 inherit_from:
-- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
+- https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
 - .rubocop_local.yml

data/.yamllint

@@ -0,0 +1,8 @@
+---
+ignore: |
+
+extends: relaxed
+
+rules:
+  line-length:
+    max: 120

data/CHANGELOG.md

@@ -4,14 +4,33 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.11]
+## [2.1.2] - 2020-11-24
+Custom Metrics for AsyncWorker #60
+Change error metric name to errors_total #64
+
+## [2.1.1] - 2020-11-04
+MessagingClient and Source fix #61
+Fix availability_check in app check #62
+
+## [2.1.0] - 2020-11-02
+Add Availability checks for sources via Kafka #54
+Common Metrics exporter #57
+Rubocop rules from insights-api-common + yamllint #59
+
+## [2.0.0] - 2020-10-20
+Operations/API clients refactoring
+
+## [1.0.12] - 2020-10-05
+Add Operations Async Worker class #55
+
+## [1.0.11] - 2020-09-04
 Make Collector Poll Time a parameter so we can tweak the collection interval #51
 
-## [1.0.10]
+## [1.0.10] - 2020-08-26
 Add HealthCheck class for operations workers #48
 Set the LOG_LEVEL if present #50
 
-## [1.0.9]
+## [1.0.9] - 2020-08-17
 Added refresh-type to save and sweep inventory #45
 
 ## [1.0.8] - 2020-08-12
@@ -56,7 +75,12 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.2...HEAD
+[2.1.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.1...v2.1.2
+[2.1.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...v2.1.1
+[2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...v2.1.0
+[2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0
+[1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12
 [1.0.11]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.10...v1.0.11
 [1.0.10]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.9...v1.0.10
 [1.0.9]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.8...v1.0.9

data/lib/topological_inventory/providers/common.rb

@@ -1,10 +1,9 @@
 require "topological_inventory/providers/common/version"
 require "topological_inventory/providers/common/logging"
-require "topological_inventory/providers/common/operations/processor"
-require "topological_inventory/providers/common/operations/endpoint_client"
 require "topological_inventory/providers/common/operations/health_check"
 require "topological_inventory/providers/common/collectors_pool"
 require "topological_inventory/providers/common/collector"
+require "topological_inventory/providers/common/metrics"
 
 module TopologicalInventory
   module Providers

data/lib/topological_inventory/providers/common/logging.rb

@@ -23,11 +23,14 @@ module TopologicalInventory
         end
 
         def availability_check(message, severity = :info)
-          log_with_prefix("Source#availability_check", message, severity)
+          send("#{severity}_ext", "Source#availability_check", message)
         end
 
-        def log_with_prefix(prefix, message, severity)
-          send(severity, "#{prefix} - #{message}") if respond_to?(severity)
+        %w[debug info warn error fatal].each do |severity|
+          define_method("#{severity}_ext".to_sym) do |prefix, message|
+            ext_message = [prefix, message].compact.join(' - ')
+            send(severity, ext_message)
+          end
         end
       end
 

data/lib/topological_inventory/providers/common/messaging_client.rb

@@ -0,0 +1,40 @@
+require "more_core_extensions/core_ext/module/cache_with_timeout"
+require "manageiq-messaging"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class MessagingClient
+        # Kafka host name
+        attr_accessor :queue_host
+        # Kafka port
+        attr_accessor :queue_port
+
+        def initialize
+          self.queue_host = ENV['QUEUE_HOST'] || 'localhost'
+          self.queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
+        end
+
+        def self.default
+          @@default ||= new
+        end
+
+        def self.configure
+          if block_given?
+            yield(default)
+          else
+            default
+          end
+        end
+
+        cache_with_timeout(:client) do
+          ManageIQ::Messaging::Client.open(:protocol => :Kafka, :host => default.queue_host, :port => default.queue_port)
+        end
+
+        def client
+          self.class.client
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/metrics.rb

@@ -0,0 +1,84 @@
+require "benchmark"
+require "prometheus_exporter"
+require "prometheus_exporter/server"
+require "prometheus_exporter/client"
+require "prometheus_exporter/instrumentation"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Metrics
+        ERROR_COUNTER_MESSAGE = "total number of errors".freeze
+
+        def initialize(port = 9394)
+          return if port == 0
+
+          configure_server(port)
+          configure_metrics
+        end
+
+        def stop_server
+          @server&.stop
+        end
+
+        def record_error(type = :general)
+          @error_counter&.observe(1, :type => type.to_s)
+        end
+
+        def record_operation(name, labels = {})
+          @status_counter&.observe(1, (labels || {}).merge(:name => name))
+        end
+
+        def record_operation_time(name, labels = {}, &block)
+          record_time(@duration_seconds, (labels || {}).merge(:name => name), &block)
+        end
+
+        # Common method for gauge
+        def record_gauge(metric, opt, value: nil, labels: {})
+          case opt
+          when :set then
+            metric&.observe(value.to_i, labels)
+          when :add then
+            metric&.increment(labels)
+          when :remove then
+            metric&.decrement(labels)
+          end
+        end
+
+        # Common method for histogram
+        def record_time(metric, labels = {})
+          result = nil
+          time = Benchmark.realtime { result = yield }
+          metric&.observe(time, labels)
+          result
+        end
+
+        private
+
+        def configure_server(port)
+          @server = PrometheusExporter::Server::WebServer.new(:port => port)
+          @server.start
+
+          PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(:collector => @server.collector)
+        end
+
+        def configure_metrics
+          PrometheusExporter::Instrumentation::Process.start
+          PrometheusExporter::Metric::Base.default_prefix = default_prefix
+
+          @duration_seconds = PrometheusExporter::Metric::Histogram.new('duration_seconds', 'Duration of processed operation')
+          @error_counter = PrometheusExporter::Metric::Counter.new('errors_total', ERROR_COUNTER_MESSAGE)
+          @status_counter = PrometheusExporter::Metric::Counter.new('status_counter', 'number of processed operations')
+
+          [@duration_seconds, @error_counter, @status_counter].each do |metric|
+            @server.collector.register_metric(metric)
+          end
+        end
+
+        def default_prefix
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/sources_api.rb

@@ -0,0 +1,61 @@
+require "topological_inventory/providers/common/sources_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module SourcesApi
+          AUTH_NOT_NECESSARY = "n/a".freeze
+
+          def sources_api
+            @sources_api ||= TopologicalInventory::Providers::Common::SourcesApiClient.new(identity)
+          end
+
+          def endpoint
+            @endpoint ||= sources_api.fetch_default_endpoint(source_id)
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Endpoint for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def authentication
+            @authentication ||= if endpoint.receptor_node.present?
+                                  AUTH_NOT_NECESSARY
+                                else
+                                  sources_api.fetch_authentication(source_id, endpoint)
+                                end
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Authentication for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def application
+            @application ||= sources_api.fetch_application(source_id)
+          rescue => e
+            metrics&.record_error(:sources_api)
+            logger.error_ext(operation, "Failed to fetch Application for Source #{source_id}: #{e.message}")
+            nil
+          end
+
+          def on_premise?
+            @on_premise ||= endpoint&.receptor_node.to_s.strip.present?
+          end
+
+          def verify_ssl_mode
+            endpoint&.verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+          end
+
+          def full_hostname(endpoint)
+            if on_premise?
+              "receptor://#{endpoint.receptor_node}"
+            else
+              endpoint.host.tap { |host| host << ":#{endpoint.port}" if endpoint.port }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/statuses.rb

@@ -0,0 +1,19 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module Statuses
+          def operation_status
+            return @statuses if @statuses.present?
+
+            @statuses = {}
+            %i[success error skipped not_implemented].each do |status|
+              @statuses[status] = status.to_s
+            end
+            @statuses
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/topology_api.rb

@@ -0,0 +1,26 @@
+require "topological_inventory/providers/common/topology_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module TopologyApi
+          # @identity attr_reader is expected
+          def topology_api
+            @topology_api ||= TopologicalInventory::Providers::Common::TopologyApiClient.new(identity)
+          end
+
+          def update_task(task_id, source_id: nil, state:, status:, target_type: nil, target_source_ref: nil, context: nil)
+            topology_api.update_task(task_id,
+                                     :source_id         => source_id,
+                                     :state             => state,
+                                     :status            => status,
+                                     :target_type       => target_type,
+                                     :target_source_ref => target_source_ref,
+                                     :context           => context)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/x_rh_headers.rb

@@ -0,0 +1,24 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module XRhHeaders
+          def account_number_by_identity(identity)
+            return @account_number if @account_number
+            return if identity.try(:[], 'x-rh-identity').nil?
+
+            identity_hash = JSON.parse(Base64.decode64(identity['x-rh-identity']))
+            @account_number = identity_hash.dig('identity', 'account_number')
+          rescue JSON::ParserError => e
+            logger.error_ext(operation, "Failed to parse identity header: #{e.message}")
+            nil
+          end
+
+          def identity_by_account_number(account_number)
+            @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => account_number, "user" => {"is_org_admin" => true}}}.to_json)}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/async_worker.rb

@@ -0,0 +1,61 @@
+require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/mixins/statuses"
+require "topological_inventory/providers/common/operations/health_check"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class AsyncWorker
+          include Logging
+          include TopologicalInventory::Providers::Common::Mixins::Statuses
+
+          def initialize(processor, queue: nil, metrics: nil)
+            @processor = processor
+            @queue     = queue || Queue.new
+            @metrics   = metrics
+          end
+
+          def start
+            return if thread.present?
+
+            @thread = Thread.new { listen }
+          end
+
+          def stop
+            thread&.exit
+          end
+
+          def enqueue(msg)
+            queue << msg
+          end
+
+          def listen
+            loop do
+              # the queue thread waits for a message to come during `Queue#pop`
+              msg = queue.pop
+              process_message(msg)
+            end
+          end
+
+          private
+
+          attr_reader :thread, :queue, :processor, :metrics
+
+          def process_message(message)
+            result = processor.process!(message, metrics)
+            metrics&.record_operation(message.message, :status => result)
+          rescue => err
+            model, method = message.message.to_s.split(".")
+            logger.error("#{model}##{method}: async worker failure: #{err.cause}\n#{err}\n#{err.backtrace.join("\n")}")
+            metrics&.record_operation(message.message, :status => operation_status[:error])
+          ensure
+            message.ack
+            TopologicalInventory::Providers::Common::Operations::HealthCheck.touch_file
+            logger.debug("Operations::AsyncWorker queue length: #{queue.length}") if queue.length >= 20 && queue.length % 5 == 0
+          end
+        end
+      end
+    end
+  end
+end