topological_inventory-providers-common 2.0.0 → 2.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 134e945b201072b8bdd7fc6af73b04a6f60698da831ef3989ab973453a1a5159
-  data.tar.gz: 20e4f260c411551572b557f4e4454fdc03c9e3b2ac9c8f5997e6600780298c1a
+  metadata.gz: 9c17638d4fd82db196de0019069a765bf1c6becdb20498765507ae35a02727eb
+  data.tar.gz: a395cd7243591bc3dab9f841e491bb978a2502d2ba1cfec3fdb004de784523dd
 SHA512:
-  metadata.gz: 0a847566f6fb20e9518faef660c626788e6afd20d26cec28a5899e645c68cedd7c017974d0cb1c3c2524253007386b9728a1bb67a0f0a9101fe62bd388f03ccb
-  data.tar.gz: 6e786659b1bf913fe1e22d9c0618986d03dc4bc169d782e446ecd089b97a136b639f83be85c562006dccde83a8d025b102863b565aa571746767fa292e4dae89
+  metadata.gz: ba2b33ec9748ef58b4ca7dc30f14d34443c17026e7915fb2bd9aef2a4ec95c4c287a8149a371b91cab88974a0341f7dfd362a987ca55ea644a5b67646adafde6
+  data.tar.gz: 41edf911fa71ef7dcdaa220a417a236e8a593a01e227ab4f0f88e70318141f8736ad8bd9bff7e642d6cef1134d017baa8aea0c1c194987b08770a721a7a2ff00

data/.codeclimate.yml

@@ -0,0 +1,31 @@
+---
+version: '2'
+prepare:
+  fetch:
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
+    path: ".rubocop_base.yml"
+  - url: https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_cc_base.yml
+    path: ".rubocop_cc_base.yml"
+checks:
+  argument-count:
+    enabled: false
+  complex-logic:
+    enabled: false
+  file-lines:
+    enabled: false
+  method-complexity:
+    config:
+      threshold: 11
+  method-count:
+    enabled: false
+  method-lines:
+    enabled: false
+  nested-control-flow:
+    enabled: false
+  return-statements:
+    enabled: false
+plugins:
+  rubocop:
+    enabled: true
+    config: ".rubocop_cc.yml"
+    channel: rubocop-1-0

data/.rubocop.yml

@@ -1,3 +1,3 @@
 inherit_from:
-- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
+- https://raw.githubusercontent.com/RedHatInsights/insights-api-common-rails/master/.rubocop_base.yml
 - .rubocop_local.yml

data/.yamllint

@@ -0,0 +1,8 @@
+---
+ignore: |
+
+extends: relaxed
+
+rules:
+  line-length:
+    max: 120

data/CHANGELOG.md

@@ -3,11 +3,30 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [2.1.4] - 2020-12-14
+Common Counter metrics init #71
 
-## [2.0.0]
+## [2.1.3] - 2020-12-14
+Update travis URL in README file #67
+Kafka availability_checks fix #68
+
+## [2.1.2] - 2020-11-24
+Custom Metrics for AsyncWorker #60
+Change error metric name to errors_total #64
+
+## [2.1.1] - 2020-11-04
+MessagingClient and Source fix #61
+Fix availability_check in app check #62
+
+## [2.1.0] - 2020-11-02
+Add Availability checks for sources via Kafka #54
+Common Metrics exporter #57
+Rubocop rules from insights-api-common + yamllint #59
+
+## [2.0.0] - 2020-10-20
 Operations/API clients refactoring
 
-## [1.0.12] - 2020-10-01
+## [1.0.12] - 2020-10-05
 Add Operations Async Worker class #55
 
 ## [1.0.11] - 2020-09-04
@@ -62,7 +81,11 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.3...HEAD
+[2.1.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.2...v2.1.3
+[2.1.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.1...v2.1.2
+[2.1.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...v2.1.1
+[2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...v2.1.0
 [2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0
 [1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12
 [1.0.11]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.10...v1.0.11

data/README.md

@@ -1,6 +1,6 @@
 # TopologicalInventory::Providers::Common
 
-[![Build Status](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common)
+[![Build Status](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common)
 [![Maintainability](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/maintainability)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/test_coverage)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/test_coverage)
 [![security](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master.svg)](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master)

data/lib/topological_inventory/providers/common.rb

@@ -3,6 +3,7 @@ require "topological_inventory/providers/common/logging"
 require "topological_inventory/providers/common/operations/health_check"
 require "topological_inventory/providers/common/collectors_pool"
 require "topological_inventory/providers/common/collector"
+require "topological_inventory/providers/common/metrics"
 
 module TopologicalInventory
   module Providers

data/lib/topological_inventory/providers/common/messaging_client.rb

@@ -0,0 +1,40 @@
+require "more_core_extensions/core_ext/module/cache_with_timeout"
+require "manageiq-messaging"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class MessagingClient
+        # Kafka host name
+        attr_accessor :queue_host
+        # Kafka port
+        attr_accessor :queue_port
+
+        def initialize
+          self.queue_host = ENV['QUEUE_HOST'] || 'localhost'
+          self.queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
+        end
+
+        def self.default
+          @@default ||= new
+        end
+
+        def self.configure
+          if block_given?
+            yield(default)
+          else
+            default
+          end
+        end
+
+        cache_with_timeout(:client) do
+          ManageIQ::Messaging::Client.open(:protocol => :Kafka, :host => default.queue_host, :port => default.queue_port)
+        end
+
+        def client
+          self.class.client
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/metrics.rb

@@ -0,0 +1,109 @@
+require "benchmark"
+require "prometheus_exporter"
+require "prometheus_exporter/server"
+require "prometheus_exporter/client"
+require "prometheus_exporter/instrumentation"
+require "topological_inventory/providers/common/mixins/statuses"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Metrics
+        include TopologicalInventory::Providers::Common::Mixins::Statuses
+
+        ERROR_COUNTER_MESSAGE = "total number of errors".freeze
+        ERROR_TYPES = %i[general].freeze
+        OPERATIONS  = %w[].freeze
+
+        def initialize(port = 9394)
+          return if port == 0
+
+          configure_server(port)
+          configure_metrics
+
+          init_counters
+        end
+
+        def stop_server
+          @server&.stop
+        end
+
+        def record_error(type = :general)
+          @error_counter&.observe(1, :type => type.to_s)
+        end
+
+        def record_refresh_timing(labels = {}, &block)
+          record_time(@refresh_timer, labels, &block)
+        end
+
+        def record_operation(name, labels = {})
+          @status_counter&.observe(1, (labels || {}).merge(:name => name))
+        end
+
+        def record_operation_time(name, labels = {}, &block)
+          record_time(@duration_seconds, (labels || {}).merge(:name => name), &block)
+        end
+
+        # Common method for gauge
+        def record_gauge(metric, opt, value: nil, labels: {})
+          case opt
+          when :set then
+            metric&.observe(value.to_i, labels)
+          when :add then
+            metric&.increment(labels)
+          when :remove then
+            metric&.decrement(labels)
+          end
+        end
+
+        # Common method for histogram
+        def record_time(metric, labels = {})
+          result = nil
+          time = Benchmark.realtime { result = yield }
+          metric&.observe(time, labels)
+          result
+        end
+
+        private
+
+        # Set all values to 0 (otherwise the counter is undefined)
+        def init_counters
+          self.class::ERROR_TYPES.each do |err_type|
+            @error_counter&.observe(0, :type => err_type)
+          end
+
+          self.class::OPERATIONS.each do |op|
+            operation_status.each_key do |status|
+              @status_counter&.observe(0, :name => op, :status => status.to_s)
+            end
+          end
+        end
+
+        def configure_server(port)
+          @server = PrometheusExporter::Server::WebServer.new(:port => port)
+          @server.start
+
+          PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(:collector => @server.collector)
+        end
+
+        def configure_metrics
+          PrometheusExporter::Instrumentation::Process.start
+          PrometheusExporter::Metric::Base.default_prefix = default_prefix
+
+          @duration_seconds = PrometheusExporter::Metric::Histogram.new('duration_seconds', 'Duration of processed operation')
+          @refresh_timer = PrometheusExporter::Metric::Histogram.new('refresh_time', 'Duration of full refresh')
+          @error_counter = PrometheusExporter::Metric::Counter.new('errors_total', ERROR_COUNTER_MESSAGE)
+          @status_counter = PrometheusExporter::Metric::Counter.new('status_counter', 'number of processed operations')
+
+          [@duration_seconds, @refresh_timer, @error_counter, @status_counter].each do |metric|
+            @server.collector.register_metric(metric)
+          end
+        end
+
+        def default_prefix
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/mixins/sources_api.rb

@@ -14,6 +14,7 @@ module TopologicalInventory
           def endpoint
             @endpoint ||= sources_api.fetch_default_endpoint(source_id)
           rescue => e
+            metrics&.record_error(:sources_api)
             logger.error_ext(operation, "Failed to fetch Endpoint for Source #{source_id}: #{e.message}")
             nil
           end
@@ -25,6 +26,7 @@ module TopologicalInventory
                                   sources_api.fetch_authentication(source_id, endpoint)
                                 end
           rescue => e
+            metrics&.record_error(:sources_api)
             logger.error_ext(operation, "Failed to fetch Authentication for Source #{source_id}: #{e.message}")
             nil
           end
@@ -32,6 +34,7 @@ module TopologicalInventory
           def application
             @application ||= sources_api.fetch_application(source_id)
           rescue => e
+            metrics&.record_error(:sources_api)
             logger.error_ext(operation, "Failed to fetch Application for Source #{source_id}: #{e.message}")
             nil
           end

data/lib/topological_inventory/providers/common/mixins/statuses.rb

@@ -0,0 +1,19 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Mixins
+        module Statuses
+          def operation_status
+            return @statuses if @statuses.present?
+
+            @statuses = {}
+            %i[success error skipped not_implemented].each do |status|
+              @statuses[status] = status.to_s
+            end
+            @statuses
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/async_worker.rb

@@ -1,4 +1,5 @@
 require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/mixins/statuses"
 require "topological_inventory/providers/common/operations/health_check"
 
 module TopologicalInventory
@@ -7,10 +8,12 @@ module TopologicalInventory
       module Operations
         class AsyncWorker
           include Logging
+          include TopologicalInventory::Providers::Common::Mixins::Statuses
 
-          def initialize(processor, queue = nil)
+          def initialize(processor, queue: nil, metrics: nil)
             @processor = processor
-            @queue = queue || Queue.new
+            @queue     = queue || Queue.new
+            @metrics   = metrics
           end
 
           def start
@@ -37,15 +40,17 @@ module TopologicalInventory
 
           private
 
-          attr_reader :thread, :queue, :processor
+          attr_reader :thread, :queue, :processor, :metrics
 
-          def process_message(msg)
-            processor.process!(msg)
+          def process_message(message)
+            result = processor.process!(message, metrics)
+            metrics&.record_operation(message.message, :status => result)
           rescue => err
-            model, method = msg.message.to_s.split(".")
+            model, method = message.message.to_s.split(".")
             logger.error("#{model}##{method}: async worker failure: #{err.cause}\n#{err}\n#{err.backtrace.join("\n")}")
+            metrics&.record_operation(message.message, :status => operation_status[:error])
           ensure
-            msg.ack
+            message.ack
             TopologicalInventory::Providers::Common::Operations::HealthCheck.touch_file
             logger.debug("Operations::AsyncWorker queue length: #{queue.length}") if queue.length >= 20 && queue.length % 5 == 0
           end

data/lib/topological_inventory/providers/common/operations/processor.rb

@@ -0,0 +1,77 @@
+require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/mixins/statuses"
+require "topological_inventory/providers/common/mixins/topology_api"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class Processor
+          include Logging
+          include Mixins::Statuses
+          include Mixins::TopologyApi
+
+          def self.process!(message, metrics)
+            new(message, metrics).process
+          end
+
+          def initialize(message, metrics)
+            self.message            = message
+            self.metrics            = metrics
+            self.model, self.method = message.message.split(".")
+
+            self.params   = message.payload["params"]
+            self.identity = message.payload["request_context"]
+          end
+
+          def process
+            logger.info(status_log_msg)
+            impl = operation_class&.new(params, identity, metrics)
+            if impl&.respond_to?(method)
+              with_time_measure do
+                result = impl.send(method)
+
+                logger.info(status_log_msg("Complete"))
+                result
+              end
+            else
+              logger.warn(status_log_msg("Not Implemented!"))
+              complete_task("not implemented") if params["task_id"]
+              operation_status[:not_implemented]
+            end
+          rescue StandardError, NotImplementedError => e
+            complete_task(e.message) if params["task_id"]
+            raise
+          end
+
+          private
+
+          attr_accessor :message, :identity, :model, :method, :metrics, :params
+
+          def operation_class
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          def with_time_measure
+            if metrics.present?
+              metrics.record_operation_time(message.message) { yield }
+            else
+              yield
+            end
+          end
+
+          def complete_task(msg, status = "error")
+            update_task(params["task_id"],
+                        :state   => "completed",
+                        :status  => status,
+                        :context => {:error => "#{model}##{method} - #{msg}"})
+          end
+
+          def status_log_msg(status = nil)
+            "Processing #{model}##{method} [#{params}]...#{status}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -1,7 +1,9 @@
 require "topological_inventory/providers/common/logging"
 require "active_support/core_ext/numeric/time"
 require "topological_inventory/providers/common/mixins/sources_api"
+require "topological_inventory/providers/common/mixins/statuses"
 require "topological_inventory/providers/common/mixins/x_rh_headers"
+require "topological_inventory/providers/common/messaging_client"
 
 module TopologicalInventory
   module Providers
@@ -11,8 +13,11 @@ module TopologicalInventory
           include Logging
           include Mixins::SourcesApi
           include Mixins::XRhHeaders
+          include Mixins::Statuses
 
           STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
+          EVENT_AVAILABILITY_STATUS            = "availability_status".freeze
+          SERVICE_NAME                         = "platform.sources.status".freeze
 
           ERROR_MESSAGES = {
             :authentication_not_found          => "Authentication not found in Sources API",
@@ -20,36 +25,41 @@ module TopologicalInventory
           }.freeze
 
           LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
-          AUTH_NOT_NECESSARY        = "n/a".freeze
+          AUTH_NOT_NECESSARY = "n/a".freeze
 
           attr_accessor :identity, :operation, :params, :request_context, :source_id, :account_number
 
-          def initialize(params = {}, request_context = nil)
-            self.operation       = 'Source'
-            self.params          = params
-            self.request_context = request_context
-            self.source_id       = params['source_id']
-
-            self.account_number  = params['external_tenant']
-            self.identity        = identity_by_account_number(account_number)
+          def initialize(params = {}, request_context = nil, metrics = nil)
+            self.metrics           = metrics
+            self.operation         = 'Source'
+            self.params            = params
+            self.request_context   = request_context
+            self.source_id         = params['source_id']
+            self.account_number    = params['external_tenant']
+            self.updates_via_kafka = ENV['UPDATE_SOURCES_VIA_API'].blank?
+            self.identity          = identity_by_account_number(account_number)
           end
 
           def availability_check
             self.operation += '#availability_check'
 
-            return if params_missing?
+            return operation_status[:error] if params_missing?
 
-            return if checked_recently?
+            return operation_status[:skipped] if checked_recently?
 
             status, error_message = connection_status
 
             update_source_and_subresources(status, error_message)
 
             logger.availability_check("Completed: Source #{source_id} is #{status}")
+
+            operation_status[:success]
           end
 
           private
 
+          attr_accessor :metrics, :updates_via_kafka
+
           def required_params
             %w[source_id]
           end
@@ -73,7 +83,7 @@ module TopologicalInventory
                                  application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
                                end
 
-            logger.availability_check("Skipping, last check at #{endpoint.last_checked_at || application.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
+            logger.availability_check("Skipping, last check at #{endpoint&.last_checked_at || application&.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
 
             checked_recently
           end
@@ -114,10 +124,15 @@ module TopologicalInventory
           def update_source_and_subresources(status, error_message = nil)
             logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
 
-            update_source(status)
-
-            update_endpoint(status, error_message) if endpoint
-            update_application(status) if application
+            if updates_via_kafka
+              update_source_by_kafka(status)
+              update_endpoint_by_kafka(status, error_message) if endpoint
+              update_application_by_kafka(status) if application
+            else
+              update_source(status)
+              update_endpoint(status, error_message) if endpoint
+              update_application(status) if application
+            end
           end
 
           def update_source(status)
@@ -128,9 +143,18 @@ module TopologicalInventory
 
             sources_api.update_source(source_id, source)
           rescue ::SourcesApiClient::ApiError => e
+            metrics&.record_error(:sources_api)
             logger.availability_check("Failed to update Source id:#{source_id} - #{e.message}", :error)
           end
 
+          def update_source_by_kafka(status)
+            availability_status_message(
+              :resource_type => "Source",
+              :resource_id   => source_id,
+              :status        => status
+            )
+          end
+
           def update_endpoint(status, error_message)
             if endpoint.nil?
               logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
@@ -146,19 +170,59 @@ module TopologicalInventory
 
             sources_api.update_endpoint(endpoint.id, endpoint_update)
           rescue ::SourcesApiClient::ApiError => e
+            metrics&.record_error(:sources_api)
             logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
           end
 
+          def update_endpoint_by_kafka(status, error_message)
+            if endpoint.nil?
+              logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
+              return
+            end
+
+            availability_status_message(
+              :resource_type => "Endpoint",
+              :resource_id   => endpoint.id,
+              :status        => status,
+              :error         => error_message
+            )
+          end
+
           def update_application(status)
-            application_update                   = ::SourcesApiClient::Application.new
-            application_update.last_checked_at   = check_time
-            application_update.last_available_at = check_time if status == STATUS_AVAILABLE
+            application_update                     = ::SourcesApiClient::Application.new
+            application_update.last_checked_at     = check_time
+            application_update.last_available_at   = check_time if status == STATUS_AVAILABLE
 
             sources_api.update_application(application.id, application_update)
           rescue ::SourcesApiClient::ApiError => e
+            metrics&.record_error(:sources_api)
             logger.availability_check("Failed to update Application id: #{application.id} - #{e.message}", :error)
           end
 
+          def update_application_by_kafka(status)
+            availability_status_message(
+              :resource_type => "Application",
+              :resource_id   => application.id,
+              :status        => status
+            )
+          end
+
+          def availability_status_message(payload)
+            messaging_client.publish_topic(
+              :service => SERVICE_NAME,
+              :event   => EVENT_AVAILABILITY_STATUS,
+              :payload => payload.to_json
+            )
+          rescue => err
+            logger.availability_check("Failed to update #{payload[:resource_type]} id: #{payload[:resource_id]} - #{err.message}", :error)
+          ensure
+            messaging_client&.close
+          end
+
+          def messaging_client
+            TopologicalInventory::Providers::Common::MessagingClient.default.client
+          end
+
           def check_time
             @check_time ||= Time.now.utc
           end