topological_inventory-providers-common 2.1.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: faf7ab98d098c6dfed7cac60aa40dca8e84a26721eac35ed8a487c84c57d4375
-  data.tar.gz: 122a119549a916b7d908c7655a94ed855af4ddb49220e05cedd11e89972abcd3
+  metadata.gz: 3292228e7688ff8aa1d33e40d3a50a6e30928cc4b8e950166e9d7d82df61ee53
+  data.tar.gz: a1f98d554c706ff3dc37ea9659ac88ddf9b7e7801958a68fbf6b1824e74f32d7
 SHA512:
-  metadata.gz: 90d69186763762d8d62c41282f27b926b6aee15fbd2b916d2f911eab519f586650e97e804ea07e3c2c1121b953a305f949912648e05ffe016ef27cd4273b0747
-  data.tar.gz: f8864ca924b2e5b494d3e0fe2403bcd0f1c03450fe9ce755fb6bf80c832e8a1dafa0c788a9998ae50c2e63eb414852bdeadd593ca2ebfd8389ae1858e6517129
+  metadata.gz: d44938aa30f1002deb30d15f61f6c89d80da94f6387e3985468ca76ac219d64ad3b57a91d7638480c14d32a17610545dae82abe3044a1328b2ef10ea6f5c9773
+  data.tar.gz: fb858ce78c5a5123e78fb55cb69fe01d7d682baa4c83661ed7ab3e665da95cab283992356a89cd73da1e1cd6580e5b18c8b2e480c70b2a8c9b9cf604b06dbefb

data/CHANGELOG.md

@@ -3,6 +3,22 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [3.0.0] - 2021-03-08
+Clowder migration #75 
+
+## [2.1.5] - 2021-01-07
+Include x-rh-id header in Kafka Availability-Status message #73
+
+## [2.1.4] - 2020-12-14
+Common Counter metrics init #71
+
+## [2.1.3] - 2020-12-14
+Update travis URL in README file #67
+Kafka availability_checks fix #68
+
+## [2.1.2] - 2020-11-24
+Custom Metrics for AsyncWorker #60
+Change error metric name to errors_total #64
 
 ## [2.1.1] - 2020-11-04
 MessagingClient and Source fix #61
@@ -71,7 +87,14 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v3.0.0...HEAD
+[3.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.5...v3.0.0
+[2.1.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.4...v2.1.5
+[2.1.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.4...v2.1.5
+[2.1.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.3...v2.1.4
+[2.1.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.2...v2.1.3
+[2.1.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.1...v2.1.2
+[2.1.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...v2.1.1
 [2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...v2.1.0
 [2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0
 [1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12

data/README.md

@@ -1,6 +1,6 @@
 # TopologicalInventory::Providers::Common
 
-[![Build Status](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common)
+[![Build Status](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common)
 [![Maintainability](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/maintainability)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/test_coverage)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/test_coverage)
 [![security](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master.svg)](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master)

data/lib/topological_inventory/providers/common/clowder_config.rb

@@ -0,0 +1,68 @@
+require 'clowder-common-ruby'
+require 'singleton'
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class ClowderConfig
+        include Singleton
+
+        def self.clowder_enabled?
+          ::ClowderCommonRuby::Config.clowder_enabled?
+        end
+
+        def self.instance
+          @instance ||= {}.tap do |options|
+            if clowder_enabled?
+              config                        = ::ClowderCommonRuby::Config.load
+              options["awsAccessKeyId"]     = config.logging.cloudwatch.accessKeyId
+              options["awsRegion"]          = config.logging.cloudwatch.region
+              options["awsSecretAccessKey"] = config.logging.cloudwatch.secretAccessKey
+              broker                        = config.kafka.brokers.first
+              options["kafkaHost"]          = broker.hostname
+              options["kafkaPort"]          = broker.port
+
+              options["kafkaTopics"] = {}.tap do |topics|
+                config.kafka.topics.each do |topic|
+                  topics[topic.requestedName.to_s] = topic.name.to_s
+                end
+              end
+              options["logGroup"]    = config.logging.cloudwatch.logGroup
+              options["metricsPort"] = config.metricsPort
+              options["metricsPath"] = config.metricsPath # not supported by PrometheusExporter
+            else
+              options["awsAccessKeyId"]     = ENV['CW_AWS_ACCESS_KEY_ID']
+              options["awsRegion"]          = 'us-east-1'
+              options["awsSecretAccessKey"] = ENV['CW_AWS_SECRET_ACCESS_KEY']
+              options["kafkaBrokers"]       = ["#{ENV['QUEUE_HOST']}:#{ENV['QUEUE_PORT']}"]
+              options["kafkaHost"]          = ENV['QUEUE_HOST'] || 'localhost'
+              options["kafkaPort"]          = (ENV['QUEUE_PORT'] || '9092').to_i
+              options["kafkaTopics"]        = {}
+              options["logGroup"]           = 'platform-dev'
+              options["metricsPort"]        = (ENV['METRICS_PORT'] || 9394).to_i
+            end
+          end
+        end
+
+        def self.fill_args_operations(args)
+          args[:metrics_port] = instance['metricsPort']
+          args[:queue_host]   = instance['kafkaHost']
+          args[:queue_port]   = instance['kafkaPort']
+          args
+        end
+
+        def self.kafka_topic(name)
+          instance["kafkaTopics"][name] || name
+        end
+      end
+    end
+  end
+end
+
+# ManageIQ Message Client depends on these variables
+ENV["QUEUE_HOST"] = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaHost"]
+ENV["QUEUE_PORT"] = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaPort"].to_s
+
+# ManageIQ Logger depends on these variables
+ENV['CW_AWS_ACCESS_KEY_ID']     = TopologicalInventory::Providers::Common::ClowderConfig.instance["awsAccessKeyId"]
+ENV['CW_AWS_SECRET_ACCESS_KEY'] = TopologicalInventory::Providers::Common::ClowderConfig.instance["awsSecretAccessKey"]

data/lib/topological_inventory/providers/common/messaging_client.rb

@@ -1,5 +1,6 @@
 require "more_core_extensions/core_ext/module/cache_with_timeout"
 require "manageiq-messaging"
+require "topological_inventory/providers/common/clowder_config"
 
 module TopologicalInventory
   module Providers
@@ -11,8 +12,8 @@ module TopologicalInventory
         attr_accessor :queue_port
 
         def initialize
-          self.queue_host = ENV['QUEUE_HOST'] || 'localhost'
-          self.queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
+          self.queue_host = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaHost"]
+          self.queue_port = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaPort"].to_i
         end
 
         def self.default

data/lib/topological_inventory/providers/common/metrics.rb

@@ -3,18 +3,25 @@ require "prometheus_exporter"
 require "prometheus_exporter/server"
 require "prometheus_exporter/client"
 require "prometheus_exporter/instrumentation"
+require "topological_inventory/providers/common/mixins/statuses"
 
 module TopologicalInventory
   module Providers
     module Common
       class Metrics
+        include TopologicalInventory::Providers::Common::Mixins::Statuses
+
         ERROR_COUNTER_MESSAGE = "total number of errors".freeze
+        ERROR_TYPES = %i[general].freeze
+        OPERATIONS  = %w[].freeze
 
         def initialize(port = 9394)
           return if port == 0
 
           configure_server(port)
           configure_metrics
+
+          init_counters
         end
 
         def stop_server
@@ -25,6 +32,10 @@ module TopologicalInventory
           @error_counter&.observe(1, :type => type.to_s)
         end
 
+        def record_refresh_timing(labels = {}, &block)
+          record_time(@refresh_timer, labels, &block)
+        end
+
         def record_operation(name, labels = {})
           @status_counter&.observe(1, (labels || {}).merge(:name => name))
         end
@@ -55,6 +66,19 @@ module TopologicalInventory
 
         private
 
+        # Set all values to 0 (otherwise the counter is undefined)
+        def init_counters
+          self.class::ERROR_TYPES.each do |err_type|
+            @error_counter&.observe(0, :type => err_type)
+          end
+
+          self.class::OPERATIONS.each do |op|
+            operation_status.each_key do |status|
+              @status_counter&.observe(0, :name => op, :status => status.to_s)
+            end
+          end
+        end
+
         def configure_server(port)
           @server = PrometheusExporter::Server::WebServer.new(:port => port)
           @server.start
@@ -67,10 +91,11 @@ module TopologicalInventory
           PrometheusExporter::Metric::Base.default_prefix = default_prefix
 
           @duration_seconds = PrometheusExporter::Metric::Histogram.new('duration_seconds', 'Duration of processed operation')
-          @error_counter = PrometheusExporter::Metric::Counter.new("error", ERROR_COUNTER_MESSAGE)
+          @refresh_timer = PrometheusExporter::Metric::Histogram.new('refresh_time', 'Duration of full refresh')
+          @error_counter = PrometheusExporter::Metric::Counter.new('errors_total', ERROR_COUNTER_MESSAGE)
           @status_counter = PrometheusExporter::Metric::Counter.new('status_counter', 'number of processed operations')
 
-          [@duration_seconds, @error_counter, @status_counter].each do |metric|
+          [@duration_seconds, @refresh_timer, @error_counter, @status_counter].each do |metric|
             @server.collector.register_metric(metric)
           end
         end

data/lib/topological_inventory/providers/common/operations/async_worker.rb

@@ -1,4 +1,5 @@
 require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/mixins/statuses"
 require "topological_inventory/providers/common/operations/health_check"
 
 module TopologicalInventory
@@ -7,10 +8,12 @@ module TopologicalInventory
       module Operations
         class AsyncWorker
           include Logging
+          include TopologicalInventory::Providers::Common::Mixins::Statuses
 
-          def initialize(processor, queue = nil)
+          def initialize(processor, queue: nil, metrics: nil)
             @processor = processor
-            @queue = queue || Queue.new
+            @queue     = queue || Queue.new
+            @metrics   = metrics
           end
 
           def start
@@ -37,15 +40,17 @@ module TopologicalInventory
 
           private
 
-          attr_reader :thread, :queue, :processor
+          attr_reader :thread, :queue, :processor, :metrics
 
-          def process_message(msg)
-            processor.process!(msg)
+          def process_message(message)
+            result = processor.process!(message, metrics)
+            metrics&.record_operation(message.message, :status => result)
           rescue => err
-            model, method = msg.message.to_s.split(".")
+            model, method = message.message.to_s.split(".")
             logger.error("#{model}##{method}: async worker failure: #{err.cause}\n#{err}\n#{err.backtrace.join("\n")}")
+            metrics&.record_operation(message.message, :status => operation_status[:error])
           ensure
-            msg.ack
+            message.ack
             TopologicalInventory::Providers::Common::Operations::HealthCheck.touch_file
             logger.debug("Operations::AsyncWorker queue length: #{queue.length}") if queue.length >= 20 && queue.length % 5 == 0
           end

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -17,7 +17,7 @@ module TopologicalInventory
 
           STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
           EVENT_AVAILABILITY_STATUS            = "availability_status".freeze
-          SERVICE_NAME                         = "platform.sources.status".freeze
+          KAFKA_TOPIC_NAME                     = "platform.sources.status".freeze
 
           ERROR_MESSAGES = {
             :authentication_not_found          => "Authentication not found in Sources API",
@@ -30,7 +30,7 @@ module TopologicalInventory
           attr_accessor :identity, :operation, :params, :request_context, :source_id, :account_number
 
           def initialize(params = {}, request_context = nil, metrics = nil)
-            self.metrics         = metrics
+            self.metrics           = metrics
             self.operation         = 'Source'
             self.params            = params
             self.request_context   = request_context
@@ -208,10 +208,11 @@ module TopologicalInventory
           end
 
           def availability_status_message(payload)
-            messaging_client.publish_message(
-              :service => SERVICE_NAME,
-              :message => EVENT_AVAILABILITY_STATUS,
-              :payload => payload.to_json
+            messaging_client.publish_topic(
+              :service => TopologicalInventory::Providers::Common::ClowderConfig.kafka_topic(KAFKA_TOPIC_NAME),
+              :event   => EVENT_AVAILABILITY_STATUS,
+              :payload => payload.to_json,
+              :headers => identity
             )
           rescue => err
             logger.availability_check("Failed to update #{payload[:resource_type]} id: #{payload[:resource_id]} - #{err.message}", :error)

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "2.1.1".freeze
+      VERSION = "3.0.0".freeze
     end
   end
 end

data/spec/support/shared/availability_check.rb

@@ -38,12 +38,15 @@ RSpec.shared_examples "availability_check" do
 
   def kafka_message(resource_type, resource_id, status, error_message = nil)
     res = {
-      :service => described_class::SERVICE_NAME,
-      :message => described_class::EVENT_AVAILABILITY_STATUS,
+      :service => described_class::KAFKA_TOPIC_NAME,
+      :event   => described_class::EVENT_AVAILABILITY_STATUS,
       :payload => {
         :resource_type => resource_type,
         :resource_id   => resource_id,
         :status        => status
+      },
+      :headers => {
+        "x-rh-identity" => "eyJpZGVudGl0eSI6eyJhY2NvdW50X251bWJlciI6IjExMDAxIiwidXNlciI6eyJpc19vcmdfYWRtaW4iOnRydWV9fX0="
       }
     }
     res[:payload][:error] = error_message if error_message
@@ -67,15 +70,15 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_available, ''])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_available)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_available, '')
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Application", application_id, status_available)
         )
 
@@ -89,15 +92,15 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_unavailable, error_message)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Application", application_id, status_unavailable)
         )
 
@@ -110,7 +113,7 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
@@ -126,11 +129,11 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_unavailable, error_message)
         )
 
@@ -258,11 +261,11 @@ RSpec.shared_examples "availability_check" do
 
           expect(subject).to receive(:connection_status).and_return([status_available, ''])
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Source", source_id, status_available)
           )
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Application", application_id, status_available)
           )
 
@@ -307,11 +310,11 @@ RSpec.shared_examples "availability_check" do
 
           expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Source", source_id, status_unavailable)
           )
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Application", application_id, status_unavailable)
           )
 

data/spec/topological_inventory/providers/common/operations/async_worker_spec.rb

@@ -3,33 +3,42 @@ require "topological_inventory/providers/common/operations/async_worker"
 describe TopologicalInventory::Providers::Common::Operations::AsyncWorker do
   let(:queue) { double }
   let(:impl) { double }
+  let(:metrics) { double('metrics') }
   let(:msg) { double }
-  subject { described_class.new(impl, queue) }
+  let(:operation) { "Source.availability_check" }
+
+  subject { described_class.new(impl, :queue => queue, :metrics => metrics) }
 
   before do
     allow(queue).to receive(:length).and_return(0)
-    allow(msg).to receive(:message).and_return("Source.availability_check")
+    allow(msg).to receive(:message).and_return(operation)
   end
 
   context "when the message is able to be processed" do
+    let(:result) { subject.operation_status[:success] }
     before do
-      allow(impl).to receive(:process!).with(msg)
+      allow(impl).to receive(:process!).with(msg, metrics).and_return(result)
       allow(msg).to receive(:ack)
     end
 
     it "drains messages that are added to the queue" do
-      expect(impl).to receive(:process!).with(msg).once
+      expect(impl).to receive(:process!).with(msg, metrics).once
+      expect(metrics).to receive(:record_operation).with(operation, :status => result)
       subject.send(:process_message, msg)
     end
   end
 
   context "when the message results in an error" do
+    let(:result) { subject.operation_status[:error] }
     before do
-      allow(impl).to receive(:process!).with(msg).and_raise(StandardError.new("boom!"))
+      allow(impl).to receive(:process!).with(msg, metrics).and_raise(StandardError.new("boom!"))
     end
 
     it "ack's the message on failure" do
+      allow(subject).to receive(:logger).and_return(double.as_null_object)
+      
       expect(msg).to receive(:ack).once
+      expect(metrics).to receive(:record_operation).with(operation, :status => result)
       subject.send(:process_message, msg)
     end
   end

data/topological_inventory-providers-common.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'activesupport', '~> 5.2.4.3'
+  spec.add_runtime_dependency 'clowder-common-ruby', '~> 0.2.1'
   spec.add_runtime_dependency 'config', '~> 1.7', '>= 1.7.2'
   spec.add_runtime_dependency 'json', '~> 2.3'
   spec.add_runtime_dependency 'manageiq-loggers', '>= 0.4.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-providers-common
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-04 00:00:00.000000000 Z
+date: 2021-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 5.2.4.3
+- !ruby/object:Gem::Dependency
+  name: clowder-common-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: config
   requirement: !ruby/object:Gem::Requirement
@@ -304,6 +318,7 @@ files:
 - bin/console
 - bin/setup
 - lib/topological_inventory/providers/common.rb
+- lib/topological_inventory/providers/common/clowder_config.rb
 - lib/topological_inventory/providers/common/collector.rb
 - lib/topological_inventory/providers/common/collector/inventory_collection_storage.rb
 - lib/topological_inventory/providers/common/collector/inventory_collection_wrapper.rb