topological_inventory-providers-common 2.1.2 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 661072299fccb0117c3dcbe1fba36e4836a82ac1a6d98abbaab1808ea437e6d7
-  data.tar.gz: a1a45e527dc14cb4b09004d2df98f075818fb35d616d84d954839c289dac5c8a
+  metadata.gz: b0d5e54970be70204d7ac119fdd23e0a68bbc01a3e1d3c64520ce3fc6129cfe7
+  data.tar.gz: 624c68af0b57940bc5436099f1a4238eb5405a9e9b87706911278b5c3fcf44e0
 SHA512:
-  metadata.gz: ba101d97cca75814a073eb70f8528e3debb0f4270fb4cfb98cc9db55f53ff790efcd99380158e8c808b317603a9ce1a7a30d13f791e03741b699e06f0556965e
-  data.tar.gz: 3e4340ad6743693f104d0c847995bb866860ef2911c57865ce042389cb3872badf05f2b46119664650214285f509495f656dc5cf82f77cec9cf5e2a253a0f2cc
+  metadata.gz: ac55e0263630b78533d49b061d972beab29e249911f785f536a8acbb3cada1d88e3935849d089791f9be001b87a0f3100b4e44bc3595527d169107ef14a5e9b0
+  data.tar.gz: df3b8288f73292d647a77629d5f5ae78a21dad4eeb206e6b1e4e2d077ffd475c60c705f0365311b374fd88544a81943403484089f829aa3611919e95d2ee734f

data/CHANGELOG.md

@@ -3,6 +3,21 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [3.0.1] - 2021-04-30
+More generic activesupport dependency #77
+
+## [3.0.0] - 2021-03-08
+Clowder migration #75 
+
+## [2.1.5] - 2021-01-07
+Include x-rh-id header in Kafka Availability-Status message #73
+
+## [2.1.4] - 2020-12-14
+Common Counter metrics init #71
+
+## [2.1.3] - 2020-12-14
+Update travis URL in README file #67
+Kafka availability_checks fix #68
 
 ## [2.1.2] - 2020-11-24
 Custom Metrics for AsyncWorker #60
@@ -75,11 +90,17 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.2...HEAD
-[2.1.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.1...v2.1.2
-[2.1.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...v2.1.1
-[2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...v2.1.0
-[2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v3.0.1.freeze...HEAD
+[3.0.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v3.0.0.freeze...v3.0.1.freeze
+[3.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.5.freeze...v3.0.0.freeze
+[2.1.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.4.freeze...v2.1.5.freeze
+[2.1.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.4.freeze...v2.1.5.freeze
+[2.1.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.3.freeze...v2.1.4.freeze
+[2.1.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.2.freeze...v2.1.3.freeze
+[2.1.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.1.freeze...v2.1.2.freeze
+[2.1.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0.freeze...v2.1.1.freeze
+[2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0.freeze...v2.1.0.freeze
+[2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0.freeze
 [1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12
 [1.0.11]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.10...v1.0.11
 [1.0.10]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.9...v1.0.10

data/README.md

@@ -1,6 +1,6 @@
 # TopologicalInventory::Providers::Common
 
-[![Build Status](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common)
+[![Build Status](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common)
 [![Maintainability](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/maintainability)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/test_coverage)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/test_coverage)
 [![security](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master.svg)](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master)

data/lib/topological_inventory/providers/common/clowder_config.rb

@@ -0,0 +1,68 @@
+require 'clowder-common-ruby'
+require 'singleton'
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class ClowderConfig
+        include Singleton
+
+        def self.clowder_enabled?
+          ::ClowderCommonRuby::Config.clowder_enabled?
+        end
+
+        def self.instance
+          @instance ||= {}.tap do |options|
+            if clowder_enabled?
+              config                        = ::ClowderCommonRuby::Config.load
+              options["awsAccessKeyId"]     = config.logging.cloudwatch.accessKeyId
+              options["awsRegion"]          = config.logging.cloudwatch.region
+              options["awsSecretAccessKey"] = config.logging.cloudwatch.secretAccessKey
+              broker                        = config.kafka.brokers.first
+              options["kafkaHost"]          = broker.hostname
+              options["kafkaPort"]          = broker.port
+
+              options["kafkaTopics"] = {}.tap do |topics|
+                config.kafka.topics.each do |topic|
+                  topics[topic.requestedName.to_s] = topic.name.to_s
+                end
+              end
+              options["logGroup"]    = config.logging.cloudwatch.logGroup
+              options["metricsPort"] = config.metricsPort
+              options["metricsPath"] = config.metricsPath # not supported by PrometheusExporter
+            else
+              options["awsAccessKeyId"]     = ENV['CW_AWS_ACCESS_KEY_ID']
+              options["awsRegion"]          = 'us-east-1'
+              options["awsSecretAccessKey"] = ENV['CW_AWS_SECRET_ACCESS_KEY']
+              options["kafkaBrokers"]       = ["#{ENV['QUEUE_HOST']}:#{ENV['QUEUE_PORT']}"]
+              options["kafkaHost"]          = ENV['QUEUE_HOST'] || 'localhost'
+              options["kafkaPort"]          = (ENV['QUEUE_PORT'] || '9092').to_i
+              options["kafkaTopics"]        = {}
+              options["logGroup"]           = 'platform-dev'
+              options["metricsPort"]        = (ENV['METRICS_PORT'] || 9394).to_i
+            end
+          end
+        end
+
+        def self.fill_args_operations(args)
+          args[:metrics_port] = instance['metricsPort']
+          args[:queue_host]   = instance['kafkaHost']
+          args[:queue_port]   = instance['kafkaPort']
+          args
+        end
+
+        def self.kafka_topic(name)
+          instance["kafkaTopics"][name] || name
+        end
+      end
+    end
+  end
+end
+
+# ManageIQ Message Client depends on these variables
+ENV["QUEUE_HOST"] = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaHost"]
+ENV["QUEUE_PORT"] = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaPort"].to_s
+
+# ManageIQ Logger depends on these variables
+ENV['CW_AWS_ACCESS_KEY_ID']     = TopologicalInventory::Providers::Common::ClowderConfig.instance["awsAccessKeyId"]
+ENV['CW_AWS_SECRET_ACCESS_KEY'] = TopologicalInventory::Providers::Common::ClowderConfig.instance["awsSecretAccessKey"]

data/lib/topological_inventory/providers/common/messaging_client.rb

@@ -1,5 +1,6 @@
 require "more_core_extensions/core_ext/module/cache_with_timeout"
 require "manageiq-messaging"
+require "topological_inventory/providers/common/clowder_config"
 
 module TopologicalInventory
   module Providers
@@ -11,8 +12,8 @@ module TopologicalInventory
         attr_accessor :queue_port
 
         def initialize
-          self.queue_host = ENV['QUEUE_HOST'] || 'localhost'
-          self.queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
+          self.queue_host = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaHost"]
+          self.queue_port = TopologicalInventory::Providers::Common::ClowderConfig.instance["kafkaPort"].to_i
         end
 
         def self.default

data/lib/topological_inventory/providers/common/metrics.rb

@@ -3,18 +3,25 @@ require "prometheus_exporter"
 require "prometheus_exporter/server"
 require "prometheus_exporter/client"
 require "prometheus_exporter/instrumentation"
+require "topological_inventory/providers/common/mixins/statuses"
 
 module TopologicalInventory
   module Providers
     module Common
       class Metrics
+        include TopologicalInventory::Providers::Common::Mixins::Statuses
+
         ERROR_COUNTER_MESSAGE = "total number of errors".freeze
+        ERROR_TYPES = %i[general].freeze
+        OPERATIONS  = %w[].freeze
 
         def initialize(port = 9394)
           return if port == 0
 
           configure_server(port)
           configure_metrics
+
+          init_counters
         end
 
         def stop_server
@@ -25,6 +32,10 @@ module TopologicalInventory
           @error_counter&.observe(1, :type => type.to_s)
         end
 
+        def record_refresh_timing(labels = {}, &block)
+          record_time(@refresh_timer, labels, &block)
+        end
+
         def record_operation(name, labels = {})
           @status_counter&.observe(1, (labels || {}).merge(:name => name))
         end
@@ -55,6 +66,19 @@ module TopologicalInventory
 
         private
 
+        # Set all values to 0 (otherwise the counter is undefined)
+        def init_counters
+          self.class::ERROR_TYPES.each do |err_type|
+            @error_counter&.observe(0, :type => err_type)
+          end
+
+          self.class::OPERATIONS.each do |op|
+            operation_status.each_key do |status|
+              @status_counter&.observe(0, :name => op, :status => status.to_s)
+            end
+          end
+        end
+
         def configure_server(port)
           @server = PrometheusExporter::Server::WebServer.new(:port => port)
           @server.start
@@ -67,10 +91,11 @@ module TopologicalInventory
           PrometheusExporter::Metric::Base.default_prefix = default_prefix
 
           @duration_seconds = PrometheusExporter::Metric::Histogram.new('duration_seconds', 'Duration of processed operation')
+          @refresh_timer = PrometheusExporter::Metric::Histogram.new('refresh_time', 'Duration of full refresh')
           @error_counter = PrometheusExporter::Metric::Counter.new('errors_total', ERROR_COUNTER_MESSAGE)
           @status_counter = PrometheusExporter::Metric::Counter.new('status_counter', 'number of processed operations')
 
-          [@duration_seconds, @error_counter, @status_counter].each do |metric|
+          [@duration_seconds, @refresh_timer, @error_counter, @status_counter].each do |metric|
             @server.collector.register_metric(metric)
           end
         end

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -17,7 +17,7 @@ module TopologicalInventory
 
           STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
           EVENT_AVAILABILITY_STATUS            = "availability_status".freeze
-          SERVICE_NAME                         = "platform.sources.status".freeze
+          KAFKA_TOPIC_NAME                     = "platform.sources.status".freeze
 
           ERROR_MESSAGES = {
             :authentication_not_found          => "Authentication not found in Sources API",
@@ -208,10 +208,11 @@ module TopologicalInventory
           end
 
           def availability_status_message(payload)
-            messaging_client.publish_message(
-              :service => SERVICE_NAME,
-              :message => EVENT_AVAILABILITY_STATUS,
-              :payload => payload.to_json
+            messaging_client.publish_topic(
+              :service => TopologicalInventory::Providers::Common::ClowderConfig.kafka_topic(KAFKA_TOPIC_NAME),
+              :event   => EVENT_AVAILABILITY_STATUS,
+              :payload => payload.to_json,
+              :headers => identity
             )
           rescue => err
             logger.availability_check("Failed to update #{payload[:resource_type]} id: #{payload[:resource_id]} - #{err.message}", :error)

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "2.1.2".freeze
+      VERSION = "3.0.1".freeze
     end
   end
 end

data/spec/support/shared/availability_check.rb

@@ -38,12 +38,15 @@ RSpec.shared_examples "availability_check" do
 
   def kafka_message(resource_type, resource_id, status, error_message = nil)
     res = {
-      :service => described_class::SERVICE_NAME,
-      :message => described_class::EVENT_AVAILABILITY_STATUS,
+      :service => described_class::KAFKA_TOPIC_NAME,
+      :event   => described_class::EVENT_AVAILABILITY_STATUS,
       :payload => {
         :resource_type => resource_type,
         :resource_id   => resource_id,
         :status        => status
+      },
+      :headers => {
+        "x-rh-identity" => "eyJpZGVudGl0eSI6eyJhY2NvdW50X251bWJlciI6IjExMDAxIiwidXNlciI6eyJpc19vcmdfYWRtaW4iOnRydWV9fX0="
       }
     }
     res[:payload][:error] = error_message if error_message
@@ -67,15 +70,15 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_available, ''])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_available)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_available, '')
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Application", application_id, status_available)
         )
 
@@ -89,15 +92,15 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_unavailable, error_message)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Application", application_id, status_unavailable)
         )
 
@@ -110,7 +113,7 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
@@ -126,11 +129,11 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_unavailable, error_message)
         )
 
@@ -258,11 +261,11 @@ RSpec.shared_examples "availability_check" do
 
           expect(subject).to receive(:connection_status).and_return([status_available, ''])
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Source", source_id, status_available)
           )
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Application", application_id, status_available)
           )
 
@@ -307,11 +310,11 @@ RSpec.shared_examples "availability_check" do
 
           expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Source", source_id, status_unavailable)
           )
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Application", application_id, status_unavailable)
           )
 

data/topological_inventory-providers-common.gemspec

@@ -23,7 +23,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency 'activesupport', '~> 5.2.4.3'
+  spec.add_runtime_dependency 'activesupport', '~> 5.2', '>= 5.2.4.3'
+  spec.add_runtime_dependency 'clowder-common-ruby', '~> 0.2.1'
   spec.add_runtime_dependency 'config', '~> 1.7', '>= 1.7.2'
   spec.add_runtime_dependency 'json', '~> 2.3'
   spec.add_runtime_dependency 'manageiq-loggers', '>= 0.4.2'

metadata

@@ -1,20 +1,23 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-providers-common
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 3.0.1
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-24 00:00:00.000000000 Z
+date: 2021-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.4.3
   type: :runtime
@@ -22,8 +25,25 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.4.3
+- !ruby/object:Gem::Dependency
+  name: clowder-common-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: config
   requirement: !ruby/object:Gem::Requirement
@@ -304,6 +324,7 @@ files:
 - bin/console
 - bin/setup
 - lib/topological_inventory/providers/common.rb
+- lib/topological_inventory/providers/common/clowder_config.rb
 - lib/topological_inventory/providers/common/collector.rb
 - lib/topological_inventory/providers/common/collector/inventory_collection_storage.rb
 - lib/topological_inventory/providers/common/collector/inventory_collection_wrapper.rb
@@ -358,7 +379,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Common classes for topological-inventory collectors/operations