topological_inventory-providers-common 2.1.0 → 2.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aad209b97efea9bb19ce488782adb1dabfa8d650c12f8637e2b348ed4ec95a15
-  data.tar.gz: 69b9bbed6a2d725990206ab3bfc7b13ec25b10f7e0c1a979b48c61b67e53fd0d
+  metadata.gz: 75348e5abd6addefc5c90886fe1b31fc8405239998e030f6f740ebd2b034d726
+  data.tar.gz: 7b6c51b8299d5209587253954d41e8c7898c0f15fcab3fcdbcdfd631d8311aff
 SHA512:
-  metadata.gz: 45722b70f009a764d8cd804424157cc98b1a225b72326b8b8971fecd093fa6260446777db76bc3502e487d27dc82b2b4325959302f4a1abe13fbbf8c015c1d54
-  data.tar.gz: b1a6de8e2eb93b2b26d4b609f17a1a518cf4b318dbcf117c6225fe297bb3ec97f870b944e151786977ad90f42c16746eeeadf6a3d113b648921e278cb831aadd
+  metadata.gz: d2e5ed6d7135c0ace48bcd109c8b757c8b3f93633053bb8878cbcdc69ea13d69b18461db763ca97ad140d729078d15c1b05b59462fdfcc28a9b487fac0a73d24
+  data.tar.gz: 6967cf0f588c7fc477e308f1cc3d9ad9be1487cb8932506fb5cbd7604d372486a33eb3d503d77a21e7eb3a08e5a02dfeba056f1d04336b44b3037cd28a2f81d0

data/CHANGELOG.md

@@ -3,8 +3,25 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [2.1.5] - 2021-01-07
+Include x-rh-id header in Kafka Availability-Status message #73
 
-## [2.1.0]
+## [2.1.4] - 2020-12-14
+Common Counter metrics init #71
+
+## [2.1.3] - 2020-12-14
+Update travis URL in README file #67
+Kafka availability_checks fix #68
+
+## [2.1.2] - 2020-11-24
+Custom Metrics for AsyncWorker #60
+Change error metric name to errors_total #64
+
+## [2.1.1] - 2020-11-04
+MessagingClient and Source fix #61
+Fix availability_check in app check #62
+
+## [2.1.0] - 2020-11-02
 Add Availability checks for sources via Kafka #54
 Common Metrics exporter #57
 Rubocop rules from insights-api-common + yamllint #59
@@ -12,7 +29,7 @@ Rubocop rules from insights-api-common + yamllint #59
 ## [2.0.0] - 2020-10-20
 Operations/API clients refactoring
 
-## [1.0.12] - 2020-10-01
+## [1.0.12] - 2020-10-05
 Add Operations Async Worker class #55
 
 ## [1.0.11] - 2020-09-04
@@ -67,7 +84,12 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.5...HEAD
+[2.1.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.4...v2.1.5
+[2.1.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.3...v2.1.4
+[2.1.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.2...v2.1.3
+[2.1.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.1...v2.1.2
+[2.1.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.1.0...v2.1.1
 [2.1.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v2.0.0...v2.1.0
 [2.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.12...v2.0.0
 [1.0.12]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.11...v1.0.12

data/README.md

@@ -1,6 +1,6 @@
 # TopologicalInventory::Providers::Common
 
-[![Build Status](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.org/RedHatInsights/topological_inventory-providers-common)
+[![Build Status](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common.svg?branch=master)](https://travis-ci.com/RedHatInsights/topological_inventory-providers-common)
 [![Maintainability](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/maintainability)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/df747492b802bfea3c83/test_coverage)](https://codeclimate.com/github/RedHatInsights/topological_inventory-providers-common/test_coverage)
 [![security](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master.svg)](https://hakiri.io/github/RedHatInsights/topological_inventory-providers-common/master)

data/lib/topological_inventory/providers/common/messaging_client.rb

@@ -11,8 +11,8 @@ module TopologicalInventory
         attr_accessor :queue_port
 
         def initialize
-          @queue_host = ENV['QUEUE_HOST'] || 'localhost'
-          @queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
+          self.queue_host = ENV['QUEUE_HOST'] || 'localhost'
+          self.queue_port = (ENV['QUEUE_PORT'] || 9092).to_i
         end
 
         def self.default
@@ -28,7 +28,7 @@ module TopologicalInventory
         end
 
         cache_with_timeout(:client) do
-          ManageIQ::Messaging::Client.open(:protocol => :Kafka, :host => @queue_host, :port => @queue_port)
+          ManageIQ::Messaging::Client.open(:protocol => :Kafka, :host => default.queue_host, :port => default.queue_port)
         end
 
         def client

data/lib/topological_inventory/providers/common/metrics.rb

@@ -3,18 +3,25 @@ require "prometheus_exporter"
 require "prometheus_exporter/server"
 require "prometheus_exporter/client"
 require "prometheus_exporter/instrumentation"
+require "topological_inventory/providers/common/mixins/statuses"
 
 module TopologicalInventory
   module Providers
     module Common
       class Metrics
+        include TopologicalInventory::Providers::Common::Mixins::Statuses
+
         ERROR_COUNTER_MESSAGE = "total number of errors".freeze
+        ERROR_TYPES = %i[general].freeze
+        OPERATIONS  = %w[].freeze
 
         def initialize(port = 9394)
           return if port == 0
 
           configure_server(port)
           configure_metrics
+
+          init_counters
         end
 
         def stop_server
@@ -25,6 +32,10 @@ module TopologicalInventory
           @error_counter&.observe(1, :type => type.to_s)
         end
 
+        def record_refresh_timing(labels = {}, &block)
+          record_time(@refresh_timer, labels, &block)
+        end
+
         def record_operation(name, labels = {})
           @status_counter&.observe(1, (labels || {}).merge(:name => name))
         end
@@ -55,6 +66,19 @@ module TopologicalInventory
 
         private
 
+        # Set all values to 0 (otherwise the counter is undefined)
+        def init_counters
+          self.class::ERROR_TYPES.each do |err_type|
+            @error_counter&.observe(0, :type => err_type)
+          end
+
+          self.class::OPERATIONS.each do |op|
+            operation_status.each_key do |status|
+              @status_counter&.observe(0, :name => op, :status => status.to_s)
+            end
+          end
+        end
+
         def configure_server(port)
           @server = PrometheusExporter::Server::WebServer.new(:port => port)
           @server.start
@@ -67,10 +91,11 @@ module TopologicalInventory
           PrometheusExporter::Metric::Base.default_prefix = default_prefix
 
           @duration_seconds = PrometheusExporter::Metric::Histogram.new('duration_seconds', 'Duration of processed operation')
-          @error_counter = PrometheusExporter::Metric::Counter.new("error", ERROR_COUNTER_MESSAGE)
+          @refresh_timer = PrometheusExporter::Metric::Histogram.new('refresh_time', 'Duration of full refresh')
+          @error_counter = PrometheusExporter::Metric::Counter.new('errors_total', ERROR_COUNTER_MESSAGE)
           @status_counter = PrometheusExporter::Metric::Counter.new('status_counter', 'number of processed operations')
 
-          [@duration_seconds, @error_counter, @status_counter].each do |metric|
+          [@duration_seconds, @refresh_timer, @error_counter, @status_counter].each do |metric|
             @server.collector.register_metric(metric)
           end
         end

data/lib/topological_inventory/providers/common/operations/async_worker.rb

@@ -1,4 +1,5 @@
 require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/mixins/statuses"
 require "topological_inventory/providers/common/operations/health_check"
 
 module TopologicalInventory
@@ -7,10 +8,12 @@ module TopologicalInventory
       module Operations
         class AsyncWorker
           include Logging
+          include TopologicalInventory::Providers::Common::Mixins::Statuses
 
-          def initialize(processor, queue = nil)
+          def initialize(processor, queue: nil, metrics: nil)
             @processor = processor
-            @queue = queue || Queue.new
+            @queue     = queue || Queue.new
+            @metrics   = metrics
           end
 
           def start
@@ -37,15 +40,17 @@ module TopologicalInventory
 
           private
 
-          attr_reader :thread, :queue, :processor
+          attr_reader :thread, :queue, :processor, :metrics
 
-          def process_message(msg)
-            processor.process!(msg)
+          def process_message(message)
+            result = processor.process!(message, metrics)
+            metrics&.record_operation(message.message, :status => result)
           rescue => err
-            model, method = msg.message.to_s.split(".")
+            model, method = message.message.to_s.split(".")
             logger.error("#{model}##{method}: async worker failure: #{err.cause}\n#{err}\n#{err.backtrace.join("\n")}")
+            metrics&.record_operation(message.message, :status => operation_status[:error])
           ensure
-            msg.ack
+            message.ack
             TopologicalInventory::Providers::Common::Operations::HealthCheck.touch_file
             logger.debug("Operations::AsyncWorker queue length: #{queue.length}") if queue.length >= 20 && queue.length % 5 == 0
           end

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -30,7 +30,7 @@ module TopologicalInventory
           attr_accessor :identity, :operation, :params, :request_context, :source_id, :account_number
 
           def initialize(params = {}, request_context = nil, metrics = nil)
-            self.metrics         = metrics
+            self.metrics           = metrics
             self.operation         = 'Source'
             self.params            = params
             self.request_context   = request_context
@@ -83,7 +83,7 @@ module TopologicalInventory
                                  application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
                                end
 
-            logger.availability_check("Skipping, last check at #{endpoint.last_checked_at || application.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
+            logger.availability_check("Skipping, last check at #{endpoint&.last_checked_at || application&.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
 
             checked_recently
           end
@@ -208,13 +208,14 @@ module TopologicalInventory
           end
 
           def availability_status_message(payload)
-            messaging_client.publish_message(
+            messaging_client.publish_topic(
               :service => SERVICE_NAME,
-              :message => EVENT_AVAILABILITY_STATUS,
-              :payload => payload.to_json
+              :event   => EVENT_AVAILABILITY_STATUS,
+              :payload => payload.to_json,
+              :headers => identity
             )
           rescue => err
-            logger.availability_check("Failed to update Application id: #{application.id} - #{err.message}", :error)
+            logger.availability_check("Failed to update #{payload[:resource_type]} id: #{payload[:resource_id]} - #{err.message}", :error)
           ensure
             messaging_client&.close
           end

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "2.1.0".freeze
+      VERSION = "2.1.5".freeze
     end
   end
 end

data/spec/support/shared/availability_check.rb

@@ -39,11 +39,14 @@ RSpec.shared_examples "availability_check" do
   def kafka_message(resource_type, resource_id, status, error_message = nil)
     res = {
       :service => described_class::SERVICE_NAME,
-      :message => described_class::EVENT_AVAILABILITY_STATUS,
+      :event   => described_class::EVENT_AVAILABILITY_STATUS,
       :payload => {
         :resource_type => resource_type,
         :resource_id   => resource_id,
         :status        => status
+      },
+      :headers => {
+        "x-rh-identity" => "eyJpZGVudGl0eSI6eyJhY2NvdW50X251bWJlciI6IjExMDAxIiwidXNlciI6eyJpc19vcmdfYWRtaW4iOnRydWV9fX0="
       }
     }
     res[:payload][:error] = error_message if error_message
@@ -67,15 +70,15 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_available, ''])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_available)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_available, '')
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Application", application_id, status_available)
         )
 
@@ -89,15 +92,15 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_unavailable, error_message)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Application", application_id, status_unavailable)
         )
 
@@ -110,7 +113,7 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
@@ -126,11 +129,11 @@ RSpec.shared_examples "availability_check" do
 
         expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Source", source_id, status_unavailable)
         )
 
-        expect(kafka_client).to receive(:publish_message).with(
+        expect(kafka_client).to receive(:publish_topic).with(
           kafka_message("Endpoint", endpoint_id, status_unavailable, error_message)
         )
 
@@ -258,11 +261,11 @@ RSpec.shared_examples "availability_check" do
 
           expect(subject).to receive(:connection_status).and_return([status_available, ''])
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Source", source_id, status_available)
           )
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Application", application_id, status_available)
           )
 
@@ -307,11 +310,11 @@ RSpec.shared_examples "availability_check" do
 
           expect(subject).to receive(:connection_status).and_return([status_unavailable, error_message])
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Source", source_id, status_unavailable)
           )
 
-          expect(kafka_client).to receive(:publish_message).with(
+          expect(kafka_client).to receive(:publish_topic).with(
             kafka_message("Application", application_id, status_unavailable)
           )
 

data/spec/topological_inventory/providers/common/operations/async_worker_spec.rb

@@ -3,33 +3,42 @@ require "topological_inventory/providers/common/operations/async_worker"
 describe TopologicalInventory::Providers::Common::Operations::AsyncWorker do
   let(:queue) { double }
   let(:impl) { double }
+  let(:metrics) { double('metrics') }
   let(:msg) { double }
-  subject { described_class.new(impl, queue) }
+  let(:operation) { "Source.availability_check" }
+
+  subject { described_class.new(impl, :queue => queue, :metrics => metrics) }
 
   before do
     allow(queue).to receive(:length).and_return(0)
-    allow(msg).to receive(:message).and_return("Source.availability_check")
+    allow(msg).to receive(:message).and_return(operation)
   end
 
   context "when the message is able to be processed" do
+    let(:result) { subject.operation_status[:success] }
     before do
-      allow(impl).to receive(:process!).with(msg)
+      allow(impl).to receive(:process!).with(msg, metrics).and_return(result)
       allow(msg).to receive(:ack)
     end
 
     it "drains messages that are added to the queue" do
-      expect(impl).to receive(:process!).with(msg).once
+      expect(impl).to receive(:process!).with(msg, metrics).once
+      expect(metrics).to receive(:record_operation).with(operation, :status => result)
       subject.send(:process_message, msg)
     end
   end
 
   context "when the message results in an error" do
+    let(:result) { subject.operation_status[:error] }
     before do
-      allow(impl).to receive(:process!).with(msg).and_raise(StandardError.new("boom!"))
+      allow(impl).to receive(:process!).with(msg, metrics).and_raise(StandardError.new("boom!"))
     end
 
     it "ack's the message on failure" do
+      allow(subject).to receive(:logger).and_return(double.as_null_object)
+      
       expect(msg).to receive(:ack).once
+      expect(metrics).to receive(:record_operation).with(operation, :status => result)
       subject.send(:process_message, msg)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-providers-common
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.5
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-02 00:00:00.000000000 Z
+date: 2021-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport