topological_inventory-providers-common 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcac7f624e6e6d2ba88bed97e3d089cfd4258e38b968cbed141528888ede0ca0
-  data.tar.gz: 7fe176f46948a2359192f503091bce1fadfe3abbb089e4c8efc5606ea07e6ed0
+  metadata.gz: 00d8f69979f477e6b74d6c7b2785647cc043b3909a45ff424562504146955e2b
+  data.tar.gz: 1b009a746f92137d0502c71bf67346ba86de56aba196eb05e52a6280b86861ec
 SHA512:
-  metadata.gz: 2609f90ca604593f8d1bed9c0567feae6151bb6a0805a6299a9077e8b812a58d49573e32eb1cc7a0ff9cf70369c8d7936f88887c4c0b6a7af7f9e50d1013275c
-  data.tar.gz: 36a45b32003a82705b03d15bc15fbdae18038fc04c2238db55b0077c50868b39ea199de86185a6b00c2d9afc6ce76d9308db8c0ef5ddbc844ef57340f71d6eb7
+  metadata.gz: 84a4936e15d4fa024d425d430ba2489485259e59be735e86aa3801852b5745825b324a58b2e847781700607d76c1e5e2deb13bb453f019dba2b0e5382f705c8b
+  data.tar.gz: 10e5edc584fbfcfd85ae196aef5f83c1e7e42fb0e4325f29b302feecc8c2937925580e23d903442546e8af826afb91fd8c9ea2164812f4e1baabed2087fb6025

data/.github/workflows/gem-push.yml

@@ -0,0 +1,23 @@
+name: Release Ruby Gem
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+    - 'lib/topological_inventory/providers/common/version.rb'
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Publish to Rubygems
+      uses: cadwallion/publish-rubygems-action@v1.0
+      env:
+        GITHUB_TOKEN: ${{secrets.GH_API_KEY}}
+        RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
+        RELEASE_COMMAND: rake release
+

data/.gitignore

@@ -11,3 +11,6 @@
 
 # rspec failure tracking
 .rspec_status
+
+# generated rubocop file
+.rubocop-https*

data/.rubocop.yml

@@ -0,0 +1,3 @@
+inherit_from:
+- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
+- .rubocop_local.yml

data/.rubocop_cc.yml

@@ -0,0 +1,4 @@
+inherit_from:
+- .rubocop_base.yml
+- .rubocop_cc_base.yml
+- .rubocop_local.yml

data/.rubocop_local.yml

@@ -0,0 +1,2 @@
+#AllCops:
+#  Exclude:

data/.travis.yml

@@ -1,4 +1,5 @@
 ---
+dist: xenial
 sudo: false
 language: ruby
 cache: bundler
@@ -8,3 +9,10 @@ rvm:
 before_install:
   - 'echo ''gem: --no-ri --no-rdoc --no-document'' > ~/.gemrc'
   - gem install bundler
+before_script:
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
+    > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+  - "./cc-test-reporter before-build"
+after_script:
+  - "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"

data/CHANGELOG.md

@@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.3] - 2020-06-?
+## [1.0.4] - 2020-06-09
+Common availability check operation #25
+Rubocop and codecoverage #29
+
+## [1.0.3] - 2020-06-04
 ### Changed
 
 Bump Sources API client to 3.0 #26
@@ -25,7 +29,8 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...HEAD
+[1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
 [1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3
 [1.0.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.1...v1.0.2
 [1.0.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.0...v1.0.1

data/lib/topological_inventory/providers/common/logging.rb

@@ -21,6 +21,14 @@ module TopologicalInventory
           msg = "[#{status.to_s.upcase}] Sweeping inactive records, :sweep_scope => #{sweep_scope}, :source_uid => #{source}, :refresh_state_uuid => #{refresh_state_uuid}"
           info(msg)
         end
+
+        def availability_check(message, severity = :info)
+          log_with_prefix("Source#availability_check", message, severity)
+        end
+
+        def log_with_prefix(prefix, message, severity)
+          send(severity, "#{prefix} - #{message}") if respond_to?(severity)
+        end
       end
 
       class Logger < ManageIQ::Loggers::CloudWatch

data/lib/topological_inventory/providers/common/operations/endpoint_client.rb

@@ -46,6 +46,9 @@ module TopologicalInventory
 
           def default_endpoint
             @default_endpoint ||= sources_api.fetch_default_endpoint(source_id)
+            raise "Sources API: Endpoint not found! (source id: #{source_id})" if @default_endpoint.nil?
+
+            @default_endpoint
           end
 
           def authentication

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -0,0 +1,140 @@
+require "topological_inventory/providers/common/logging"
+require "active_support/core_ext/numeric/time"
+require "topological_inventory/providers/common/operations/sources_api_client"
+
+module TopologicalInventory
+  module Providers
+  module Common
+    module Operations
+      class Source
+        include Logging
+
+        STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
+
+        ERROR_MESSAGES = {
+          :authentication_not_found => "Authentication not found in Sources API",
+          :endpoint_not_found       => "Endpoint not found in Sources API",
+        }.freeze
+
+        LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
+
+        attr_accessor :params, :request_context, :source_id
+
+        def initialize(params = {}, request_context = nil)
+          self.params          = params
+          self.request_context = request_context
+          self.source_id       = params['source_id']
+        end
+
+        def availability_check
+          return if params_missing?
+
+          return if checked_recently?
+
+          status, error_message = connection_status
+
+          update_source_and_endpoint(status, error_message)
+
+          logger.availability_check("Completed: Source #{source_id} is #{status}")
+        end
+
+        private
+
+        def required_params
+          %w[source_id]
+        end
+
+        def params_missing?
+          is_missing = false
+          required_params.each do |attr|
+            if (is_missing = params[attr].blank?)
+              logger.availability_check("Missing #{attr} for the availability_check request [Source ID: #{source_id}]", :error)
+              break
+            end
+          end
+
+          is_missing
+        end
+
+        def checked_recently?
+          return false if endpoint.nil?
+
+          checked_recently = endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
+          logger.availability_check("Skipping, last check at #{endpoint.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
+
+          checked_recently
+        end
+
+        def connection_status
+          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_not_found]] unless endpoint
+          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
+
+          check_time
+          connection_check
+        end
+
+        # @return [Array<String, String|nil] - STATUS_[UN]AVAILABLE, error message
+        def connection_check
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+
+        def update_source_and_endpoint(status, error_message = nil)
+          logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
+
+          update_source(status)
+          update_endpoint(status, error_message)
+        end
+
+        def update_source(status)
+          source = ::SourcesApiClient::Source.new
+          source.availability_status = status
+          source.last_checked_at     = check_time
+          source.last_available_at   = check_time if status == STATUS_AVAILABLE
+
+          api_client.update_source(source_id, source)
+        rescue ::SourcesApiClient::ApiError => e
+          logger.availability_check("Failed to update Source id:#{source_id} - #{e.message}", :error)
+        end
+
+        def update_endpoint(status, error_message)
+          if endpoint.nil?
+            logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
+            return
+          end
+
+          endpoint_update = ::SourcesApiClient::Endpoint.new
+
+          endpoint_update.availability_status       = status
+          endpoint_update.availability_status_error = error_message.to_s
+          endpoint_update.last_checked_at           = check_time
+          endpoint_update.last_available_at         = check_time if status == STATUS_AVAILABLE
+
+          api_client.update_endpoint(endpoint.id, endpoint_update)
+        rescue ::SourcesApiClient::ApiError => e
+          logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
+        end
+
+        def endpoint
+          @endpoint ||= api_client.fetch_default_endpoint(source_id)
+        end
+
+        def authentication
+          @authentication ||= api_client.fetch_authentication(source_id, endpoint)
+        end
+
+        def check_time
+          @check_time ||= Time.now.utc
+        end
+
+        def identity
+          @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => params["external_tenant"], "user" => {"is_org_admin" => true}}}.to_json)}
+        end
+
+        def api_client
+          @api_client ||= TopologicalInventory::Providers::Common::Operations::SourcesApiClient.new(identity)
+        end
+      end
+    end
+  end
+  end
+end

data/lib/topological_inventory/providers/common/operations/sources_api_client.rb

@@ -5,6 +5,8 @@ module TopologicalInventory
     module Common
       module Operations
         class SourcesApiClient < ::SourcesApiClient::ApiClient
+          delegate :update_source, :update_endpoint, :to => :api
+
           INTERNAL_API_PATH = '//internal/v1.0'.freeze
 
           def initialize(identity = nil)
@@ -20,21 +22,23 @@ module TopologicalInventory
 
           def fetch_default_endpoint(source_id)
             endpoints = api.list_source_endpoints(source_id)&.data || []
-            endpoint = endpoints.find(&:default)
-
-            raise "Sources API: Endpoint not found! (source id: #{source_id})" if endpoint.nil?
-
-            endpoint
+            endpoints.find(&:default)
           end
 
-          def fetch_authentication(source_id, default_endpoint = nil)
+          def fetch_authentication(source_id, default_endpoint = nil, authtype = nil)
             endpoint = default_endpoint || fetch_default_endpoint(source_id)
             return if endpoint.nil?
 
             endpoint_authentications = api.list_endpoint_authentications(endpoint.id.to_s).data || []
             return if endpoint_authentications.empty?
 
-            auth_id = endpoint_authentications.first.id
+            auth_id = if authtype.nil?
+                        endpoint_authentications.first&.id
+                      else
+                        endpoint_authentications.detect { |a| a.authtype = authtype }&.id
+                      end
+            return if auth_id.nil?
+
             fetch_authentication_with_password(auth_id)
           end
 

data/lib/topological_inventory/providers/common/save_inventory/saver.rb

@@ -8,13 +8,14 @@ module TopologicalInventory
           # As defined in:
           # https://github.com/zendesk/ruby-kafka/blob/02f7e2816e1130c5202764c275e36837f57ca4af/lib/kafka/protocol/message.rb#L11-L17
           # There is at least 112 bytes that are added as a message header, so we need to keep room for that. Lets make
-          # it 200 bytes, just for sure.
-          KAFKA_RESERVED_HEADER_SIZE = 200
+          # it 512 bytes, just for sure.
+          KAFKA_PAYLOAD_MAX_BYTES_DEFAULT = 750_000
+          KAFKA_RESERVED_HEADER_SIZE = 512
 
-          def initialize(client:, logger:, max_bytes: 1_000_000)
+          def initialize(client:, logger:, max_bytes: KAFKA_PAYLOAD_MAX_BYTES_DEFAULT)
             @client    = client
             @logger    = logger
-            @max_bytes = max_bytes - KAFKA_RESERVED_HEADER_SIZE
+            @max_bytes = payload_max_size(max_bytes)
           end
 
           attr_reader :client, :logger, :max_bytes
@@ -117,6 +118,14 @@ module TopologicalInventory
             new_inventory[:collections]             = []
             new_inventory
           end
+
+          def payload_max_size(max_bytes)
+            if ENV['KAFKA_PAYLOAD_MAX_BYTES']
+              max_bytes.clamp(5_000, ENV['KAFKA_PAYLOAD_MAX_BYTES'].to_i) - KAFKA_RESERVED_HEADER_SIZE
+            else
+              max_bytes - KAFKA_RESERVED_HEADER_SIZE
+            end
+          end
         end
       end
     end

data/lib/topological_inventory/providers/common/version.rb

@@ -1,7 +1,7 @@
 module TopologicalInventory
   module Providers
     module Common
-      VERSION = "1.0.3"
+      VERSION = "1.0.4"
     end
   end
 end

data/topological_inventory-providers-common.gemspec

@@ -34,4 +34,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'rubocop', '~>0.69.0'
+  spec.add_development_dependency 'rubocop-performance', '~>1.3'
+  spec.add_development_dependency "simplecov", "~> 0.17.1"
+  spec.add_development_dependency 'webmock'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topological_inventory-providers-common
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-04 00:00:00.000000000 Z
+date: 2020-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -162,6 +162,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.69.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.69.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.1
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Common classes for topological-inventory collectors/operations
 email:
 - mslemr@redhat.com
@@ -169,8 +225,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/gem-push.yml"
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
+- ".rubocop_cc.yml"
+- ".rubocop_local.yml"
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
@@ -188,6 +248,7 @@ files:
 - lib/topological_inventory/providers/common/logging.rb
 - lib/topological_inventory/providers/common/operations/endpoint_client.rb
 - lib/topological_inventory/providers/common/operations/processor.rb
+- lib/topological_inventory/providers/common/operations/source.rb
 - lib/topological_inventory/providers/common/operations/sources_api_client.rb
 - lib/topological_inventory/providers/common/operations/topology_api_client.rb
 - lib/topological_inventory/providers/common/save_inventory/exception.rb