topological_inventory-providers-common 1.0.2 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76ba202072a52538f20a377fcf8f85923ba04e2ad745d64dc59c95c0332e6474
-  data.tar.gz: 74664f658d5299244d9276a964ac9c2a8c15a96840c0d6749dac09b567e3b968
+  metadata.gz: 868c8f764cd11d9888ed99698a309cc199fdf92823963f4e5cae71208d506ff7
+  data.tar.gz: 39c352e8b1306bdb87675bf0ccc1fe08f02b0c13555b2466c4c38d7a76979aca
 SHA512:
-  metadata.gz: bfd655f70da56219cb312d4c5b46403b71b6401ac9dd47e2ed6827f079a4df7d6edc5e89e33c293438a7c2dcbdbc024382605917b9ec8e5ec3a75d9553cd6eb5
-  data.tar.gz: 642ea847c5f186e62983a7df70bc39feaf7aa8a841d07ef5f0dcaf9a7cdb59866eace13dcbc9730712bbc8b7f2761f51b1df79befc357fc58f02ef16fab192b5
+  metadata.gz: c2a8af390114548a89c7dc5bb43f1a6e3784a5a59ebd2ce1c7451c15726015df1353e3dcbc3755d7637d696707b0d6db93e5d4907920a1c4b09d3f6410130a77
+  data.tar.gz: 46e6596954367336de4049f8024da8b5e63e5be4c60636c77f91f22c3e407229b38aca604137907d1f5cd6411201d9c3b57e4ad5bab2cfb749b2c04e9ca8eeab

data/.github/workflows/gem-push.yml

@@ -0,0 +1,49 @@
+name: Release Ruby Gem
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+    - 'lib/topological_inventory/providers/common/version.rb'
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+
+    - name: Read the version.rb
+      id: version_file
+      run: |
+        echo ::set-output name=data::$(grep VERSION lib/topological_inventory/providers/common/version.rb | awk {'print $3'} | tr -d '"')
+
+    - name: Echo the gem version
+      run: |
+        echo "v${{ steps.version_file.outputs.data }}"
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: "v${{ steps.version_file.outputs.data }}"
+        release_name: "v${{ steps.version_file.outputs.data }}"

data/.gitignore

@@ -11,3 +11,6 @@
 
 # rspec failure tracking
 .rspec_status
+
+# generated rubocop file
+.rubocop-https*

data/.rubocop.yml

@@ -0,0 +1,3 @@
+inherit_from:
+- https://raw.githubusercontent.com/ManageIQ/guides/master/.rubocop_base.yml
+- .rubocop_local.yml

data/.rubocop_cc.yml

@@ -0,0 +1,4 @@
+inherit_from:
+- .rubocop_base.yml
+- .rubocop_cc_base.yml
+- .rubocop_local.yml

data/.rubocop_local.yml

@@ -0,0 +1,2 @@
+#AllCops:
+#  Exclude:

data/.travis.yml

@@ -1,4 +1,5 @@
 ---
+dist: xenial
 sudo: false
 language: ruby
 cache: bundler
@@ -8,3 +9,10 @@ rvm:
 before_install:
   - 'echo ''gem: --no-ri --no-rdoc --no-document'' > ~/.gemrc'
   - gem install bundler
+before_script:
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
+    > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+  - "./cc-test-reporter before-build"
+after_script:
+  - "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"

data/CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.7] - 2020-07-27
+Update operations/source model for receptor-enabled availability checks #36
+Add check for Application subresource under a Source during Availability check #40
+Remove infinite loop in error messages #43
+
+## [1.0.6] - 2020-07-06
+Add some error handling if Sources does not have endpoints/authentications for a source #38
+Specs for Collector #35
+
+## [1.0.5] - 2020-06-18
+Change release workflow to do everything manually #32
+Add specs to released files #33
+
+## [1.0.4] - 2020-06-18
+Common availability check operation #25
+Rubocop and codecoverage #29
+Add github workflow to release to rubygems automatically #31
+
+## [1.0.3] - 2020-06-04
+### Changed
+
+Bump Sources API client to 3.0 #26
+
 ## [1.0.2] - 2020-05-15
 ### Changed
 
@@ -20,7 +43,12 @@ manageiq-loggers to >= 0.4.2 #20
 ## [1.0.0] - 2020-03-19
 ### Initial release to rubygems.org
 
-[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.1...HEAD
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.7...HEAD
+[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...v1.0.7
+[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
+[1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
+[1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
+[1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3
 [1.0.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.1...v1.0.2
 [1.0.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.0...v1.0.1
 [1.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/releases/v1.0.0

data/Gemfile

@@ -6,9 +6,6 @@ require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundle
 # Specify your gem's dependencies in topological_inventory-providers-common.gemspec
 gemspec
 
-gem "sources-api-client", "~> 1.0"
-gem "topological_inventory-ingress_api-client", "~> 1.0"
-
 group :development, :test do
   gem 'rake', '>= 12.3.3'
   gem 'pry-byebug'

data/lib/topological_inventory/providers/common/logging.rb

@@ -21,6 +21,14 @@ module TopologicalInventory
           msg = "[#{status.to_s.upcase}] Sweeping inactive records, :sweep_scope => #{sweep_scope}, :source_uid => #{source}, :refresh_state_uuid => #{refresh_state_uuid}"
           info(msg)
         end
+
+        def availability_check(message, severity = :info)
+          log_with_prefix("Source#availability_check", message, severity)
+        end
+
+        def log_with_prefix(prefix, message, severity)
+          send(severity, "#{prefix} - #{message}") if respond_to?(severity)
+        end
       end
 
       class Logger < ManageIQ::Loggers::CloudWatch

data/lib/topological_inventory/providers/common/operations/endpoint_client.rb

@@ -46,6 +46,9 @@ module TopologicalInventory
 
           def default_endpoint
             @default_endpoint ||= sources_api.fetch_default_endpoint(source_id)
+            raise "Sources API: Endpoint not found! (source id: #{source_id})" if @default_endpoint.nil?
+
+            @default_endpoint
           end
 
           def authentication

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -0,0 +1,191 @@
+require "topological_inventory/providers/common/logging"
+require "active_support/core_ext/numeric/time"
+require "topological_inventory/providers/common/operations/sources_api_client"
+
+module TopologicalInventory
+  module Providers
+  module Common
+    module Operations
+      class Source
+        include Logging
+
+        STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
+
+        ERROR_MESSAGES = {
+          :authentication_not_found          => "Authentication not found in Sources API",
+          :endpoint_or_application_not_found => "Endpoint or Application not found in Sources API",
+        }.freeze
+
+        LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
+        AUTH_NOT_NECESSARY = "n/a".freeze
+
+        attr_accessor :params, :request_context, :source_id, :account_number
+
+        def initialize(params = {}, request_context = nil)
+          self.params          = params
+          self.request_context = request_context
+          self.source_id       = params['source_id']
+          self.account_number  = params['external_tenant']
+        end
+
+        def availability_check
+          return if params_missing?
+
+          return if checked_recently?
+
+          status, error_message = connection_status
+
+          update_source_and_subresources(status, error_message)
+
+          logger.availability_check("Completed: Source #{source_id} is #{status}")
+        end
+
+        private
+
+        def required_params
+          %w[source_id]
+        end
+
+        def params_missing?
+          is_missing = false
+          required_params.each do |attr|
+            if (is_missing = params[attr].blank?)
+              logger.availability_check("Missing #{attr} for the availability_check request [Source ID: #{source_id}]", :error)
+              break
+            end
+          end
+
+          is_missing
+        end
+
+        def checked_recently?
+          checked_recently = if endpoint.present?
+                               endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
+                             elsif application.present?
+                               application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
+                             end
+
+          logger.availability_check("Skipping, last check at #{endpoint.last_checked_at || application.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
+
+          checked_recently
+        end
+
+        def connection_status
+          # we need either an endpoint or application to check the source.
+          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_or_application_not_found]] unless endpoint || application
+
+          check_time
+          if endpoint
+            endpoint_connection_check
+          elsif application
+            application_connection_check
+          end
+        end
+
+        def endpoint_connection_check
+          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
+
+          # call down into the operations pod implementation of `Source#connection_check`
+          connection_check
+        end
+
+        def application_connection_check
+          case application.availability_status
+          when "available"
+            [STATUS_AVAILABLE, nil]
+          when "unavailable"
+            [STATUS_UNAVAILABLE, "Application id #{application.id} unavailable"]
+          end
+        end
+
+        # @return [Array<String, String|nil] - STATUS_[UN]AVAILABLE, error message
+        def connection_check
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+
+        def update_source_and_subresources(status, error_message = nil)
+          logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
+
+          update_source(status)
+
+          update_endpoint(status, error_message) if endpoint
+          update_application(status) if application
+        end
+
+        def update_source(status)
+          source = ::SourcesApiClient::Source.new
+          source.availability_status = status
+          source.last_checked_at     = check_time
+          source.last_available_at   = check_time if status == STATUS_AVAILABLE
+
+          api_client.update_source(source_id, source)
+        rescue ::SourcesApiClient::ApiError => e
+          logger.availability_check("Failed to update Source id:#{source_id} - #{e.message}", :error)
+        end
+
+        def update_endpoint(status, error_message)
+          if endpoint.nil?
+            logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
+            return
+          end
+
+          endpoint_update = ::SourcesApiClient::Endpoint.new
+
+          endpoint_update.availability_status       = status
+          endpoint_update.availability_status_error = error_message.to_s
+          endpoint_update.last_checked_at           = check_time
+          endpoint_update.last_available_at         = check_time if status == STATUS_AVAILABLE
+
+          api_client.update_endpoint(endpoint.id, endpoint_update)
+        rescue ::SourcesApiClient::ApiError => e
+          logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
+        end
+
+        def update_application(status)
+          application_update = ::SourcesApiClient::Application.new
+          application_update.last_checked_at     = check_time
+          application_update.last_available_at   = check_time if status == STATUS_AVAILABLE
+
+          api_client.update_application(application.id, application_update)
+        rescue ::SourcesApiClient::ApiError => e
+          logger.availability_check("Failed to update Application id: #{application.id} - #{e.message}", :error)
+        end
+
+        def endpoint
+          @endpoint ||= api_client.fetch_default_endpoint(source_id)
+        rescue e
+          logger.availability_check("Failed to fetch Endpoint for Source #{source_id}: #{e.message}", :error)
+        end
+
+        def authentication
+          @authentication ||= if endpoint.receptor_node.present?
+                                AUTH_NOT_NECESSARY
+                              else
+                                api_client.fetch_authentication(source_id, endpoint)
+                              end
+        rescue e
+          logger.availability_check("Failed to fetch Authentication for Source #{source_id}: #{e.message}", :error)
+        end
+
+        def application
+          @application ||= api_client.fetch_application(source_id)
+        rescue e
+          logger.availability_check("Failed to fetch Application for Source #{source_id}: #{e.message}", :error)
+        end
+
+        def check_time
+          @check_time ||= Time.now.utc
+        end
+
+        def identity
+          @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => account_number, "user" => {"is_org_admin" => true}}}.to_json)}
+        end
+
+        def api_client
+          @api_client ||= TopologicalInventory::Providers::Common::Operations::SourcesApiClient.new(identity)
+        end
+      end
+    end
+  end
+  end
+end

data/lib/topological_inventory/providers/common/operations/sources_api_client.rb

@@ -5,6 +5,8 @@ module TopologicalInventory
     module Common
       module Operations
         class SourcesApiClient < ::SourcesApiClient::ApiClient
+          delegate :update_source, :update_endpoint, :update_application, :to => :api
+
           INTERNAL_API_PATH = '//internal/v1.0'.freeze
 
           def initialize(identity = nil)
@@ -20,21 +22,28 @@ module TopologicalInventory
 
           def fetch_default_endpoint(source_id)
             endpoints = api.list_source_endpoints(source_id)&.data || []
-            endpoint = endpoints.find(&:default)
-
-            raise "Sources API: Endpoint not found! (source id: #{source_id})" if endpoint.nil?
+            endpoints.find(&:default)
+          end
 
-            endpoint
+          def fetch_application(source_id)
+            applications = api.list_source_applications(source_id)&.data || []
+            applications.first
           end
 
-          def fetch_authentication(source_id, default_endpoint = nil)
+          def fetch_authentication(source_id, default_endpoint = nil, authtype = nil)
             endpoint = default_endpoint || fetch_default_endpoint(source_id)
             return if endpoint.nil?
 
             endpoint_authentications = api.list_endpoint_authentications(endpoint.id.to_s).data || []
             return if endpoint_authentications.empty?
 
-            auth_id = endpoint_authentications.first.id
+            auth_id = if authtype.nil?
+                        endpoint_authentications.first&.id
+                      else
+                        endpoint_authentications.detect { |a| a.authtype = authtype }&.id
+                      end
+            return if auth_id.nil?
+
             fetch_authentication_with_password(auth_id)
           end
 

data/lib/topological_inventory/providers/common/save_inventory/saver.rb

@@ -8,13 +8,14 @@ module TopologicalInventory
           # As defined in:
           # https://github.com/zendesk/ruby-kafka/blob/02f7e2816e1130c5202764c275e36837f57ca4af/lib/kafka/protocol/message.rb#L11-L17
           # There is at least 112 bytes that are added as a message header, so we need to keep room for that. Lets make
-          # it 200 bytes, just for sure.
-          KAFKA_RESERVED_HEADER_SIZE = 200
+          # it 512 bytes, just for sure.
+          KAFKA_PAYLOAD_MAX_BYTES_DEFAULT = 750_000
+          KAFKA_RESERVED_HEADER_SIZE = 512
 
-          def initialize(client:, logger:, max_bytes: 1_000_000)
+          def initialize(client:, logger:, max_bytes: KAFKA_PAYLOAD_MAX_BYTES_DEFAULT)
             @client    = client
             @logger    = logger
-            @max_bytes = max_bytes - KAFKA_RESERVED_HEADER_SIZE
+            @max_bytes = payload_max_size(max_bytes)
           end
 
           attr_reader :client, :logger, :max_bytes
@@ -117,6 +118,14 @@ module TopologicalInventory
             new_inventory[:collections]             = []
             new_inventory
           end
+
+          def payload_max_size(max_bytes)
+            if ENV['KAFKA_PAYLOAD_MAX_BYTES']
+              max_bytes.clamp(5_000, ENV['KAFKA_PAYLOAD_MAX_BYTES'].to_i) - KAFKA_RESERVED_HEADER_SIZE
+            else
+              max_bytes - KAFKA_RESERVED_HEADER_SIZE
+            end
+          end
         end
       end
     end