topological_inventory-providers-common 1.0.9 → 2.1.0

data/lib/topological_inventory/providers/common/operations/health_check.rb

@@ -0,0 +1,15 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class HealthCheck
+          HEARTBEAT_FILE = '/tmp/healthy'.freeze
+
+          def self.touch_file
+            FileUtils.touch(HEARTBEAT_FILE)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/processor.rb

@@ -1,4 +1,6 @@
-require "topological_inventory/providers/common/operations/topology_api_client"
+require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/mixins/statuses"
+require "topological_inventory/providers/common/mixins/topology_api"
 
 module TopologicalInventory
   module Providers
@@ -6,127 +8,67 @@ module TopologicalInventory
       module Operations
         class Processor
           include Logging
-          include TopologyApiClient
+          include Mixins::Statuses
+          include Mixins::TopologyApi
 
-          SLEEP_POLL = 10
-          POLL_TIMEOUT = 1800
-
-          def self.process!(message)
-            model, method = message.headers['message_type'].to_s.split(".")
-            new(model, method, message.payload).process
+          def self.process!(message, metrics)
+            new(message, metrics).process
           end
 
-          # @param payload [Hash] https://github.com/ManageIQ/topological_inventory-api/blob/master/app/controllers/api/v0/service_plans_controller.rb#L32-L41
-          def initialize(model, method, payload, metrics = nil)
-            self.model           = model
-            self.method          = method
-            self.params          = payload["params"]
-            self.identity        = payload["request_context"]
-            self.metrics         = metrics
+          def initialize(message, metrics)
+            self.message            = message
+            self.metrics            = metrics
+            self.model, self.method = message.message.split(".")
+
+            self.params   = message.payload["params"]
+            self.identity = message.payload["request_context"]
           end
 
           def process
-            logger.info("Processing #{model}##{method} [#{params}]...")
-            result = order_service(params)
-            logger.info("Processing #{model}##{method} [#{params}]...Complete")
-
-            result
+            logger.info(status_log_msg)
+            impl = operation_class&.new(params, identity, metrics)
+            if impl&.respond_to?(method)
+              with_time_measure do
+                result = impl.send(method)
+
+                logger.info(status_log_msg("Complete"))
+                result
+              end
+            else
+              logger.warn(status_log_msg("Not Implemented!"))
+              complete_task("not implemented") if params["task_id"]
+              operation_status[:not_implemented]
+            end
+          rescue StandardError, NotImplementedError => e
+            complete_task(e.message) if params["task_id"]
+            raise
           end
 
           private
 
-          attr_accessor :identity, :model, :method, :metrics, :params
-
-          def endpoint_client(source_id, task_id, identity)
-            raise NotImplementedError, "#{__method__} must be implemented in a subclass as kind of TopologicalInventory::Providers::Common::EndpointClient class"
-          end
-
-          def order_service(params)
-            task_id, service_offering_id, service_plan_id, order_params = params.values_at("task_id", "service_offering_id", "service_plan_id", "order_params")
-
-            service_plan     = topology_api_client.show_service_plan(service_plan_id) if service_plan_id.present?
-            service_offering_id = service_plan.service_offering_id if service_offering_id.nil? && service_plan.present?
-            service_offering = topology_api_client.show_service_offering(service_offering_id)
-
-            source_id        = service_offering.source_id
-            client = endpoint_client(source_id, task_id, identity)
-
-            logger.info("Ordering #{service_offering.name}...")
-            remote_service_instance = client.order_service(service_offering, service_plan.presence, order_params)
-            logger.info("Ordering #{service_offering.name}...Complete")
+          attr_accessor :message, :identity, :model, :method, :metrics, :params
 
-            poll_order_complete_thread(task_id, source_id, remote_service_instance)
-          rescue StandardError => err
-            metrics&.record_error
-            logger.error("[Task #{task_id}] Ordering error: #{err}\n#{err.backtrace.join("\n")}")
-            update_task(task_id, :state => "completed", :status => "error", :context => {:error => err.to_s})
+          def operation_class
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
           end
 
-          def poll_order_complete_thread(task_id, source_id, remote_svc_instance)
-            Thread.new do
-              begin
-                poll_order_complete(task_id, source_id, remote_svc_instance)
-              rescue StandardError => err
-                logger.error("[Task #{task_id}] Waiting for complete: #{err}\n#{err.backtrace.join("\n")}")
-                update_task(task_id, :state => "completed", :status => "warn", :context => {:error => err.to_s})
-              end
+          def with_time_measure
+            if metrics.present?
+              metrics.record_operation_time(message.message) { yield }
+            else
+              yield
             end
           end
 
-          def poll_order_complete(task_id, source_id, remote_svc_instance)
-            client = endpoint_client(source_id, task_id, identity)
-
-            context = {
-              :service_instance => {
-                :source_id  => source_id,
-                :source_ref => client.source_ref_of(remote_svc_instance)
-              }
-            }
-
-            remote_svc_instance = client.wait_for_provision_complete(task_id, remote_svc_instance, context)
-
-            if client.provisioned_successfully?(remote_svc_instance)
-              if (service_instance = load_topological_svc_instance(source_id, client.source_ref_of(remote_svc_instance))).present?
-                context[:service_instance][:id] = service_instance.id
-                context[:service_instance][:url] = svc_instance_url(service_instance)
-              else
-                logger.warn("Failed to get service_instance API URL (endpoint's service instance: #{remote_svc_instance.inspect})")
-              end
-            end
-            update_task(task_id, :state => "completed", :status => client.task_status_for(remote_svc_instance), :context => context)
+          def complete_task(msg, status = "error")
+            update_task(params["task_id"],
+                        :state   => "completed",
+                        :status  => status,
+                        :context => {:error => "#{model}##{method} - #{msg}"})
           end
 
-          def load_topological_svc_instance(source_id, source_ref)
-            api = topology_api_client.api_client
-
-            count = 0
-            timeout_count = POLL_TIMEOUT / SLEEP_POLL
-
-            header_params = { 'Accept' => api.select_header_accept(['application/json']) }
-            query_params = { :'source_id' => source_id, :'source_ref' => source_ref }
-            return_type = 'ServiceInstancesCollection'
-
-            service_instance = nil
-            loop do
-              data, _status_code, _headers = api.call_api(:GET, "/service_instances",
-                                                          :header_params => header_params,
-                                                          :query_params  => query_params,
-                                                          :auth_names    => ['UserSecurity'],
-                                                          :return_type   => return_type)
-
-              service_instance = data.data&.first if data.meta.count > 0
-              break if service_instance.present?
-
-              break if (count += 1) >= timeout_count
-
-              sleep(SLEEP_POLL) # seconds
-            end
-
-            if service_instance.nil?
-              logger.error("Failed to find service_instance by source_id [#{source_id}] source_ref [#{source_ref}]")
-            end
-
-            service_instance
+          def status_log_msg(status = nil)
+            "Processing #{model}##{method} [#{params}]...#{status}"
           end
         end
       end

data/lib/topological_inventory/providers/common/operations/source.rb

@@ -1,194 +1,233 @@
 require "topological_inventory/providers/common/logging"
 require "active_support/core_ext/numeric/time"
-require "topological_inventory/providers/common/operations/sources_api_client"
+require "topological_inventory/providers/common/mixins/sources_api"
+require "topological_inventory/providers/common/mixins/statuses"
+require "topological_inventory/providers/common/mixins/x_rh_headers"
+require "topological_inventory/providers/common/messaging_client"
 
 module TopologicalInventory
   module Providers
-  module Common
-    module Operations
-      class Source
-        include Logging
-
-        STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
-
-        ERROR_MESSAGES = {
-          :authentication_not_found          => "Authentication not found in Sources API",
-          :endpoint_or_application_not_found => "Endpoint or Application not found in Sources API",
-        }.freeze
+    module Common
+      module Operations
+        class Source
+          include Logging
+          include Mixins::SourcesApi
+          include Mixins::XRhHeaders
+          include Mixins::Statuses
+
+          STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
+          EVENT_AVAILABILITY_STATUS            = "availability_status".freeze
+          SERVICE_NAME                         = "platform.sources.status".freeze
+
+          ERROR_MESSAGES = {
+            :authentication_not_found          => "Authentication not found in Sources API",
+            :endpoint_or_application_not_found => "Endpoint or Application not found in Sources API",
+          }.freeze
+
+          LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
+          AUTH_NOT_NECESSARY = "n/a".freeze
+
+          attr_accessor :identity, :operation, :params, :request_context, :source_id, :account_number
+
+          def initialize(params = {}, request_context = nil, metrics = nil)
+            self.metrics         = metrics
+            self.operation         = 'Source'
+            self.params            = params
+            self.request_context   = request_context
+            self.source_id         = params['source_id']
+            self.account_number    = params['external_tenant']
+            self.updates_via_kafka = ENV['UPDATE_SOURCES_VIA_API'].blank?
+            self.identity          = identity_by_account_number(account_number)
+          end
 
-        LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
-        AUTH_NOT_NECESSARY = "n/a".freeze
+          def availability_check
+            self.operation += '#availability_check'
 
-        attr_accessor :params, :request_context, :source_id, :account_number
+            return operation_status[:error] if params_missing?
 
-        def initialize(params = {}, request_context = nil)
-          self.params          = params
-          self.request_context = request_context
-          self.source_id       = params['source_id']
-          self.account_number  = params['external_tenant']
-        end
+            return operation_status[:skipped] if checked_recently?
 
-        def availability_check
-          return if params_missing?
+            status, error_message = connection_status
 
-          return if checked_recently?
+            update_source_and_subresources(status, error_message)
 
-          status, error_message = connection_status
+            logger.availability_check("Completed: Source #{source_id} is #{status}")
 
-          update_source_and_subresources(status, error_message)
+            operation_status[:success]
+          end
 
-          logger.availability_check("Completed: Source #{source_id} is #{status}")
-        end
+          private
 
-        private
+          attr_accessor :metrics, :updates_via_kafka
 
-        def required_params
-          %w[source_id]
-        end
+          def required_params
+            %w[source_id]
+          end
 
-        def params_missing?
-          is_missing = false
-          required_params.each do |attr|
-            if (is_missing = params[attr].blank?)
-              logger.availability_check("Missing #{attr} for the availability_check request [Source ID: #{source_id}]", :error)
-              break
+          def params_missing?
+            is_missing = false
+            required_params.each do |attr|
+              if (is_missing = params[attr].blank?)
+                logger.availability_check("Missing #{attr} for the availability_check request [Source ID: #{source_id}]", :error)
+                break
+              end
             end
-          end
 
-          is_missing
-        end
+            is_missing
+          end
 
-        def checked_recently?
-          checked_recently = if endpoint.present?
-                               endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
-                             elsif application.present?
-                               application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
-                             end
+          def checked_recently?
+            checked_recently = if endpoint.present?
+                                 endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
+                               elsif application.present?
+                                 application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
+                               end
 
-          logger.availability_check("Skipping, last check at #{endpoint.last_checked_at || application.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
+            logger.availability_check("Skipping, last check at #{endpoint.last_checked_at || application.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
 
-          checked_recently
-        end
+            checked_recently
+          end
 
-        def connection_status
-          # we need either an endpoint or application to check the source.
-          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_or_application_not_found]] unless endpoint || application
+          def connection_status
+            # we need either an endpoint or application to check the source.
+            return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_or_application_not_found]] unless endpoint || application
 
-          check_time
-          if endpoint
-            endpoint_connection_check
-          elsif application
-            application_connection_check
+            check_time
+            if endpoint
+              endpoint_connection_check
+            elsif application
+              application_connection_check
+            end
           end
-        end
 
-        def endpoint_connection_check
-          return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
+          def endpoint_connection_check
+            return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
 
-          # call down into the operations pod implementation of `Source#connection_check`
-          connection_check
-        end
-
-        def application_connection_check
-          case application.availability_status
-          when "available"
-            [STATUS_AVAILABLE, nil]
-          when "unavailable"
-            [STATUS_UNAVAILABLE, "Application id #{application.id} unavailable"]
+            # call down into the operations pod implementation of `Source#connection_check`
+            connection_check
           end
-        end
 
-        # @return [Array<String, String|nil] - STATUS_[UN]AVAILABLE, error message
-        def connection_check
-          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
-        end
-
-        def update_source_and_subresources(status, error_message = nil)
-          logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
+          def application_connection_check
+            case application.availability_status
+            when "available"
+              [STATUS_AVAILABLE, nil]
+            when "unavailable"
+              [STATUS_UNAVAILABLE, "Application id #{application.id} unavailable"]
+            end
+          end
 
-          update_source(status)
+          # @return [Array<String, String|nil] - STATUS_[UN]AVAILABLE, error message
+          def connection_check
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
 
-          update_endpoint(status, error_message) if endpoint
-          update_application(status) if application
-        end
+          def update_source_and_subresources(status, error_message = nil)
+            logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
+
+            if updates_via_kafka
+              update_source_by_kafka(status)
+              update_endpoint_by_kafka(status, error_message) if endpoint
+              update_application_by_kafka(status) if application
+            else
+              update_source(status)
+              update_endpoint(status, error_message) if endpoint
+              update_application(status) if application
+            end
+          end
 
-        def update_source(status)
-          source = ::SourcesApiClient::Source.new
-          source.availability_status = status
-          source.last_checked_at     = check_time
-          source.last_available_at   = check_time if status == STATUS_AVAILABLE
+          def update_source(status)
+            source                     = ::SourcesApiClient::Source.new
+            source.availability_status = status
+            source.last_checked_at     = check_time
+            source.last_available_at   = check_time if status == STATUS_AVAILABLE
 
-          api_client.update_source(source_id, source)
-        rescue ::SourcesApiClient::ApiError => e
-          logger.availability_check("Failed to update Source id:#{source_id} - #{e.message}", :error)
-        end
+            sources_api.update_source(source_id, source)
+          rescue ::SourcesApiClient::ApiError => e
+            metrics&.record_error(:sources_api)
+            logger.availability_check("Failed to update Source id:#{source_id} - #{e.message}", :error)
+          end
 
-        def update_endpoint(status, error_message)
-          if endpoint.nil?
-            logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
-            return
+          def update_source_by_kafka(status)
+            availability_status_message(
+              :resource_type => "Source",
+              :resource_id   => source_id,
+              :status        => status
+            )
           end
 
-          endpoint_update = ::SourcesApiClient::Endpoint.new
+          def update_endpoint(status, error_message)
+            if endpoint.nil?
+              logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
+              return
+            end
 
-          endpoint_update.availability_status       = status
-          endpoint_update.availability_status_error = error_message.to_s
-          endpoint_update.last_checked_at           = check_time
-          endpoint_update.last_available_at         = check_time if status == STATUS_AVAILABLE
+            endpoint_update = ::SourcesApiClient::Endpoint.new
 
-          api_client.update_endpoint(endpoint.id, endpoint_update)
-        rescue ::SourcesApiClient::ApiError => e
-          logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
-        end
+            endpoint_update.availability_status       = status
+            endpoint_update.availability_status_error = error_message.to_s
+            endpoint_update.last_checked_at           = check_time
+            endpoint_update.last_available_at         = check_time if status == STATUS_AVAILABLE
 
-        def update_application(status)
-          application_update = ::SourcesApiClient::Application.new
-          application_update.last_checked_at     = check_time
-          application_update.last_available_at   = check_time if status == STATUS_AVAILABLE
+            sources_api.update_endpoint(endpoint.id, endpoint_update)
+          rescue ::SourcesApiClient::ApiError => e
+            metrics&.record_error(:sources_api)
+            logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
+          end
 
-          api_client.update_application(application.id, application_update)
-        rescue ::SourcesApiClient::ApiError => e
-          logger.availability_check("Failed to update Application id: #{application.id} - #{e.message}", :error)
-        end
+          def update_endpoint_by_kafka(status, error_message)
+            if endpoint.nil?
+              logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
+              return
+            end
 
-        def endpoint
-          @endpoint ||= api_client.fetch_default_endpoint(source_id)
-        rescue => e
-          logger.availability_check("Failed to fetch Endpoint for Source #{source_id}: #{e.message}", :error)
-          nil
-        end
+            availability_status_message(
+              :resource_type => "Endpoint",
+              :resource_id   => endpoint.id,
+              :status        => status,
+              :error         => error_message
+            )
+          end
 
-        def authentication
-          @authentication ||= if endpoint.receptor_node.present?
-                                AUTH_NOT_NECESSARY
-                              else
-                                api_client.fetch_authentication(source_id, endpoint)
-                              end
-        rescue => e
-          logger.availability_check("Failed to fetch Authentication for Source #{source_id}: #{e.message}", :error)
-          nil
-        end
+          def update_application(status)
+            application_update                     = ::SourcesApiClient::Application.new
+            application_update.last_checked_at     = check_time
+            application_update.last_available_at   = check_time if status == STATUS_AVAILABLE
 
-        def application
-          @application ||= api_client.fetch_application(source_id)
-        rescue => e
-          logger.availability_check("Failed to fetch Application for Source #{source_id}: #{e.message}", :error)
-          nil
-        end
+            sources_api.update_application(application.id, application_update)
+          rescue ::SourcesApiClient::ApiError => e
+            metrics&.record_error(:sources_api)
+            logger.availability_check("Failed to update Application id: #{application.id} - #{e.message}", :error)
+          end
 
-        def check_time
-          @check_time ||= Time.now.utc
-        end
+          def update_application_by_kafka(status)
+            availability_status_message(
+              :resource_type => "Application",
+              :resource_id   => application.id,
+              :status        => status
+            )
+          end
 
-        def identity
-          @identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => account_number, "user" => {"is_org_admin" => true}}}.to_json)}
-        end
+          def availability_status_message(payload)
+            messaging_client.publish_message(
+              :service => SERVICE_NAME,
+              :message => EVENT_AVAILABILITY_STATUS,
+              :payload => payload.to_json
+            )
+          rescue => err
+            logger.availability_check("Failed to update Application id: #{application.id} - #{err.message}", :error)
+          ensure
+            messaging_client&.close
+          end
+
+          def messaging_client
+            TopologicalInventory::Providers::Common::MessagingClient.default.client
+          end
 
-        def api_client
-          @api_client ||= TopologicalInventory::Providers::Common::Operations::SourcesApiClient.new(identity)
+          def check_time
+            @check_time ||= Time.now.utc
+          end
         end
       end
     end
   end
-  end
 end