topological_inventory-providers-common 1.0.1

data/lib/topological_inventory/providers/common/collectors_pool.rb

@@ -0,0 +1,135 @@
+require "config"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class CollectorsPool
+        SECRET_FILENAME = "credentials".freeze
+
+        def initialize(config_name, metrics, collector_poll_time: 60, thread_pool_size: 2)
+          self.config_name         = config_name
+          self.collector_status    = Concurrent::Map.new
+          self.metrics             = metrics
+          self.collector_poll_time = collector_poll_time
+          self.secrets             = nil
+          self.thread_pool         = Concurrent::FixedThreadPool.new(thread_pool_size)
+          self.updated_at          = {}
+        end
+
+        def run!
+          loop do
+            reload_config
+            reload_secrets
+
+            # Secret is deployed just after config map, we should wait for it
+            queue_collectors if secrets_newer_than_config?
+
+            sleep(5)
+          end
+        end
+
+        def stop!
+          collectors.each_value(&:stop)
+
+          thread_pool.shutdown
+          # Wait for end of collectors to ensure metrics are stopped after them
+          thread_pool.wait_for_termination
+        end
+
+        protected
+
+        attr_accessor :collectors, :collector_poll_time, :collector_status, :thread_pool, :config_name,
+                      :metrics, :secrets, :updated_at
+
+        def reload_config
+          config_file = File.join(path_to_config, "#{sanitize_filename(config_name)}.yml")
+          raise "Configuration file #{config_file} doesn't exist" unless File.exist?(config_file)
+
+          ::Config.load_and_set_settings(config_file)
+        end
+
+        def reload_secrets
+          path = File.join(path_to_secrets, SECRET_FILENAME)
+          raise "Secrets file missing at #{path}" unless File.exists?(path)
+          file         = File.read(path)
+          self.secrets = JSON.parse(file)
+        end
+
+        # @param [Hash] source from Settings
+        # @return [Hash|nil] {"username":, "password":}
+        def secrets_for_source(source)
+          secrets[source.source]
+        end
+
+        def queue_collectors
+          ::Settings.sources.to_a.each do |source|
+            # Skip if collector is running/queued or just finished
+            next if queued_or_updated_recently?(source)
+
+            # Check if secrets for this source are present
+            next if (source_secret = secrets_for_source(source)).nil?
+
+            # Check if necessary endpoint/auth data are not blank (provider specific)
+            next unless source_valid?(source, source_secret)
+
+            collector_status[source.source] = {:status => :queued}
+            # Add source to collector's queue
+            thread_pool.post do
+              begin
+                collector = new_collector(source, source_secret)
+                collector.collect!
+              ensure
+                collector_status[source.source] = {:status => :ready, :last_updated_at => Time.now}
+              end
+            end
+          end
+        end
+
+        def queued_or_updated_recently?(source)
+          return false if collector_status[source.source].nil?
+          return true if collector_status[source.source][:status] == :queued
+
+          if (last_updated_at = collector_status[source.source][:last_updated_at]).nil?
+            # should never happen
+            last_updated_at = Time.now
+            collector_status[source.source] = {:status => :ready, :last_updated_at => last_updated_at}
+          end
+
+          last_updated_at > Time.now - collector_poll_time.to_i
+        end
+
+        def secrets_newer_than_config?
+          return false if ::Settings.updated_at.nil? || secrets["updated_at"].nil?
+
+          updated_at[:config] = Time.parse(::Settings.updated_at)
+          updated_at[:secret] = Time.parse(secrets["updated_at"])
+
+          logger.info("Reloading Sources data => Config [updated_at: #{updated_at[:config].to_s}], Secrets [updated at: #{updated_at[:secret]}]") if updated_at[:config] <= updated_at[:secret]
+
+          updated_at[:config] <= updated_at[:secret]
+        end
+
+        def source_valid?(source, secret)
+          true
+        end
+
+        def path_to_config
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+
+        def path_to_secrets
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+
+        def sanitize_filename(filename)
+          # Remove any character that aren't 0-9, A-Z, or a-z, / or -
+          filename.gsub(/[^0-9A-Z\/\-]/i, '_')
+        end
+
+        def new_collector(source, source_secret)
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/logging.rb

@@ -0,0 +1,22 @@
+require "manageiq/loggers"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class << self
+        attr_writer :logger
+      end
+
+      def self.logger
+        @logger ||= ManageIQ::Loggers::Container.new
+        @logger
+      end
+
+      module Logging
+        def logger
+          TopologicalInventory::Providers::Common.logger
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/endpoint_client.rb

@@ -0,0 +1,62 @@
+require "topological_inventory/providers/common/operations/topology_api_client"
+require "topological_inventory/providers/common/operations/sources_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class EndpointClient
+          include TopologyApiClient
+
+          def initialize(source_id, task_id, identity = nil)
+            self.identity   = identity
+            self.source_id  = source_id
+            self.task_id    = task_id
+          end
+
+          def order_service(service_offering, service_plan, order_params)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          def source_ref_of(endpoint_svc_instance)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          def wait_for_provision_complete(source_id, endpoint_svc_instance, context = {})
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          def provisioned_successfully?(endpoint_svc_instance)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          # Endpoint for conversion of provisioned service's status to
+          # TopologicalInventory Task's status
+          def task_status_for(endpoint_svc_instance)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          private
+
+          attr_accessor :identity, :task_id, :source_id
+
+          def sources_api
+            @sources_api ||= SourcesApiClient.new(identity)
+          end
+
+          def default_endpoint
+            @default_endpoint ||= sources_api.fetch_default_endpoint(source_id)
+          end
+
+          def authentication
+            @authentication ||= sources_api.fetch_authentication(source_id, default_endpoint)
+          end
+
+          def verify_ssl_mode
+            default_endpoint.verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/processor.rb

@@ -0,0 +1,135 @@
+require "topological_inventory/providers/common/operations/topology_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class Processor
+          include Logging
+          include TopologyApiClient
+
+          SLEEP_POLL = 10
+          POLL_TIMEOUT = 1800
+
+          def self.process!(message)
+            model, method = message.headers['message_type'].to_s.split(".")
+            new(model, method, message.payload).process
+          end
+
+          # @param payload [Hash] https://github.com/ManageIQ/topological_inventory-api/blob/master/app/controllers/api/v0/service_plans_controller.rb#L32-L41
+          def initialize(model, method, payload, metrics = nil)
+            self.model           = model
+            self.method          = method
+            self.params          = payload["params"]
+            self.identity        = payload["request_context"]
+            self.metrics         = metrics
+          end
+
+          def process
+            logger.info("Processing #{model}##{method} [#{params}]...")
+            result = order_service(params)
+            logger.info("Processing #{model}##{method} [#{params}]...Complete")
+
+            result
+          end
+
+          private
+
+          attr_accessor :identity, :model, :method, :metrics, :params
+
+          def endpoint_client(source_id, task_id, identity)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass as kind of TopologicalInventory::Providers::Common::EndpointClient class"
+          end
+
+          def order_service(params)
+            task_id, service_offering_id, service_plan_id, order_params = params.values_at("task_id", "service_offering_id", "service_plan_id", "order_params")
+
+            service_plan     = topology_api_client.show_service_plan(service_plan_id) if service_plan_id.present?
+            service_offering_id = service_plan.service_offering_id if service_offering_id.nil? && service_plan.present?
+            service_offering = topology_api_client.show_service_offering(service_offering_id)
+
+            source_id        = service_offering.source_id
+            client = endpoint_client(source_id, task_id, identity)
+
+            logger.info("Ordering #{service_offering.name}...")
+            remote_service_instance = client.order_service(service_offering, service_plan.presence, order_params)
+            logger.info("Ordering #{service_offering.name}...Complete")
+
+            poll_order_complete_thread(task_id, source_id, remote_service_instance)
+          rescue StandardError => err
+            metrics&.record_error
+            logger.error("[Task #{task_id}] Ordering error: #{err}\n#{err.backtrace.join("\n")}")
+            update_task(task_id, :state => "completed", :status => "error", :context => {:error => err.to_s})
+          end
+
+          def poll_order_complete_thread(task_id, source_id, remote_svc_instance)
+            Thread.new do
+              begin
+                poll_order_complete(task_id, source_id, remote_svc_instance)
+              rescue StandardError => err
+                logger.error("[Task #{task_id}] Waiting for complete: #{err}\n#{err.backtrace.join("\n")}")
+                update_task(task_id, :state => "completed", :status => "warn", :context => {:error => err.to_s})
+              end
+            end
+          end
+
+          def poll_order_complete(task_id, source_id, remote_svc_instance)
+            client = endpoint_client(source_id, task_id, identity)
+
+            context = {
+              :service_instance => {
+                :source_id  => source_id,
+                :source_ref => client.source_ref_of(remote_svc_instance)
+              }
+            }
+
+            remote_svc_instance = client.wait_for_provision_complete(task_id, remote_svc_instance, context)
+
+            if client.provisioned_successfully?(remote_svc_instance)
+              if (service_instance = load_topological_svc_instance(source_id, client.source_ref_of(remote_svc_instance))).present?
+                context[:service_instance][:id] = service_instance.id
+                context[:service_instance][:url] = svc_instance_url(service_instance)
+              else
+                logger.warn("Failed to get service_instance API URL (endpoint's service instance: #{remote_svc_instance.inspect})")
+              end
+            end
+            update_task(task_id, :state => "completed", :status => client.task_status_for(remote_svc_instance), :context => context)
+          end
+
+          def load_topological_svc_instance(source_id, source_ref)
+            api = topology_api_client.api_client
+
+            count = 0
+            timeout_count = POLL_TIMEOUT / SLEEP_POLL
+
+            header_params = { 'Accept' => api.select_header_accept(['application/json']) }
+            query_params = { :'source_id' => source_id, :'source_ref' => source_ref }
+            return_type = 'ServiceInstancesCollection'
+
+            service_instance = nil
+            loop do
+              data, _status_code, _headers = api.call_api(:GET, "/service_instances",
+                                                          :header_params => header_params,
+                                                          :query_params  => query_params,
+                                                          :auth_names    => ['UserSecurity'],
+                                                          :return_type   => return_type)
+
+              service_instance = data.data&.first if data.meta.count > 0
+              break if service_instance.present?
+
+              break if (count += 1) >= timeout_count
+
+              sleep(SLEEP_POLL) # seconds
+            end
+
+            if service_instance.nil?
+              logger.error("Failed to find service_instance by source_id [#{source_id}] source_ref [#{source_ref}]")
+            end
+
+            service_instance
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/sources_api_client.rb

@@ -0,0 +1,85 @@
+require "sources-api-client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class SourcesApiClient < ::SourcesApiClient::ApiClient
+          INTERNAL_API_PATH = '//internal/v1.0'.freeze
+
+          def initialize(identity = nil)
+            super(::SourcesApiClient::Configuration.default)
+            self.identity = identity
+            self.api = init_default_api
+          end
+
+          def init_default_api
+            default_headers.merge!(identity) if identity.present?
+            ::SourcesApiClient::DefaultApi.new(self)
+          end
+
+          def fetch_default_endpoint(source_id)
+            endpoints = api.list_source_endpoints(source_id)&.data || []
+            endpoint = endpoints.find(&:default)
+
+            raise "Sources API: Endpoint not found! (source id: #{source_id})" if endpoint.nil?
+
+            endpoint
+          end
+
+          def fetch_authentication(source_id, default_endpoint = nil)
+            endpoint = default_endpoint || fetch_default_endpoint(source_id)
+            return if endpoint.nil?
+
+            endpoint_authentications = api.list_endpoint_authentications(endpoint.id.to_s).data || []
+            return if endpoint_authentications.empty?
+
+            auth_id = endpoint_authentications.first.id
+            fetch_authentication_with_password(auth_id)
+          end
+
+          private
+
+          attr_accessor :identity, :api, :custom_base_path
+
+          def fetch_authentication_with_password(auth_id)
+            on_internal_api do
+              local_var_path = "/authentications/#{auth_id}"
+
+              query_params = "expose_encrypted_attribute[]=password"
+
+              header_params = { 'Accept' => select_header_accept(['application/json']) }
+              return_type = 'Authentication'
+              data, _, _ = call_api(:GET, local_var_path,
+                                    :header_params => header_params,
+                                    :query_params  => query_params,
+                                    :auth_names    => ['UserSecurity'],
+                                    :return_type   => return_type)
+              data
+            end
+          end
+
+          def build_request_url(path)
+            # Add leading and trailing slashes to path
+            path = "/#{path}".gsub(/\/+/, '/')
+            URI.encode((custom_base_url || @config.base_url) + path)
+          end
+
+          def custom_base_url
+            return nil if custom_base_path.nil?
+
+            url = "#{@config.scheme}://#{[@config.host, custom_base_path].join('/').gsub(/\/+/, '/')}".sub(/\/+\z/, '')
+            URI.encode(url)
+          end
+
+          def on_internal_api
+            self.custom_base_path = INTERNAL_API_PATH
+            yield
+          ensure
+            self.custom_base_path = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/topology_api_client.rb

@@ -0,0 +1,28 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        module TopologyApiClient
+          def topology_api_client
+            @topology_api_client ||=
+              begin
+                api_client = TopologicalInventoryApiClient::ApiClient.new
+                api_client.default_headers.merge!(identity) if identity.present?
+                TopologicalInventoryApiClient::DefaultApi.new(api_client)
+              end
+          end
+
+          def update_task(task_id, state:, status:, context:)
+            task = TopologicalInventoryApiClient::Task.new("state" => state, "status" => status, "context" => context)
+            topology_api_client.update_task(task_id, task)
+          end
+
+          def svc_instance_url(service_instance)
+            rest_api_path = '/service_instances/{id}'.sub('{' + 'id' + '}', service_instance&.id.to_s)
+            topology_api_client.api_client.build_request(:GET, rest_api_path).url
+          end
+        end
+      end
+    end
+  end
+end