katello 3.14.0.rc1 → 3.14.0.rc2

data/app/lib/katello/resources/candlepin.rb

@@ -149,6 +149,12 @@ module Katello
         end # class << self
       end # UpstreamCandlepinResource
 
+      module AdminResource
+        def path
+          "#{self.prefix}/admin"
+        end
+      end
+
       module ConsumerResource
         def path(id = nil)
           "#{self.prefix}/consumers/#{id}"

data/app/lib/katello/resources/candlepin/admin.rb

@@ -0,0 +1,21 @@
+module Katello
+  module Resources
+    module Candlepin
+      class Admin < CandlepinResource
+        extend AdminResource
+
+        def self.queues
+          response = get("#{path}/queues")
+          JSON.parse(response.body).first
+        end
+
+        def self.queue_depth(queue_name)
+          queue = queues.select { |q| q['queueName'] == queue_name }
+          queue['pendingMessageCount'].to_i
+        rescue
+          nil # be graceful when candlepin is down
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/util/package_clause_generator.rb

@@ -3,6 +3,16 @@ module Katello
     class PackageClauseGenerator
       include Util::FilterClauseGenerator
 
+      def copy_clause
+        clauses = super
+        {"$and" => [{"is_modular" => false}, clauses]} unless clauses.blank?
+      end
+
+      def remove_clause
+        clauses = super
+        {"$and" => [{"is_modular" => false}, clauses]} unless clauses.blank?
+      end
+
       protected
 
       def fetch_filters

data/app/lib/katello/util/support.rb

@@ -72,6 +72,25 @@ module Katello
         stringify(params.keys) - stringify(rule.keys)
       end
 
+      def self.with_db_connection(logger = Rails.logger)
+        yield
+      rescue PG::ConnectionBad, ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished => e
+        logger.error(e.message)
+        logger.error("Lost database connection. Attempting reconnect.")
+
+        active_record_retry_connect
+
+        retry
+      end
+
+      def self.active_record_retry_connect
+        sleep 3
+        ActiveRecord::Base.connection.reconnect!
+      rescue
+        logger.error("Trying to reconnect to the database.")
+        retry
+      end
+
       # Used for retrying active record transactions when race conditions could cause
       #  RecordNotUnique exceptions
       def self.active_record_retry(retries = 3)

data/app/models/katello/glue/pulp/repo.rb

@@ -44,19 +44,6 @@ module Katello
             !repo.distributors_match?(repo_details["distributors"], smart_proxy)
           end
         end
-
-        def self.build_override_config(options)
-          config = {}
-          if options[:filters].present? && (options[:solve_dependencies] || options[:resolve_dependencies])
-            if Setting[:dependency_solving_algorithm] == 'greedy'
-              config[:recursive] = true
-            else
-              config[:recursive_conservative] = true
-            end
-          end
-
-          config
-        end
       end
     end
 

data/app/models/katello/host/subscription_facet.rb

@@ -127,7 +127,8 @@ module Katello
             guest_ids = self.candlepin_consumer.virtual_guests.pluck(:id)
           end
 
-          subscription_facets = SubscriptionFacet.where(:host_id => guest_ids)
+          subscription_facets = SubscriptionFacet.where(:host_id => guest_ids).
+                                                  where("hypervisor_host_id != ? OR hypervisor_host_id is NULL", self.host.id)
           subscription_facets.update_all(:hypervisor_host_id => self.host.id)
         elsif (virtual_host = self.candlepin_consumer.virtual_host)
           self.hypervisor_host = virtual_host

data/app/models/katello/kt_environment.rb

@@ -76,6 +76,7 @@ module Katello
 
     scoped_search :on => :name, :complete_value => true
     scoped_search :on => :organization_id, :complete_value => true, :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER
+    scoped_search :on => :id, :complete_value => true, :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER
 
     def library?
       self.library

data/app/models/katello/ping.rb

@@ -2,10 +2,12 @@ module Katello
   class Ping
     OK_RETURN_CODE = 'ok'.freeze
     FAIL_RETURN_CODE = 'FAIL'.freeze
+    WARN_RETURN_CODE = 'WARN'.freeze
     PACKAGES = %w(katello candlepin pulp qpid foreman tfm hammer).freeze
+
     class << self
       def services(capsule_id = nil)
-        services = [:pulp, :pulp_auth, :candlepin, :candlepin_auth, :foreman_tasks]
+        services = [:pulp, :pulp_auth, :candlepin, :candlepin_auth, :foreman_tasks, :katello_events, :candlepin_events]
         services += [:pulp3] if fetch_proxy(capsule_id)&.pulp3_enabled?
         services
       end
@@ -25,11 +27,13 @@ module Katello
         ping_pulp_with_auth(result[:pulp_auth], result[:pulp][:status]) if result.include?(:pulp_auth)
         ping_candlepin_with_auth(result[:candlepin_auth]) if result.include?(:candlepin_auth)
         ping_foreman_tasks(result[:foreman_tasks]) if result.include?(:foreman_tasks)
+        ping_katello_events(result[:katello_events]) if result.include?(:katello_events)
+        ping_candlepin_events(result[:candlepin_events]) if result.include?(:candlepin_events)
 
         # set overall status result code
         result = {:services => result}
         result[:services].each_value do |v|
-          result[:status] = v[:status] == OK_RETURN_CODE ? OK_RETURN_CODE : FAIL_RETURN_CODE
+          result[:status] = [OK_RETURN_CODE, WARN_RETURN_CODE].include?(v[:status]) ? OK_RETURN_CODE : FAIL_RETURN_CODE
         end
         result
       end
@@ -41,6 +45,34 @@ module Katello
         }
       end
 
+      def daemon_status_message(status)
+        "#{status[:processed_count]} Processed, #{status[:failed_count]} Failed, #{status[:queue_depth]} in queue"
+      end
+
+      def ping_katello_events(result)
+        exception_watch(result) do
+          status = Katello::EventMonitor::PollerThread.status
+
+          if status[:queue_depth] && status[:queue_depth] > 1000
+            result[:status] = WARN_RETURN_CODE
+          end
+
+          result[:message] = daemon_status_message(status)
+        end
+      end
+
+      def ping_candlepin_events(result)
+        exception_watch(result) do
+          status = Katello::CandlepinEventListener.status
+
+          if status[:queue_depth] && status[:queue_depth] > 1000
+            result[:status] = WARN_RETURN_CODE
+          end
+
+          result[:message] = daemon_status_message(status)
+        end
+      end
+
       def ping_pulp3_without_auth(service_result, capsule_id)
         exception_watch(service_result) do
           Katello::Ping.pulp3_without_auth(fetch_proxy(capsule_id).pulp3_url("api/v3"))
@@ -104,8 +136,8 @@ module Katello
       # check for exception - set the result code properly
       def exception_watch(result)
         start = Time.new
-        yield
         result[:status] = OK_RETURN_CODE
+        yield
         result[:duration_ms] = ((Time.new - start) * 1000).round.to_s
         result
       rescue => e

data/app/models/katello/repository.rb

@@ -140,7 +140,6 @@ module Katello
     scoped_search :on => :distribution_variant, :complete_value => true
     scoped_search :on => :distribution_bootable, :complete_value => true
     scoped_search :on => :distribution_uuid, :complete_value => true
-    scoped_search :on => :ignore_global_proxy, :relation => :root, :complete_value => true
     scoped_search :on => :redhat, :complete_value => { :true => true, :false => false }, :ext_method => :search_by_redhat
     scoped_search :on => :container_repository_name, :complete_value => true
     scoped_search :on => :description, :relation => :root, :only_explicit => true
@@ -157,7 +156,7 @@ module Katello
              :content_type, :product_id, :checksum_type, :docker_upstream_name, :mirror_on_sync, :"mirror_on_sync?",
              :download_policy, :verify_ssl_on_sync, :"verify_ssl_on_sync?", :upstream_username, :upstream_password,
              :ostree_upstream_sync_policy, :ostree_upstream_sync_depth, :deb_releases, :deb_components, :deb_architectures,
-             :ignore_global_proxy, :ssl_ca_cert_id, :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_id, :ssl_client_key_id,
+             :ssl_ca_cert_id, :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_id, :ssl_client_key_id,
              :ssl_client_key, :ignorable_content, :description, :docker_tags_whitelist, :ansible_collection_requirements, :http_proxy_policy, :http_proxy_id, :to => :root
 
     delegate :content_id, to: :root, allow_nil: true
@@ -315,6 +314,10 @@ module Katello
       found
     end
 
+    def siblings
+      content_view_version.archived_repos.where.not(:id => id)
+    end
+
     def clones
       self.root.repositories.where.not(:id => library_instance_id || id)
     end

data/app/models/katello/rhsm_fact_parser.rb

@@ -23,7 +23,7 @@ module Katello
     def get_facts_for_interface(interface)
       {
         'link' => true,
-        'macaddress' => facts["net.interface.#{interface}.mac_address"],
+        'macaddress' => get_rhsm_mac(interface),
         'ipaddress' => get_rhsm_ip(interface)
       }
     end
@@ -107,5 +107,10 @@ module Katello
       ip = facts["net.interface.#{interface}.ipv4_address"]
       Net::Validations.validate_ip(ip) ? ip : nil
     end
+
+    def get_rhsm_mac(interface)
+      # if slave then permanent_mac_address contains the physical mac
+      facts["net.interface.#{interface}.permanent_mac_address"] || facts["net.interface.#{interface}.mac_address"]
+    end
   end
 end

data/app/models/katello/root_repository.rb

@@ -290,7 +290,7 @@ module Katello
 
     def pulp_update_needed?
       changeable_attributes = %w(url unprotected checksum_type docker_upstream_name download_policy mirror_on_sync verify_ssl_on_sync
-                                 upstream_username upstream_password ostree_upstream_sync_policy ostree_upstream_sync_depth ignore_global_proxy ignorable_content
+                                 upstream_username upstream_password ostree_upstream_sync_policy ostree_upstream_sync_depth ignorable_content
                                  ssl_ca_cert_id ssl_client_cert_id ssl_client_key_id http_proxy_policy http_proxy_id)
       changeable_attributes += %w(name container_repository_name docker_tags_whitelist) if docker?
       changeable_attributes += %w(deb_releases deb_components deb_architectures gpg_key_id) if deb?

data/app/services/katello/candlepin/consumer.rb

@@ -100,9 +100,10 @@ module Katello
       end
 
       def virtual_guests
+        return @virtual_guests unless @virtual_guests.nil?
         return [] if self.uuid.nil?
         guest_uuids = Resources::Candlepin::Consumer.virtual_guests(self.uuid).map { |guest| guest['uuid'] }
-        ::Host.joins(:subscription_facet).where("#{Katello::Host::SubscriptionFacet.table_name}.uuid" => guest_uuids)
+        @virtual_guests = ::Host.joins(:subscription_facet).where("#{Katello::Host::SubscriptionFacet.table_name}.uuid" => guest_uuids)
       end
 
       def virtual_host

data/app/{lib/actions/candlepin/import_pool_handler.rb → services/katello/candlepin/event_handler.rb}

@@ -1,6 +1,6 @@
-module Actions
+module Katello
   module Candlepin
-    class ImportPoolHandler
+    class EventHandler
       attr_reader :message_handler
 
       def initialize(logger)

data/app/services/katello/candlepin_event_listener.rb

@@ -0,0 +1,106 @@
+module Katello
+  class CandlepinEventListener
+    PROCESSED_COUNT_CACHE_KEY = 'candlepin_events_processed'.freeze
+    FAILED_COUNT_CACHE_KEY = 'candlepin_events_failed'.freeze
+    AMQP_QUEUE_NAME = 'event.org.candlepin.audit.AMQPBusPublisher'.freeze
+
+    CandlepinEvent = Struct.new(:message_id, :subject, :content)
+
+    @logger = ::Foreman::Logging.logger('katello/candlepin_events')
+    @failed_count = 0
+    @processed_count = 0
+
+    def self.start_service
+      loop do
+        begin
+          result = Katello::CandlepinListeningService.instance.start
+
+          break if result == :connected
+
+          @logger.info("Attempting to restart Candlepin Listening Service")
+          sleep 5
+        end
+      end
+    end
+
+    def self.run
+      @thread.kill if @thread
+
+      # run in own thread so connecting to qpid won't block the main process
+      @thread = Thread.new do
+        Rails.application.executor.wrap do
+          initialize_listening_service
+          start_service
+
+          Katello::CandlepinListeningService.instance.poll_for_messages do |message|
+            if message[:result]
+              result = message[:result]
+              event = CandlepinEvent.new(result.message_id, result.subject, result.content)
+              act_on_event(event)
+            elsif message[:error]
+              @logger.error("Disconnected from Candlepin Listening Service, reconnecting")
+              start_service
+            end
+          end
+        end
+      end
+    rescue => e
+      @logger.error("Fatal error in Candlepin Listening Service")
+      close
+      raise e
+    end
+
+    def self.status
+      {
+        processed_count: Rails.cache.fetch(PROCESSED_COUNT_CACHE_KEY) { @processed_count },
+        failed_count: Rails.cache.fetch(FAILED_COUNT_CACHE_KEY) { @failed_count },
+        queue_depth: Katello::Resources::Candlepin::Admin.queue_depth(AMQP_QUEUE_NAME)
+      }
+    end
+
+    def self.reset_status
+      Rails.cache.write(PROCESSED_COUNT_CACHE_KEY, 0)
+      Rails.cache.write(FAILED_COUNT_CACHE_KEY, 0)
+    end
+
+    def self.act_on_event(event)
+      ::Katello::Util::Support.with_db_connection(@logger) do
+        ::Katello::Candlepin::EventHandler.new(@logger).handle(event)
+      end
+      @processed_count += 1
+
+      Rails.cache.write(PROCESSED_COUNT_CACHE_KEY, @processed_count, expires_in: 24.hours)
+    rescue => e
+      @failed_count += 1
+      Rails.cache.write(FAILED_COUNT_CACHE_KEY, @failed_count, expires_in: 24.hours)
+      @logger.error("Error handling Candlepin event")
+      @logger.error(e.message)
+      @logger.error(e.backtrace.join("\n"))
+    end
+
+    def self.configured?
+      SETTINGS[:katello].key?(:qpid) &&
+        SETTINGS[:katello][:qpid].key?(:url) &&
+        SETTINGS[:katello][:qpid].key?(:subscriptions_queue_address)
+    end
+
+    def self.initialize_listening_service
+      if configured?
+        Katello::CandlepinListeningService.initialize_service(@logger,
+                                           SETTINGS[:katello][:qpid][:url],
+                                           SETTINGS[:katello][:qpid][:subscriptions_queue_address])
+      else
+        fail("Katello has not been configured for qpid.url and qpid.subscriptions_queue_address")
+      end
+    rescue => e
+      @logger.error(e.message)
+      @logger.error(e.backtrace)
+    end
+
+    def self.close
+      Katello::CandlepinListeningService.close
+      @thread.kill if @thread
+      reset_status
+    end
+  end
+end

data/app/services/katello/candlepin_listening_service.rb

@@ -0,0 +1,91 @@
+module Katello
+  class CandlepinListeningService
+    TIMEOUT = Qpid::Messaging::Duration::SECOND
+    NO_MESSAGE_AVAILABLE_ERROR_TYPE = 'NoMessageAvailable'.freeze
+    SLEEP_INTERVAL = 3
+
+    class ConnectionError < StandardError
+    end
+
+    class << self
+      attr_reader :instance
+
+      def initialize_service(logger, url, address)
+        @instance = self.new(logger, url, address)
+      end
+
+      def close
+        @instance.close if @instance
+        @instance = nil
+      end
+    end
+
+    def initialize(logger, url, address)
+      @url = url
+      @address = address
+      @connection = create_connection
+      @logger = logger
+    end
+
+    def create_connection
+      Qpid::Messaging::Connection.new(:url => @url, :options => {:transport => 'ssl'})
+    end
+
+    def close
+      @logger.info("Stopping Candlepin Listening Service")
+      @thread.kill if @thread
+      @connection.close
+    end
+
+    def retrieve
+      result = @receiver.fetch(TIMEOUT)
+      result
+    rescue => e
+      if e.class.name.include? "TransportFailure"
+        raise ConnectionError, "failed to connect to #{@url}"
+      else
+        raise e unless e.class.name.include? NO_MESSAGE_AVAILABLE_ERROR_TYPE
+      end
+    ensure
+      safe_release(result) if result
+    end
+
+    def safe_release(message)
+      @session.acknowledge(:message => message, :sync => true)
+    rescue => e
+      @session.release(message)
+      raise e
+    end
+
+    def start
+      unless @connection.open?
+        @connection.open
+        @session = @connection.create_session
+        @receiver = @session.create_receiver(@address)
+        @logger.info("Candlepin Event Listener started")
+      end
+
+      :connected
+    rescue => e
+      raise e unless e.class.name.include? "TransportFailure"
+    end
+
+    def fetch_message
+      {:result => retrieve, :error => nil}
+    rescue ConnectionError => e
+      {:result => nil, :error => e.message}
+    end
+
+    def poll_for_messages
+      @thread.kill if @thread
+      @thread = Thread.new do
+        loop do
+          message = fetch_message
+          yield(message) if block_given?
+
+          sleep SLEEP_INTERVAL if message[:result].nil? && message[:error].nil?
+        end
+      end
+    end
+  end
+end