katello 3.15.1 → 3.16.0.rc2.1

data/app/models/katello/task_status.rb

@@ -151,7 +151,7 @@ module Katello
     end
 
     # TODO: break up method
-    # rubocop:disable MethodLength
+    # rubocop:disable Metrics/MethodLength
     def message
       # Retrieve a text message that may be rendered for a task's status.  This is used in various places,
       # such as System Event history.

data/app/models/katello/trace_status.rb

@@ -35,7 +35,7 @@ module Katello
     end
 
     def to_status(_options = {})
-      if host.host_traces.where(:app_type => "static").any?
+      if host.host_traces.reboot_required.any?
         REQUIRE_REBOOT
       elsif host.host_traces.where.not(:app_type => "session").any?
         REQUIRE_PROCESS_RESTART
@@ -46,8 +46,8 @@ module Katello
 
     def relevant?(_options = {})
       # traces cannot be reported from hosts lower than el7
-      return false if host.operatingsystem.try(:major).to_i.between?(1, 6)
-      host.content_facet.try(:uuid)
+      return false if host.operatingsystem.try(:major).to_i.between?(1, 6) || !host.content_facet&.tracer_installed?
+      host.content_facet.try(:uuid).present?
     end
   end
 end

data/app/models/setting/content.rb

@@ -20,6 +20,7 @@ class Setting::Content < Setting
 
     proxy_download_policies = proc { deprecate_background(hashify_parameters(::SmartProxy::DOWNLOAD_POLICIES)) }
     dependency_solving_options = proc { hashify_parameters(['conservative', 'greedy']) }
+    cdn_ssl_versions = proc { hashify_parameters(Katello::Resources::CDN::SUPPORTED_SSL_VERSIONS) }
     http_proxy_select = [{
       name: _("HTTP Proxies"),
       class: 'HttpProxy',
@@ -34,6 +35,9 @@ class Setting::Content < Setting
                       nil,
                       collection: proc { http_proxy_select }, include_blank: N_("no global default")
               ),
+      self.set('cdn_ssl_version', N_("SSL version used to communicate with the CDN"),
+               nil, N_('CDN SSL version'), nil,
+               :collection => cdn_ssl_versions),
       self.set('katello_default_provision', N_("Default provisioning template for Operating Systems created from synced content"),
                'Kickstart default', N_('Default synced OS provisioning template'),
                nil, :collection => proc { katello_template_setting_values("provision") }
@@ -141,13 +145,15 @@ class Setting::Content < Setting
                   "as long as the registering DMI UUID is not used by another host."),
                true, N_('Host Profile Assume')),
       self.set('host_tasks_workers_pool_size', N_("Amount of workers in the pool to handle the execution of host-related tasks. When set to 0, the default queue will be used instead. Restart of the dynflowd/foreman-tasks service is required."),
-               5, N_('Host Tasks Workers Pool Size'))
+               5, N_('Host Tasks Workers Pool Size')),
+      self.set('applicability_batch_size', N_("Number of host applicability calculations to process per task."),
+               50, N_('Applicability Batch Size'))
     ]
   end
 
   def self.load_defaults
     BLANK_ATTRS.concat %w(register_hostname_fact default_location_subscribed_hosts
-                          default_location_puppet_content content_default_http_proxy host_dmi_uuid_duplicates)
+                          default_location_puppet_content content_default_http_proxy host_dmi_uuid_duplicates cdn_ssl_version)
     super
   end
 

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -0,0 +1,135 @@
+module Katello
+  module Applicability
+    class ApplicableContentHelper
+      attr_accessor :content_facet, :content_unit_class, :bound_library_instance_repos
+
+      def initialize(content_facet, content_unit_class, bound_library_instance_repos)
+        self.content_facet = content_facet
+        self.content_unit_class = content_unit_class
+        self.bound_library_instance_repos = bound_library_instance_repos
+      end
+
+      def calculate_and_import
+        if self.bound_library_instance_repos.any?
+          to_add, to_remove = applicable_differences
+          ActiveRecord::Base.transaction do
+            insert(to_add) unless to_add.blank?
+            remove(to_remove) unless to_remove.blank?
+          end
+        end
+      end
+
+      def fetch_content_ids
+        if self.content_unit_class == ::Katello::Erratum
+          fetch_errata_content_ids
+        elsif self.content_unit_class == ::Katello::ModuleStream
+          fetch_module_stream_content_ids
+        else
+          fetch_rpm_content_ids
+        end
+      end
+
+      def fetch_errata_content_ids
+        # Query for all Errata ids that are attached to the host's applicable packages
+        query = 'SELECT DISTINCT katello_repository_errata.erratum_id AS id FROM katello_repository_errata
+                     INNER JOIN katello_erratum_packages
+                        ON katello_repository_errata.erratum_id = katello_erratum_packages.erratum_id
+                     INNER JOIN katello_rpms
+                        ON katello_rpms.nvra = katello_erratum_packages.nvrea
+                     INNER JOIN katello_content_facet_applicable_rpms
+                        ON katello_content_facet_applicable_rpms.rpm_id = katello_rpms.id
+                     WHERE katello_content_facet_applicable_rpms.content_facet_id = :content_facet_id
+                        AND katello_repository_errata.repository_id IN (:repo_ids)'
+
+        return Katello::Erratum.find_by_sql([query, { content_facet_id: content_facet.id, repo_ids: self.bound_library_instance_repos }]).map(&:id)
+      end
+
+      def fetch_module_stream_content_ids
+        # Query for all applicable module stream ids
+        query = 'SELECT DISTINCT katello_repository_module_streams.module_stream_id AS id FROM katello_repository_module_streams
+                     INNER JOIN katello_module_stream_rpms
+                        ON katello_repository_module_streams.module_stream_id = katello_module_stream_rpms.module_stream_id
+                     INNER JOIN katello_rpms
+                        ON katello_rpms.id = katello_module_stream_rpms.rpm_id
+                     INNER JOIN katello_content_facet_applicable_rpms
+                        ON katello_content_facet_applicable_rpms.rpm_id = katello_rpms.id
+                     WHERE katello_content_facet_applicable_rpms.content_facet_id = :content_facet_id
+                        AND katello_repository_module_streams.repository_id IN (:repo_ids)'
+
+        return Katello::ModuleStream.find_by_sql([query, { content_facet_id: content_facet.id, repo_ids: self.bound_library_instance_repos }]).map(&:id)
+      end
+
+      def fetch_rpm_content_ids
+        # Query for applicable RPM ids
+        # -> Include all non-modular rpms or rpms that exist within installed module streams
+        enabled_module_stream_ids = ::Katello::ModuleStream.
+          joins("inner join katello_available_module_streams on
+            katello_module_streams.name = katello_available_module_streams.name and
+            katello_module_streams.stream = katello_available_module_streams.stream").
+          joins("inner join katello_host_available_module_streams on
+            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
+          where("katello_host_available_module_streams.host_id = :content_facet_id and
+            katello_host_available_module_streams.status = 'enabled'",
+            :content_facet_id => self.content_facet.host.id).select(:id)
+
+        ::Katello::Rpm.
+          joins("INNER JOIN katello_repository_rpms ON
+            katello_rpms.id = katello_repository_rpms.rpm_id").
+          joins("INNER JOIN katello_installed_packages ON
+            katello_rpms.name = katello_installed_packages.name AND
+            katello_rpms.arch = katello_installed_packages.arch AND
+            katello_rpms.evr > katello_installed_packages.evr").
+          joins("LEFT JOIN katello_module_stream_rpms ON
+            katello_rpms.id = katello_module_stream_rpms.rpm_id").
+          joins("INNER JOIN katello_host_installed_packages ON
+            katello_installed_packages.id = katello_host_installed_packages.installed_package_id").
+          where("katello_repository_rpms.repository_id in (:bound_library_repos)",
+            :bound_library_repos => self.bound_library_instance_repos).
+          where("katello_host_installed_packages.host_id = :content_facet_id",
+            :content_facet_id => self.content_facet.host.id).
+          where("katello_module_stream_rpms.module_stream_id is null or
+            katello_module_stream_rpms.module_stream_id in (:enabled_module_streams)",
+            :enabled_module_streams => enabled_module_stream_ids).pluck(:id).uniq
+      end
+
+      def applicable_differences
+        consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
+        content_ids = fetch_content_ids
+
+        to_remove = consumer_ids - content_ids
+        to_add = content_ids - consumer_ids
+
+        [to_add, to_remove]
+      end
+
+      def insert(applicable_ids)
+        unless applicable_ids.empty?
+          inserts = applicable_ids.map { |applicable_id| "(#{applicable_id.to_i}, #{content_facet.id.to_i})" }
+          sql = "INSERT INTO #{content_facet_association_class.table_name} (#{content_unit_association_id}, content_facet_id) VALUES #{inserts.join(', ')}"
+          ActiveRecord::Base.connection.execute(sql)
+        end
+      end
+
+      def remove(applicable_ids)
+        content_facet_association_class.where(:content_facet_id => content_facet.id, content_unit_association_id => applicable_ids).delete_all
+      end
+
+      def content_unit_association_id
+        "#{content_unit_class.name.demodulize.underscore}_id".to_sym
+      end
+
+      def content_facet_association_class
+        # Example: ContentFacetErratum
+        self.content_unit_class.content_facet_association_class
+      end
+
+      def content_units
+        content_unit_class.name.demodulize.pluralize.underscore.to_sym
+      end
+
+      def applicable_units
+        "applicable_#{content_units}".to_sym
+      end
+    end
+  end
+end

data/app/services/katello/applicable_host_queue.rb

@@ -0,0 +1,21 @@
+module Katello
+  class ApplicableHostQueue
+    def self.batch_size
+      ::Setting::Content.find_by(name: "applicability_batch_size").value
+    end
+
+    def self.queue_depth
+      ::Katello::HostQueueElement.all.size
+    end
+
+    def self.push_host(host_id)
+      HostQueueElement.create!({ host_id: host_id })
+    end
+
+    def self.pop_hosts(amount = self.batch_size)
+      queue = HostQueueElement.group(:host_id).select("MIN(created_at) as created_at, host_id").limit(amount)
+      HostQueueElement.where(host_id: queue.map(&:host_id)).delete_all
+      queue
+    end
+  end
+end

data/app/services/katello/candlepin/consumer.rb

@@ -84,7 +84,7 @@ module Katello
         filtered = entitlements
         filtered = filtered.select { |ent| ent['pool']['id'].to_s == pool_id.to_s } if pool_id
 
-        if quantities && quantities.any?
+        if quantities&.any?
           quantities.map!(&:to_s)
           filtered = quantities.map do |quantity|
             index = filtered.index { |ent| ent['quantity'].to_s == quantity }

data/app/services/katello/candlepin/message_handler.rb

@@ -13,12 +13,11 @@ module Katello
       end
 
       def content
-        JSON.parse(@message.content)
+        @content ||= JSON.parse(message.content)
       end
 
       def event_data
-        data = content['eventData']
-        data ? JSON.parse(data) : {}
+        @event_data ||= (data = content['eventData']) ? JSON.parse(data) : {}
       end
 
       def status

data/app/services/katello/candlepin_event_listener.rb

@@ -2,29 +2,43 @@ module Katello
   class CandlepinEventListener
     STATUS_CACHE_KEY = 'candlepin_events_status'.freeze
 
-    CandlepinEvent = Struct.new(:message_id, :subject, :content)
+    Event = Struct.new(:subject, :content)
+
+    cattr_accessor :client_factory
 
-    @logger = ::Foreman::Logging.logger('katello/candlepin_events')
     @failed_count = 0
     @processed_count = 0
 
-    def self.run
-      initialize_listening_service
+    def self.logger
+      ::Foreman::Logging.logger('katello/candlepin_events')
+    end
+
+    def self.running?
+      @running == true && @client&.running?
+    end
+
+    def self.close
+      return unless running?
 
-      result = Katello::CandlepinListeningService.instance.start
-      return unless result == :connected
+      logger.info("Closing candlepin event listener")
+      @client&.close
+      reset
+    end
 
-      Katello::CandlepinListeningService.instance.poll_for_messages do |message|
-        if message[:result]
-          result = message[:result]
-          event = CandlepinEvent.new(result.message_id, result.subject, result.content)
-          act_on_event(event)
-        end
+    def self.reset
+      @processed_count = 0
+      @failed_count = 0
+      @running = false
+      Rails.cache.delete(STATUS_CACHE_KEY)
+    end
+
+    def self.run
+      @client = client_factory.call
+      @client.subscribe do |message|
+        handle_message(message)
       end
-    rescue => e
-      @logger.error("Fatal error in Candlepin Listening Service")
-      close
-      raise e
+
+      @running = true
     end
 
     def self.status(refresh: true)
@@ -32,51 +46,23 @@ module Katello
         {
           processed_count: @processed_count,
           failed_count: @failed_count,
-          running: Katello::CandlepinListeningService.instance&.running? || false
+          running: running?
         }
       end
     end
 
-    def self.reset_status
-      @processed_count = 0
-      @failed_count = 0
-      Rails.cache.delete(STATUS_CACHE_KEY)
-    end
-
-    def self.act_on_event(event)
-      ::Katello::Util::Support.with_db_connection(@logger) do
-        ::Katello::Candlepin::EventHandler.new(@logger).handle(event)
+    def self.handle_message(message)
+      ::Katello::Util::Support.with_db_connection(logger) do
+        subject = "#{message.headers['EVENT_TARGET']}.#{message.headers['EVENT_TYPE']}".downcase
+        cp_event = Event.new(subject, message.body)
+        ::Katello::Candlepin::EventHandler.new(logger).handle(cp_event)
       end
       @processed_count += 1
     rescue => e
       @failed_count += 1
-      @logger.error("Error handling Candlepin event")
-      @logger.error(e.message)
-      @logger.error(e.backtrace.join("\n"))
-    end
-
-    def self.configured?
-      SETTINGS[:katello].key?(:qpid) &&
-        SETTINGS[:katello][:qpid].key?(:url) &&
-        SETTINGS[:katello][:qpid].key?(:subscriptions_queue_address)
-    end
-
-    def self.initialize_listening_service
-      if configured?
-        Katello::CandlepinListeningService.initialize_service(@logger,
-                                           SETTINGS[:katello][:qpid][:url],
-                                           SETTINGS[:katello][:qpid][:subscriptions_queue_address])
-      else
-        fail("Katello has not been configured for qpid.url and qpid.subscriptions_queue_address")
-      end
-    rescue => e
-      @logger.error(e.message)
-      @logger.error(e.backtrace)
-    end
-
-    def self.close
-      Katello::CandlepinListeningService.close
-      reset_status
+      logger.error("Error handling Candlepin event")
+      logger.error(e.message)
+      logger.error(e.backtrace.join("\n"))
     end
   end
 end

data/app/services/katello/event_daemon.rb

@@ -18,24 +18,23 @@ module Katello
 
       def check_services(error, status)
         @service_classes.each do |service_class|
-          begin
-            status = service_class.status
-          rescue => error
-            Rails.logger.error("Error occurred while pinging #{service_class}")
-            Rails.logger.error(error.message)
-            Rails.logger.error(error.backtrace.join("\n"))
-          ensure
-            if error || !status&.dig(:running)
-              begin
-                service_class.close
-                service_class.run
-              rescue => error
-                Rails.logger.error("Error occurred while starting #{service_class}")
-                Rails.logger.error(error.message)
-                Rails.logger.error(error.backtrace.join("\n"))
-              ensure
-                error = nil
-              end
+          status = service_class.status
+        rescue => error
+          Rails.logger.error("Error occurred while pinging #{service_class}")
+          Rails.logger.error(error.message)
+          Rails.logger.error(error.backtrace.join("\n"))
+        ensure
+          if error || !status&.dig(:running)
+            begin
+              service_class.close
+              service_class.run
+              service_class.status
+            rescue => error
+              Rails.logger.error("Error occurred while starting #{service_class}")
+              Rails.logger.error(error.message)
+              Rails.logger.error(error.backtrace.join("\n"))
+            ensure
+              error = nil
             end
           end
         end
@@ -80,6 +79,7 @@ module Katello
       def stop
         return unless pid == Process.pid
         @monitor_thread.kill
+        @cache.clear
         services.values.each(&:close)
         File.unlink(pid_file) if pid_file && File.exist?(pid_file)
       end

data/app/services/katello/event_monitor/poller_thread.rb

@@ -42,7 +42,7 @@ module Katello
 
       def close
         @logger.info("Stopping Katello Event Monitor")
-        @thread.kill if @thread
+        @thread&.kill
       end
 
       def running?
@@ -85,27 +85,25 @@ module Katello
       end
 
       def poll_for_events
-        @thread.kill if @thread
+        @thread&.kill
         @thread = Thread.new do
-          begin
-            @logger.info("Polling Katello Event Queue")
-            loop do
-              Rails.application.executor.wrap do
-                Katello::Util::Support.with_db_connection(@logger) do
-                  until (event = ::Katello::EventQueue.next_event).nil?
-                    run_event(event)
-                    @processed_count += 1
-                  end
+          @logger.info("Polling Katello Event Queue")
+          loop do
+            Rails.application.executor.wrap do
+              Katello::Util::Support.with_db_connection(@logger) do
+                until (event = ::Katello::EventQueue.next_event).nil?
+                  run_event(event)
+                  @processed_count += 1
                 end
               end
-
-              sleep SLEEP_INTERVAL
             end
-          rescue => e
-            @logger.error(e.message)
-            @logger.error("Fatal error in Katello Event Monitor")
-            self.class.close
+
+            sleep SLEEP_INTERVAL
           end
+        rescue => e
+          @logger.error(e.message)
+          @logger.error("Fatal error in Katello Event Monitor")
+          self.class.close
         end
       end
     end