katello 3.18.0.rc2.1 → 4.0.0.rc2

data/app/models/katello/glue/pulp/repo.rb

@@ -113,12 +113,6 @@ module Katello
         backend_service(capsule).generate_distributors
       end
 
-      def populate_from(repos_map)
-        found = repos_map[self.pulp_id]
-        prepopulate(found) if found
-        !found.nil?
-      end
-
       def package_group_count
         content_unit_counts = 0
         if self.pulp_repo_facts

data/app/models/katello/glue/pulp/repos.rb

@@ -12,27 +12,6 @@ module Katello
       "#{path_prefix}/#{path}"
     end
 
-    def self.prepopulate!(products, environment, repos = [], content_view = nil)
-      if content_view.nil?
-        if environment.library?
-          content_view = environment.default_content_view
-        else
-          fail "No content view specified for a Non library environment #{environment.inspect}"
-        end
-      end
-
-      items = Katello.pulp_server.extensions.repository.search_by_repository_ids(Repository.in_environment(environment).pluck(:pulp_id))
-      full_repos = {}
-      items.each { |item| full_repos[item["id"]] = item }
-
-      products.each do |prod|
-        prod.repos(environment, content_view).each do |repo|
-          repo.populate_from(full_repos)
-        end
-      end
-      repos.each { |repo| repo.populate_from(full_repos) }
-    end
-
     module InstanceMethods
       def distributions(env)
         to_ret = []
@@ -125,7 +104,7 @@ module Katello
             AND inner_select.resource_id = locks_foreman_tasks_tasks.resource_id")
           .distinct
 
-        max_per_repoid.map { |x| [x.resource_id, x] }.to_h
+        max_per_repoid.index_by { |x| [x.resource_id, x] }
       end
 
       def sync_state_aggregated

data/app/models/katello/host/subscription_facet.rb

@@ -282,6 +282,10 @@ module Katello
         name.gsub('_', '-').chomp('.').downcase
       end
 
+      def unsubscribed_hypervisor?
+        self.hypervisor && !self.candlepin_consumer.entitlements?
+      end
+
       def candlepin_consumer
         @candlepin_consumer ||= Katello::Candlepin::Consumer.new(self.uuid, self.host.organization.label)
       end

data/app/models/katello/ping.rb

@@ -14,7 +14,7 @@ module Katello
 
       def services(capsule_id = nil)
         proxy = fetch_proxy(capsule_id)
-        services = [:candlepin, :candlepin_auth, :foreman_tasks, :katello_events, :candlepin_events]
+        services = [:candlepin, :candlepin_auth, :foreman_tasks, :katello_events, :candlepin_events, :katello_agent]
         services += [:pulp3] if proxy&.pulp3_enabled? && pulpcore_enabled
         if proxy.nil? || proxy.has_feature?(SmartProxy::PULP_NODE_FEATURE) || proxy.has_feature?(SmartProxy::PULP_FEATURE)
           services += [:pulp, :pulp_auth]
@@ -38,6 +38,7 @@ module Katello
         ping_foreman_tasks(result[:foreman_tasks]) if result.include?(:foreman_tasks)
         ping_katello_events(result[:katello_events]) if result.include?(:katello_events)
         ping_candlepin_events(result[:candlepin_events]) if result.include?(:candlepin_events)
+        ping_katello_agent(result[:katello_agent]) if result.include?(:katello_agent)
 
         # set overall status result code
         result = {:services => result}
@@ -56,7 +57,7 @@ module Katello
         running = status&.dig(:running)
 
         if running
-          result[:message] = "#{status[:processed_count]} Processed, #{status[:failed_count]} Failed"
+          result[:message] = "#{status[:processed_count].to_i} Processed, #{status[:failed_count].to_i} Failed"
         else
           result[:status] = FAIL_RETURN_CODE
           result[:message] = _("Not running")
@@ -65,14 +66,21 @@ module Katello
 
       def ping_katello_events(result)
         exception_watch(result) do
-          status = Katello::EventMonitor::PollerThread.status(refresh: false)
+          status = Katello::EventDaemon::Runner.service_status(:katello_events)
           event_daemon_status(status, result)
         end
       end
 
       def ping_candlepin_events(result)
         exception_watch(result) do
-          status = Katello::CandlepinEventListener.status(refresh: false)
+          status = Katello::EventDaemon::Runner.service_status(:candlepin_events)
+          event_daemon_status(status, result)
+        end
+      end
+
+      def ping_katello_agent(result)
+        exception_watch(result) do
+          status = Katello::EventDaemon::Services::AgentEventReceiver.status(refresh: false)
           event_daemon_status(status, result)
         end
       end

data/app/models/katello/pool.rb

@@ -22,6 +22,7 @@ module Katello
       where(["end_date < ?", days.to_i.days.from_now.end_of_day])
     end
     scope :upstream, -> { where.not(upstream_pool_id: nil) }
+    scope :redhat, -> { joins(:products).merge(Katello::Product.redhat).distinct }
 
     include Glue::Candlepin::Pool
     include Glue::Candlepin::CandlepinObject
@@ -51,6 +52,10 @@ module Katello
 
     DAYS_RECENTLY_EXPIRED = 30
 
+    def redhat?
+      self.class.redhat.where(:id => self.id).exists?
+    end
+
     def active?
       active
     end

data/app/models/katello/product.rb

@@ -3,9 +3,9 @@ module Katello
     audited
 
     include ForemanTasks::Concerns::ActionSubject
-    include Glue::Candlepin::Product if SETTINGS[:katello][:use_cp]
-    include Glue::Pulp::Repos if SETTINGS[:katello][:use_pulp]
-    include Glue if SETTINGS[:katello][:use_cp] || SETTINGS[:katello][:use_pulp]
+    include Glue::Candlepin::Product
+    include Glue::Pulp::Repos
+    include Glue
 
     include Katello::Authorization::Product
 

data/app/models/katello/repository.rb

@@ -12,9 +12,9 @@ module Katello
 
     include ForemanTasks::Concerns::ActionSubject
     include Glue::Candlepin::Repository
-    include Glue::Pulp::Repo if SETTINGS[:katello][:use_pulp]
+    include Glue::Pulp::Repo
 
-    include Glue if (SETTINGS[:katello][:use_cp] || SETTINGS[:katello][:use_pulp])
+    include Glue
     include Authorization::Repository
     include Katello::Engine.routes.url_helpers
 
@@ -122,6 +122,8 @@ module Katello
 
     scope :has_url, -> { joins(:root).where.not("#{RootRepository.table_name}.url" => nil) }
     scope :on_demand, -> { joins(:root).where("#{RootRepository.table_name}.download_policy" => ::Runcible::Models::YumImporter::DOWNLOAD_ON_DEMAND) }
+    scope :immediate, -> { joins(:root).where("#{RootRepository.table_name}.download_policy" => ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE) }
+    scope :non_immediate, -> { joins(:root).where.not("#{RootRepository.table_name}.download_policy" => ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE) }
     scope :in_default_view, -> { joins(:content_view_version => :content_view).where("#{Katello::ContentView.table_name}.default" => true) }
     scope :in_non_default_view, -> { joins(:content_view_version => :content_view).where("#{Katello::ContentView.table_name}.default" => false) }
     scope :deb_type, -> { with_type(DEB_TYPE) }
@@ -143,7 +145,7 @@ module Katello
     scoped_search :rename => :product, :on => :name, :relation => :product, :complete_value => true
     scoped_search :rename => :product_id, :on => :id, :relation => :product
     scoped_search :on => :content_type, :relation => :root, :complete_value => -> do
-      Katello::RepositoryTypeManager.repository_types.keys.each_with_object({}) { |value, hash| hash[value.to_sym] = value }
+      Katello::RepositoryTypeManager.repository_types.keys.index_by { |value| value.to_sym }
     end
     scoped_search :on => :content_view_id, :relation => :content_view_repositories, :validator => ScopedSearch::Validators::INTEGER, :only_explicit => true
     scoped_search :on => :distribution_version, :complete_value => true
@@ -154,6 +156,7 @@ module Katello
     scoped_search :on => :redhat, :complete_value => { :true => true, :false => false }, :ext_method => :search_by_redhat
     scoped_search :on => :container_repository_name, :complete_value => true
     scoped_search :on => :description, :relation => :root, :only_explicit => true
+    scoped_search :on => :download_policy, :relation => :root, :only_explicit => true
     scoped_search :on => :name, :relation => :product, :rename => :product_name
     scoped_search :on => :id, :relation => :product, :rename => :product_id, :only_explicit => true
     scoped_search :on => :label, :relation => :root, :complete_value => true, :only_explicit => true
@@ -287,6 +290,10 @@ module Katello
       root.download_policy == Runcible::Models::YumImporter::DOWNLOAD_ON_DEMAND
     end
 
+    def immediate?
+      root.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE
+    end
+
     def yum_gpg_key_url
       # if the repo has a gpg key return a url to access it
       if self.root.gpg_key.try(:content).present?

data/app/models/katello/subscription_status.rb

@@ -43,9 +43,10 @@ module Katello
 
     def to_status(options = {})
       return UNKNOWN unless host.subscription_facet.try(:uuid)
-      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.hypervisor && !host.subscription_facet.candlepin_consumer.entitlements?
+      return DISABLED if host.organization.simple_content_access?
+      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.unsubscribed_hypervisor?
       status_override ||= options.fetch(:status_override, nil)
-      status = status_override || Katello::Candlepin::Consumer.new(host.subscription_facet.uuid, host.organization.label).entitlement_status
+      status = status_override || host.subscription_facet.candlepin_consumer.entitlement_status
 
       case status
       when Katello::Candlepin::Consumer::ENTITLEMENTS_DISABLED

data/app/models/setting/content.rb

@@ -120,8 +120,6 @@ class Setting::Content < Setting
                '', N_('Subscription manager name registration fact'), nil),
       self.set('register_hostname_fact_strict_match', N_('If true, and register_hostname_fact is set and provided, registration will look for a new host by name only '\
                 'using that fact, and will skip all hostname matching'), false, N_('Subscription manager name registration fact strict matching'), nil),
-      self.set('erratum_install_batch_size', N_("Errata installed via katello-agent will be triggered in batches of this size. Set to 0 to install all errata in one batch."),
-               0, N_('Erratum Install Batch Size')),
       self.set('default_location_subscribed_hosts',
                N_('Default Location where new subscribed hosts will put upon registration'),
                nil, N_('Default Location subscribed hosts'), nil,

data/app/presenters/katello/host_subscription_presenter.rb

@@ -1,11 +1,10 @@
 module Katello
   class HostSubscriptionPresenter < SimpleDelegator
-    attr_accessor :quantity_consumed
+    attr_reader :quantity_consumed
 
-    def initialize(entitlement)
-      @subscription = Katello::Pool.find_by(:cp_id => entitlement['pool']['id'])
+    def initialize(pool:, entitlement:)
       @quantity_consumed = entitlement.try(:[], :quantity)
-      super(@subscription)
+      super(pool)
     end
   end
 end

data/app/presenters/katello/host_subscriptions_presenter.rb

@@ -0,0 +1,24 @@
+module Katello
+  class HostSubscriptionsPresenter
+    attr_reader :subscriptions
+
+    def initialize(host)
+      pools = host.subscription_facet&.pools || []
+      @pools = pools.group_by(&:cp_id)
+
+      entitlements = host.subscription_facet.candlepin_consumer.entitlements if @pools.any?
+      entitlements ||= []
+
+      @subscriptions = entitlements.map do |e|
+        HostSubscriptionPresenter.new(pool: pool_for_entitlement(e), entitlement: e)
+      end
+    end
+
+    private
+
+    def pool_for_entitlement(entitlement)
+      pool_cp_id = entitlement['pool']['id']
+      @pools[pool_cp_id]&.first
+    end
+  end
+end

data/app/services/katello/agent/dispatcher.rb

@@ -0,0 +1,66 @@
+module Katello
+  module Agent
+    class Dispatcher
+      @supported_messages = {}
+
+      def self.register_message(name, klass)
+        @supported_messages[name] = klass
+      end
+
+      register_message(:install_package, Katello::Agent::InstallPackageMessage)
+      register_message(:remove_package, Katello::Agent::RemovePackageMessage)
+      register_message(:update_package, Katello::Agent::UpdatePackageMessage)
+      register_message(:install_errata, Katello::Agent::InstallErrataMessage)
+      register_message(:install_package_group, Katello::Agent::InstallPackageGroupMessage)
+      register_message(:remove_package_group, Katello::Agent::RemovePackageGroupMessage)
+
+      def self.dispatch(message_type, host_ids, args)
+        message_class = @supported_messages[message_type]
+
+        fail("Unsupported message type") unless message_class
+
+        uuid_data = ::Katello::Host::ContentFacet.where(host_id: host_ids).pluck(:host_id, :uuid)
+        fail("Couldn't find all hosts specified") unless host_ids.size == uuid_data.size
+
+        host_data = uuid_data.map do |host_id, consumer_id|
+          {
+            host_id: host_id,
+            consumer_id: consumer_id,
+            history: Katello::Agent::DispatchHistory.new(host_id: host_id),
+            message: message_class.new(**args.merge(consumer_id: consumer_id))
+          }
+        end
+
+        histories = host_data.map { |attrs| attrs[:history] }
+        ActiveRecord::Base.transaction do
+          Katello::Agent::DispatchHistory.import(histories)
+
+          host_data.each do |d|
+            d[:message].dispatch_history_id = d[:history].id
+            d[:message].recipient_address = settings[:client_queue_format] % [d[:consumer_id]]
+            d[:message].reply_to = settings[:event_queue_name]
+          end
+
+          connection = Connection.new
+          connection.send_messages(host_data.map { |d| d[:message] })
+        end
+
+        histories
+      end
+
+      def self.delete_client_queue(queue_name:)
+        connection = Connection.new
+        connection.delete_client_queue(queue_name)
+      end
+
+      def self.host_queue_name(host)
+        uuid = host.content_facet.uuid
+        settings[:client_queue_format] % uuid
+      end
+
+      def self.settings
+        SETTINGS[:katello][:agent]
+      end
+    end
+  end
+end

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -60,17 +60,7 @@ module Katello
       end
 
       def fetch_rpm_content_ids
-        # Query for applicable RPM ids
-        # -> Include all non-modular rpms or rpms that exist within installed module streams
-        enabled_module_stream_ids = ::Katello::ModuleStream.
-          joins("inner join katello_available_module_streams on
-            katello_module_streams.name = katello_available_module_streams.name and
-            katello_module_streams.stream = katello_available_module_streams.stream").
-          joins("inner join katello_host_available_module_streams on
-            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
-          where("katello_host_available_module_streams.host_id = :content_facet_id and
-            katello_host_available_module_streams.status = 'enabled'",
-            :content_facet_id => self.content_facet.host.id).select(:id)
+        enabled_module_stream_ids = fetch_enabled_module_stream_ids
 
         ::Katello::Rpm.
           joins("INNER JOIN katello_repository_rpms ON
@@ -78,7 +68,8 @@ module Katello
           joins("INNER JOIN katello_installed_packages ON
             katello_rpms.name = katello_installed_packages.name AND
             katello_rpms.arch = katello_installed_packages.arch AND
-            katello_rpms.evr > katello_installed_packages.evr").
+            katello_rpms.evr > katello_installed_packages.evr AND
+            katello_installed_packages.id in (#{newest_distinct_installed_packages_query})").
           joins("LEFT JOIN katello_module_stream_rpms ON
             katello_rpms.id = katello_module_stream_rpms.rpm_id").
           joins("INNER JOIN katello_host_installed_packages ON
@@ -87,9 +78,47 @@ module Katello
             :bound_library_repos => self.bound_library_instance_repos).
           where("katello_host_installed_packages.host_id = :content_facet_id",
             :content_facet_id => self.content_facet.host.id).
-          where("katello_module_stream_rpms.module_stream_id is null or
-            katello_module_stream_rpms.module_stream_id in (:enabled_module_streams)",
-            :enabled_module_streams => enabled_module_stream_ids).pluck(:id).uniq
+          where("(katello_module_stream_rpms.module_stream_id IS NULL AND
+            katello_installed_packages.id NOT IN (:locked_modular_installed_packages)) OR
+            (katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)
+            AND katello_installed_packages.id IN (:locked_modular_installed_packages))",
+            :enabled_module_streams => enabled_module_stream_ids,
+            :locked_modular_installed_packages => locked_modular_installed_packages(enabled_module_stream_ids)).pluck(:id).uniq
+      end
+
+      def fetch_enabled_module_stream_ids
+        # Query for applicable RPM ids
+        # -> Include all non-modular rpms or rpms that exist within installed module streams
+        ::Katello::ModuleStream.
+          joins("inner join katello_available_module_streams on
+            katello_module_streams.name = katello_available_module_streams.name and
+            katello_module_streams.stream = katello_available_module_streams.stream").
+          joins("inner join katello_host_available_module_streams on
+            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
+          where("katello_host_available_module_streams.host_id = :content_facet_id and
+            katello_host_available_module_streams.status = 'enabled'",
+            :content_facet_id => self.content_facet.host.id).select(:id)
+      end
+
+      # Installed packages that are locked for the host due to enabled module stream membership
+      def locked_modular_installed_packages(enabled_module_streams)
+        rpms_in_enabled_module_streams = ::Katello::Rpm.
+          joins("INNER JOIN katello_module_stream_rpms ON katello_rpms.id = katello_module_stream_rpms.rpm_id").
+          where("katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)",
+                :enabled_module_streams => enabled_module_streams).select(:nvra, :epoch)
+
+        ::Katello::InstalledPackage.where(nvra: rpms_in_enabled_module_streams.map(&:nvra),
+                                          epoch: rpms_in_enabled_module_streams.map(&:epoch)).select(:id)
+      end
+
+      def newest_distinct_installed_packages_query
+        "SELECT DISTINCT ON (katello_installed_packages.name) katello_installed_packages.id " \
+          "FROM katello_installed_packages INNER JOIN " \
+          "katello_host_installed_packages ON " \
+          "katello_installed_packages.id = " \
+          "katello_host_installed_packages.installed_package_id " \
+          "WHERE katello_host_installed_packages.host_id = " \
+          "#{content_facet.host.id} ORDER BY katello_installed_packages.name, katello_installed_packages.evr DESC"
       end
 
       def applicable_differences

data/app/services/katello/candlepin_event_listener.rb

@@ -1,7 +1,6 @@
 module Katello
+  # TODO: Move this class to app/lib/katello/event_daemon/services with other service definitions
   class CandlepinEventListener
-    STATUS_CACHE_KEY = 'candlepin_events_status'.freeze
-
     Event = Struct.new(:subject, :content)
 
     cattr_accessor :client_factory
@@ -14,22 +13,20 @@ module Katello
     end
 
     def self.running?
-      @running == true && @client&.running?
+      @client&.running? || false
     end
 
     def self.close
-      return unless running?
-
-      logger.info("Closing candlepin event listener")
-      @client&.close
+      if @client&.close
+        logger.info("Closed candlepin event listener")
+      end
       reset
     end
 
     def self.reset
       @processed_count = 0
       @failed_count = 0
-      @running = false
-      Rails.cache.delete(STATUS_CACHE_KEY)
+      @client = nil
     end
 
     def self.run
@@ -37,18 +34,14 @@ module Katello
       @client.subscribe do |message|
         handle_message(message)
       end
-
-      @running = true
     end
 
-    def self.status(refresh: true)
-      Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
-        {
-          processed_count: @processed_count,
-          failed_count: @failed_count,
-          running: running?
-        }
-      end
+    def self.status
+      {
+        processed_count: @processed_count,
+        failed_count: @failed_count,
+        running: running?
+      }
     end
 
     def self.handle_message(message)