katello 4.1.0.rc2 → 4.1.1

data/app/lib/actions/katello/host/upload_profiles.rb

@@ -13,7 +13,6 @@ module Actions
 
           sequence do
             plan_self(:host_id => host.id, :hostname => host.name, :profile_string => profile_string)
-            plan_action(GenerateApplicability, [host])
           end
         end
 
@@ -33,52 +32,12 @@ module Actions
           Dynflow::Action::Rescue::Skip
         end
 
-        def self.upload_modules_to_pulp(available_streams, host)
-          query_name_streams = available_streams.map do |profile|
-            ::Katello::ModuleStream.where(profile.slice(:name, :stream))
-          end
-
-          updated_profiles = []
-          unless query_name_streams.empty?
-            query_name_streams = query_name_streams.inject(&:or)
-
-            bound_library_instances = host.content_facet.bound_repositories.map(&:library_instance_or_self)
-            query = ::Katello::ModuleStream.in_repositories(bound_library_instances).
-                                            select(:name, :stream, :version, :context, :arch).
-                                              merge(query_name_streams)
-
-            updated_profiles = query.map do |module_stream|
-              module_stream.slice(:name, :stream, :version, :context, :arch)
-            end
-          end
-
-          # We also need to pass module streams that are not found in the ModuleStream table
-          # but are present on the content host
-          unassociated_profiles = available_streams.select do |profile|
-            updated_profiles.none? { |p| p[:name] == profile[:name] && p[:stream] == profile[:stream] }
-          end
-
-          module_stream_profile = updated_profiles + unassociated_profiles
-
-          if SmartProxy.pulp_primary&.has_feature?(SmartProxy::PULP_FEATURE)
-            unless module_stream_profile.empty?
-              begin
-                ::Katello::Pulp::Consumer.new(host.content_facet.uuid).
-                  upload_module_stream_profile(module_stream_profile)
-              rescue RestClient::ResourceNotFound
-                Rails.logger.warn("Host with ID %s was not known to Pulp, continuing" % host.id)
-              end
-            end
-          end
-        end
-
         def import_module_streams(payload, host)
           enabled_payload = payload.map do |profile|
             profile.slice("name", "stream", "version", "context", "arch").with_indifferent_access if profile["status"] == "enabled"
           end
           enabled_payload.compact!
 
-          UploadProfiles.upload_modules_to_pulp(enabled_payload, host)
           host.import_module_streams(payload)
         end
 
@@ -124,6 +83,8 @@ module Actions
             module_streams.each do |module_stream_payload|
               import_module_streams(module_stream_payload, host)
             end
+
+            ::Katello::Host::ContentFacet.trigger_applicability_generation(host.id)
           end
         end
       end

data/app/lib/actions/katello/orphan_cleanup/remove_orphans.rb

@@ -1,13 +1,13 @@
 module Actions
   module Katello
     module OrphanCleanup
-      class RemoveOrphans < Pulp::Abstract
+      class RemoveOrphans < Actions::Base
         input_format do
           param :capsule_id
         end
         def plan(proxy)
           sequence do
-            plan_action(Actions::Pulp::Orchestration::OrphanCleanup::RemoveOrphans, proxy)
+            plan_action(Actions::Pulp::Orchestration::OrphanCleanup::RemoveOrphans, proxy) if (proxy.has_feature?(SmartProxy::PULP_FEATURE) || proxy.has_feature?(SmartProxy::PULP_NODE_FEATURE))
             if proxy.pulp3_enabled?
               plan_action(
                 Actions::Pulp3::Orchestration::OrphanCleanup::RemoveOrphans,

data/app/lib/actions/katello/repository/check_matching_content.rb

@@ -13,6 +13,7 @@ module Actions
         def run
           source_repo = ::Katello::Repository.find(input[:source_repo_id])
           target_repo = ::Katello::Repository.find(input[:target_repo_id])
+          target_repo_published = target_repo.backend_service(SmartProxy.pulp_primary).published?
 
           if source_repo.content_type == ::Katello::Repository::YUM_TYPE
             srpms_match = srpms_match?(source_repo, target_repo)
@@ -23,16 +24,14 @@ module Actions
             yum_metadata_files = yum_metadata_files_match?(source_repo, target_repo)
             checksum_match = (target_repo.saved_checksum_type == source_repo.saved_checksum_type)
 
-            published = target_repo.backend_service(SmartProxy.pulp_primary).published?
-
             output[:checksum_match] = checksum_match
-            output[:matching_content] = yum_metadata_files && srpms_match && rpms && errata && package_groups && distributions && published && checksum_match
+            output[:matching_content] = yum_metadata_files && srpms_match && rpms && errata && package_groups && distributions && target_repo_published && checksum_match
           end
 
           if source_repo.content_type == ::Katello::Repository::DEB_TYPE
             debs = debs_match?(source_repo, target_repo)
 
-            output[:matching_content] = debs && target_repo.published?
+            output[:matching_content] = debs && target_repo_published
           end
         end
 
@@ -41,26 +40,26 @@ module Actions
         end
 
         def srpms_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.srpm_ids.sort
-          target_repo_ids = target_repo.srpm_ids.sort
+          source_repo_ids = source_repo.srpm_ids.sort.uniq
+          target_repo_ids = target_repo.srpm_ids.sort.uniq
           source_repo_ids == target_repo_ids
         end
 
         def rpms_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.rpm_ids.sort
-          target_repo_ids = target_repo.rpm_ids.sort
+          source_repo_ids = source_repo.rpm_ids.sort.uniq
+          target_repo_ids = target_repo.rpm_ids.sort.uniq
           source_repo_ids == target_repo_ids
         end
 
         def errata_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.erratum_ids.sort
-          target_repo_ids = target_repo.erratum_ids.sort
+          source_repo_ids = source_repo.erratum_ids.sort.uniq
+          target_repo_ids = target_repo.erratum_ids.sort.uniq
           source_repo_ids == target_repo_ids
         end
 
         def package_groups_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.package_groups.order(:name).pluck(:name)
-          target_repo_ids = target_repo.package_groups.order(:name).pluck(:name)
+          source_repo_ids = source_repo.package_groups.order(:name).pluck(:name).uniq
+          target_repo_ids = target_repo.package_groups.order(:name).pluck(:name).uniq
           source_repo_ids == target_repo_ids
         end
 
@@ -69,8 +68,8 @@ module Actions
         end
 
         def yum_metadata_files_match?(source_repo, target_repo)
-          source_repo_items = source_repo.yum_metadata_files.pluck(:name, :checksum).sort
-          target_repo_items = target_repo.yum_metadata_files.pluck(:name, :checksum).sort
+          source_repo_items = source_repo.yum_metadata_files.pluck(:name, :checksum).sort.uniq
+          target_repo_items = target_repo.yum_metadata_files.pluck(:name, :checksum).sort.uniq
           source_repo_items == target_repo_items
         end
       end

data/app/lib/actions/katello/repository/sync.rb

@@ -21,14 +21,14 @@ module Actions
         # rubocop:disable Metrics/CyclomaticComplexity
         # rubocop:disable Metrics/PerceivedComplexity
         # rubocop:disable Metrics/AbcSize
-        def plan(repo, _pulp_sync_task_id = nil, options = {})
+        def plan(repo, options = {})
           action_subject(repo)
 
           source_url = options.fetch(:source_url, nil)
           incremental = options.fetch(:incremental, false)
           validate_contents = options.fetch(:validate_contents, false)
           skip_metadata_check = options.fetch(:skip_metadata_check, false) || (validate_contents && repo.yum?)
-          generate_applicability = repo.yum?
+          generate_applicability =  options.fetch(:generate_applicability, repo.yum?)
 
           fail ::Katello::Errors::InvalidActionOptionError, _("Unable to sync repo. This repository does not have a feed url.") if repo.url.blank? && source_url.blank?
           fail ::Katello::Errors::InvalidActionOptionError, _("Cannot validate contents on non-yum/deb repositories.") if validate_contents && !repo.yum? && !repo.deb?
@@ -68,13 +68,10 @@ module Actions
                 plan_action(Pulp::Repository::Download, :pulp_id => repo.pulp_id, :options => {:verify_all_units => true}) if validate_contents && repo.yum?
                 plan_action(Katello::Repository::MetadataGenerate, repo, :force => true) if skip_metadata_check && repo.yum?
                 plan_action(Katello::Repository::ErrataMail, repo, nil, contents_changed)
-                if generate_applicability
-                  regenerate_applicability(repo, contents_changed)
-                end
+                plan_action(Actions::Katello::Applicability::Repository::Regenerate, :repo_ids => [repo.id]) if generate_applicability
               end
               plan_self(:id => repo.id, :sync_result => output, :skip_metadata_check => skip_metadata_check, :validate_contents => validate_contents,
                         :contents_changed => contents_changed)
-              plan_action(Katello::Repository::ImportApplicability, :repo_id => repo.id, :contents_changed => contents_changed) if generate_applicability
               plan_action(Katello::Repository::SyncHook, :id => repo.id)
             end
           end
@@ -115,14 +112,6 @@ module Actions
           end
         end
 
-        def regenerate_applicability(repo, contents_changed)
-          if SETTINGS[:katello][:katello_applicability]
-            plan_action(Actions::Katello::Applicability::Repository::Regenerate, :repo_id => repo.id, :contents_changed => contents_changed)
-          else
-            plan_action(Pulp::Repository::RegenerateApplicability, :repository_id => repo.id, :contents_changed => contents_changed)
-          end
-        end
-
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end

data/app/lib/actions/katello/repository/verify_checksum.rb

@@ -13,7 +13,7 @@ module Actions
           else
             options = {}
             options[:validate_contents] = true
-            plan_action(Actions::Katello::Repository::Sync, repo, nil, options)
+            plan_action(Actions::Katello::Repository::Sync, repo, options)
           end
         end
 

data/app/lib/actions/katello/sync_plan/run.rb

@@ -19,8 +19,13 @@ module Actions
             syncable_products = sync_plan.products.syncable
             syncable_roots = ::Katello::RootRepository.where(:product_id => syncable_products).has_url
 
-            plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance).compact) unless syncable_roots.empty?
-            plan_self(:sync_plan_name => sync_plan.name)
+            sequence do
+              plan_self(:sync_plan_name => sync_plan.name)
+              if syncable_roots.any?
+                plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance).compact, generate_applicability: false)
+                plan_action(Actions::Katello::Applicability::Repository::Regenerate, :repo_ids => syncable_roots.map(&:library_instance).map(&:id))
+              end
+            end
           end
         end
 

data/app/lib/actions/pulp3/content_guard/refresh.rb

@@ -2,16 +2,12 @@ module Actions
   module Pulp3
     module ContentGuard
       class Refresh < Pulp3::Abstract
-        def plan(smart_proxy, options = {})
-          return if (::Katello::Pulp3::ContentGuard.count > 0 || options.try(:[], :update))
-          content_guard_api = ::Katello::Pulp3::Api::ContentGuard.new(smart_proxy)
-          if options.try(:[], :update)
-            content_guard_href = ::Katello::Pulp3::ContentGuard.first.href
-            content_guard_api.partial_update content_guard_href
-          else
-            content_guard_api.create
-            ::Katello::Pulp3::ContentGuard.import(smart_proxy)
-          end
+        def plan(smart_proxy)
+          plan_self(smart_proxy_id: smart_proxy.id)
+        end
+
+        def run
+          ::Katello::Pulp3::Api::ContentGuard.new(smart_proxy).refresh
         end
       end
     end

data/app/lib/actions/pulp3/content_view_version/destroy_exporter.rb

@@ -1,13 +1,13 @@
 module Actions
   module Pulp3
     module ContentViewVersion
-      class DestroyExporter < Pulp3::Abstract
+      class DestroyExporter < Pulp3::AbstractAsyncTask
         input_format do
           param :smart_proxy_id, Integer
           param :exporter_data, Hash
         end
 
-        def run
+        def invoke_external_task
           ::Katello::Pulp3::ContentViewVersion::Export.new(smart_proxy: smart_proxy).destroy_exporter(input[:exporter_data][:pulp_href])
         end
       end

data/app/lib/actions/pulp3/orchestration/orphan_cleanup/remove_orphans.rb

@@ -2,7 +2,7 @@ module Actions
   module Pulp3
     module Orchestration
       module OrphanCleanup
-        class RemoveOrphans < Pulp::Abstract
+        class RemoveOrphans < Pulp3::Abstract
           def plan(proxy)
             if proxy.pulp3_enabled?
               sequence do

data/app/lib/actions/pulp3/repository/create_remote.rb

@@ -10,6 +10,10 @@ module Actions
           repo = ::Katello::Repository.find(input[:repository_id])
           output[:response] = repo.backend_service(smart_proxy).create_remote
         end
+
+        def rescue_strategy
+          Dynflow::Action::Rescue::Skip
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -23,6 +23,7 @@ module Actions
           if version_href
             if repo.version_href != version_href || input[:force_fetch_version]
               output[:contents_changed] = true
+              repo.update(:last_contents_changed => DateTime.now)
               repo.update(:version_href => version_href)
             end
           else

data/app/lib/katello/errors.rb

@@ -53,7 +53,7 @@ module Katello
 
     class PulpcoreMissingCapabilities < StandardError
       def message
-        _("A smart proxy seems to have been refreshed without pulpcore being running.  You may want to ")
+        _("A smart proxy seems to have been refreshed without pulpcore being running. Please refresh the smart proxy after ensuring that pulpcore services are running.")
       end
     end
 

data/app/lib/katello/event_daemon/monitor.rb

@@ -38,6 +38,7 @@ module Katello
             begin
               service_class.close
               service_class.run
+              sleep 0.1
               @service_statuses[service_name] = service_class.status
             rescue => error
               Rails.logger.error("Error occurred while starting #{service_class}")

data/app/lib/katello/event_daemon/services/agent_event_receiver.rb

@@ -2,8 +2,6 @@ module Katello
   module EventDaemon
     module Services
       class AgentEventReceiver
-        STATUS_CACHE_KEY = 'katello_agent_events'.freeze
-
         class Handler
           attr_accessor :processed, :failed
 
@@ -48,14 +46,12 @@ module Katello
           @agent_connection&.open? && @thread&.status.present?
         end
 
-        def self.status(refresh: true)
-          Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
-            {
-              running: running?,
-              processed_count: @handler&.processed || 0,
-              failed_count: @handler&.failed || 0
-            }
-          end
+        def self.status
+          {
+            running: running?,
+            processed_count: @handler&.processed || 0,
+            failed_count: @handler&.failed || 0
+          }
         end
       end
     end

data/app/lib/katello/util/hostgroup_facets_helper.rb

@@ -0,0 +1,126 @@
+# Used exclusively by fix_hostgroup_facets.rake task
+module Katello
+  module Util
+    class HostgroupFacetsHelper
+      def initialize
+        @logger = Logger.new($stdout)
+      end
+
+      def interested_hostgroups
+        groups = ::Hostgroup.unscoped.where(
+                    id: Katello::Hostgroup::ContentFacet.
+                        where(content_source_id: nil,
+                              kickstart_repository_id: nil,
+                              content_view_id: nil,
+                              lifecycle_environment_id: nil).select(:hostgroup_id))
+        parents = groups.select { |group| group.parent.blank? }
+        children = groups.reject { |group| group.parent.blank? }
+        # we want the parents to get created before the children
+        # hence the order
+        parents + children
+      end
+
+      def pick_facet_values(hg)
+        # This call looks at the audit logs for a host group.
+        # Pries out information related to lce, ks, cv and content_source_id from the audit logs.
+        # The audit logs typically only contain updates.
+        # So if the user changed  just the content_view_id, then that is the only thing marked as audited_changes.
+        # Hence we need to go through all the audit logs until we have information on lce, ks, cv and cs.
+        # If there was only one audit log and that was during the creation of hostgroup
+        # the audited changes look like this
+        # ```ruby
+        # {
+        #  content_view_id: 10,
+        #  kickstart_repository_id: 1000
+        #  ......
+        # }
+        # ```
+        # However if you updated the hostgroup and set the kickstart_repository_id, or
+        # content_view_id then audited changes look like
+        # ```ruby
+        # {
+        #  content_view_id: [10, 11],
+        #  kickstart_repository_id: [1000, 1200]
+        #  ......
+        # }
+        # ```
+        # So the code says "if the attribute value is an array pick the last value else just keep the value as it is "
+
+        # Further along it is to be noted that `hostgroup.audits` returns the audits ordered by the version number in ascending order, so the latest audit will be `hostgroup.audits.last`
+
+        # We want to iterate though each audit from latest audit to start, and as soon as we find a  content_view_id key or kickstart_repository_id key or lifecycle environment_id key  or content_source_id key we want it to be set once.
+
+        # So if I had an audit history like
+        # ``` ruby
+        # {
+        #  content_view_id: 10,
+        #  kickstart_repository_id: 1000,
+        #  version:1
+        #  ......
+        # },
+        # {
+        #  content_view_id: [10, 11],
+        #  kickstart_repository_id: [1000, 1200],
+        #   version: 2
+        #  ......
+        # }
+        # ```
+
+        # The code would start at version 2, notice that cv_id and ks_repo were set there
+        # and keep them as the final.
+        # So when it goes to version 1 since cv_id and ks_repo are already set,
+        # it will ignore. It will finally
+        # return {content_view_id: 11, kickstart_repository_id: 1200}
+        facet_values = {}
+        hg.audits.reverse_each do |audit|
+          hg_changes = audit.audited_changes.slice("lifecycle_environment_id",
+                                                   "kickstart_repository_id",
+                                                   "content_view_id",
+                                                   "content_source_id")
+          facet_values = hg_changes.merge(facet_values)
+        end
+
+        values = facet_values.map do |k, v|
+          v = v[-1] if v.is_a? Array
+          [k, v]
+        end
+        values.to_h.with_indifferent_access
+      end
+
+      def main
+        bad_hgs = []
+        good_hgs = []
+
+        groups = interested_hostgroups.each do |hg|
+          facet = hg.content_facet
+          values = pick_facet_values(hg)
+          if !values.empty? && facet.update(values)
+            good_hgs << { hostgroup: hg, facet_values: values }
+          else
+            bad_hgs << { hostgroup: hg, facet_values: values }
+            facet.save(validate: false)
+          end
+        end
+
+        unless bad_hgs.empty?
+          @logger.warn "Some of the hostgroups reported a validation error. "\
+                        "The hostgroups have been updated. "\
+                        "Check via the Web UI."
+
+          bad_hgs.each do |bad_group|
+            @logger.warn "Hostgroup #{bad_group[:hostgroup]}"
+            @logger.warn "Facet Values #{bad_group[:facet_values]}"
+          end
+        end
+        unless good_hgs.empty?
+          @logger.info "Following hostgroups were succesfully updated."
+          good_hgs.each do |good_group|
+            @logger.info "Hostgroup #{good_group[:hostgroup]}"
+            @logger.info "Facet Values #{good_group[:facet_values]}"
+          end
+        end
+        @logger.info("#{groups.count} Hostgroup(s) were updated.")
+      end
+    end
+  end
+end