katello 4.1.0.rc2.2 → 4.1.0

data/app/lib/actions/katello/repository/check_matching_content.rb

@@ -13,6 +13,7 @@ module Actions
         def run
           source_repo = ::Katello::Repository.find(input[:source_repo_id])
           target_repo = ::Katello::Repository.find(input[:target_repo_id])
+          target_repo_published = target_repo.backend_service(SmartProxy.pulp_primary).published?
 
           if source_repo.content_type == ::Katello::Repository::YUM_TYPE
             srpms_match = srpms_match?(source_repo, target_repo)
@@ -23,16 +24,14 @@ module Actions
             yum_metadata_files = yum_metadata_files_match?(source_repo, target_repo)
             checksum_match = (target_repo.saved_checksum_type == source_repo.saved_checksum_type)
 
-            published = target_repo.backend_service(SmartProxy.pulp_primary).published?
-
             output[:checksum_match] = checksum_match
-            output[:matching_content] = yum_metadata_files && srpms_match && rpms && errata && package_groups && distributions && published && checksum_match
+            output[:matching_content] = yum_metadata_files && srpms_match && rpms && errata && package_groups && distributions && target_repo_published && checksum_match
           end
 
           if source_repo.content_type == ::Katello::Repository::DEB_TYPE
             debs = debs_match?(source_repo, target_repo)
 
-            output[:matching_content] = debs && target_repo.published?
+            output[:matching_content] = debs && target_repo_published
           end
         end
 
@@ -41,26 +40,26 @@ module Actions
         end
 
         def srpms_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.srpm_ids.sort
-          target_repo_ids = target_repo.srpm_ids.sort
+          source_repo_ids = source_repo.srpm_ids.sort.uniq
+          target_repo_ids = target_repo.srpm_ids.sort.uniq
           source_repo_ids == target_repo_ids
         end
 
         def rpms_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.rpm_ids.sort
-          target_repo_ids = target_repo.rpm_ids.sort
+          source_repo_ids = source_repo.rpm_ids.sort.uniq
+          target_repo_ids = target_repo.rpm_ids.sort.uniq
           source_repo_ids == target_repo_ids
         end
 
         def errata_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.erratum_ids.sort
-          target_repo_ids = target_repo.erratum_ids.sort
+          source_repo_ids = source_repo.erratum_ids.sort.uniq
+          target_repo_ids = target_repo.erratum_ids.sort.uniq
           source_repo_ids == target_repo_ids
         end
 
         def package_groups_match?(source_repo, target_repo)
-          source_repo_ids = source_repo.package_groups.order(:name).pluck(:name)
-          target_repo_ids = target_repo.package_groups.order(:name).pluck(:name)
+          source_repo_ids = source_repo.package_groups.order(:name).pluck(:name).uniq
+          target_repo_ids = target_repo.package_groups.order(:name).pluck(:name).uniq
           source_repo_ids == target_repo_ids
         end
 
@@ -69,8 +68,8 @@ module Actions
         end
 
         def yum_metadata_files_match?(source_repo, target_repo)
-          source_repo_items = source_repo.yum_metadata_files.pluck(:name, :checksum).sort
-          target_repo_items = target_repo.yum_metadata_files.pluck(:name, :checksum).sort
+          source_repo_items = source_repo.yum_metadata_files.pluck(:name, :checksum).sort.uniq
+          target_repo_items = target_repo.yum_metadata_files.pluck(:name, :checksum).sort.uniq
           source_repo_items == target_repo_items
         end
       end

data/app/lib/actions/katello/repository/sync.rb

@@ -21,14 +21,14 @@ module Actions
         # rubocop:disable Metrics/CyclomaticComplexity
         # rubocop:disable Metrics/PerceivedComplexity
         # rubocop:disable Metrics/AbcSize
-        def plan(repo, _pulp_sync_task_id = nil, options = {})
+        def plan(repo, options = {})
           action_subject(repo)
 
           source_url = options.fetch(:source_url, nil)
           incremental = options.fetch(:incremental, false)
           validate_contents = options.fetch(:validate_contents, false)
           skip_metadata_check = options.fetch(:skip_metadata_check, false) || (validate_contents && repo.yum?)
-          generate_applicability = repo.yum?
+          generate_applicability =  options.fetch(:generate_applicability, repo.yum?)
 
           fail ::Katello::Errors::InvalidActionOptionError, _("Unable to sync repo. This repository does not have a feed url.") if repo.url.blank? && source_url.blank?
           fail ::Katello::Errors::InvalidActionOptionError, _("Cannot validate contents on non-yum/deb repositories.") if validate_contents && !repo.yum? && !repo.deb?
@@ -68,13 +68,10 @@ module Actions
                 plan_action(Pulp::Repository::Download, :pulp_id => repo.pulp_id, :options => {:verify_all_units => true}) if validate_contents && repo.yum?
                 plan_action(Katello::Repository::MetadataGenerate, repo, :force => true) if skip_metadata_check && repo.yum?
                 plan_action(Katello::Repository::ErrataMail, repo, nil, contents_changed)
-                if generate_applicability
-                  regenerate_applicability(repo, contents_changed)
-                end
+                plan_action(Actions::Katello::Applicability::Repository::Regenerate, :repo_ids => [repo.id]) if generate_applicability
               end
               plan_self(:id => repo.id, :sync_result => output, :skip_metadata_check => skip_metadata_check, :validate_contents => validate_contents,
                         :contents_changed => contents_changed)
-              plan_action(Katello::Repository::ImportApplicability, :repo_id => repo.id, :contents_changed => contents_changed) if generate_applicability
               plan_action(Katello::Repository::SyncHook, :id => repo.id)
             end
           end
@@ -115,14 +112,6 @@ module Actions
           end
         end
 
-        def regenerate_applicability(repo, contents_changed)
-          if SETTINGS[:katello][:katello_applicability]
-            plan_action(Actions::Katello::Applicability::Repository::Regenerate, :repo_id => repo.id, :contents_changed => contents_changed)
-          else
-            plan_action(Pulp::Repository::RegenerateApplicability, :repository_id => repo.id, :contents_changed => contents_changed)
-          end
-        end
-
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end

data/app/lib/actions/katello/repository/verify_checksum.rb

@@ -13,7 +13,7 @@ module Actions
           else
             options = {}
             options[:validate_contents] = true
-            plan_action(Actions::Katello::Repository::Sync, repo, nil, options)
+            plan_action(Actions::Katello::Repository::Sync, repo, options)
           end
         end
 

data/app/lib/actions/katello/sync_plan/run.rb

@@ -19,8 +19,13 @@ module Actions
             syncable_products = sync_plan.products.syncable
             syncable_roots = ::Katello::RootRepository.where(:product_id => syncable_products).has_url
 
-            plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance).compact) unless syncable_roots.empty?
-            plan_self(:sync_plan_name => sync_plan.name)
+            sequence do
+              plan_self(:sync_plan_name => sync_plan.name)
+              if syncable_roots.any?
+                plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance).compact, generate_applicability: false)
+                plan_action(Actions::Katello::Applicability::Repository::Regenerate, :repo_ids => syncable_roots.map(&:library_instance).map(&:id))
+              end
+            end
           end
         end
 

data/app/lib/actions/pulp3/content_guard/refresh.rb

@@ -2,16 +2,12 @@ module Actions
   module Pulp3
     module ContentGuard
       class Refresh < Pulp3::Abstract
-        def plan(smart_proxy, options = {})
-          return if (::Katello::Pulp3::ContentGuard.count > 0 || options.try(:[], :update))
-          content_guard_api = ::Katello::Pulp3::Api::ContentGuard.new(smart_proxy)
-          if options.try(:[], :update)
-            content_guard_href = ::Katello::Pulp3::ContentGuard.first.href
-            content_guard_api.partial_update content_guard_href
-          else
-            content_guard_api.create
-            ::Katello::Pulp3::ContentGuard.import(smart_proxy)
-          end
+        def plan(smart_proxy)
+          plan_self(smart_proxy_id: smart_proxy.id)
+        end
+
+        def run
+          ::Katello::Pulp3::Api::ContentGuard.new(smart_proxy).refresh
         end
       end
     end

data/app/lib/actions/pulp3/orchestration/orphan_cleanup/remove_orphans.rb

@@ -2,7 +2,7 @@ module Actions
   module Pulp3
     module Orchestration
       module OrphanCleanup
-        class RemoveOrphans < Pulp::Abstract
+        class RemoveOrphans < Pulp3::Abstract
           def plan(proxy)
             if proxy.pulp3_enabled?
               sequence do

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -23,6 +23,7 @@ module Actions
           if version_href
             if repo.version_href != version_href || input[:force_fetch_version]
               output[:contents_changed] = true
+              repo.update(:last_contents_changed => DateTime.now)
               repo.update(:version_href => version_href)
             end
           else

data/app/lib/katello/errors.rb

@@ -53,7 +53,7 @@ module Katello
 
     class PulpcoreMissingCapabilities < StandardError
       def message
-        _("A smart proxy seems to have been refreshed without pulpcore being running.  You may want to ")
+        _("A smart proxy seems to have been refreshed without pulpcore being running. Please refresh the smart proxy after ensuring that pulpcore services are running.")
       end
     end
 

data/app/lib/katello/event_daemon/monitor.rb

@@ -38,6 +38,7 @@ module Katello
             begin
               service_class.close
               service_class.run
+              sleep 0.1
               @service_statuses[service_name] = service_class.status
             rescue => error
               Rails.logger.error("Error occurred while starting #{service_class}")

data/app/lib/katello/event_daemon/services/agent_event_receiver.rb

@@ -2,8 +2,6 @@ module Katello
   module EventDaemon
     module Services
       class AgentEventReceiver
-        STATUS_CACHE_KEY = 'katello_agent_events'.freeze
-
         class Handler
           attr_accessor :processed, :failed
 
@@ -48,14 +46,12 @@ module Katello
           @agent_connection&.open? && @thread&.status.present?
         end
 
-        def self.status(refresh: true)
-          Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
-            {
-              running: running?,
-              processed_count: @handler&.processed || 0,
-              failed_count: @handler&.failed || 0
-            }
-          end
+        def self.status
+          {
+            running: running?,
+            processed_count: @handler&.processed || 0,
+            failed_count: @handler&.failed || 0
+          }
         end
       end
     end

data/app/lib/katello/util/hostgroup_facets_helper.rb

@@ -0,0 +1,126 @@
+# Used exclusively by fix_hostgroup_facets.rake task
+module Katello
+  module Util
+    class HostgroupFacetsHelper
+      def initialize
+        @logger = Logger.new($stdout)
+      end
+
+      def interested_hostgroups
+        groups = ::Hostgroup.unscoped.where(
+                    id: Katello::Hostgroup::ContentFacet.
+                        where(content_source_id: nil,
+                              kickstart_repository_id: nil,
+                              content_view_id: nil,
+                              lifecycle_environment_id: nil).select(:hostgroup_id))
+        parents = groups.select { |group| group.parent.blank? }
+        children = groups.reject { |group| group.parent.blank? }
+        # we want the parents to get created before the children
+        # hence the order
+        parents + children
+      end
+
+      def pick_facet_values(hg)
+        # This call looks at the audit logs for a host group.
+        # Pries out information related to lce, ks, cv and content_source_id from the audit logs.
+        # The audit logs typically only contain updates.
+        # So if the user changed  just the content_view_id, then that is the only thing marked as audited_changes.
+        # Hence we need to go through all the audit logs until we have information on lce, ks, cv and cs.
+        # If there was only one audit log and that was during the creation of hostgroup
+        # the audited changes look like this
+        # ```ruby
+        # {
+        #  content_view_id: 10,
+        #  kickstart_repository_id: 1000
+        #  ......
+        # }
+        # ```
+        # However if you updated the hostgroup and set the kickstart_repository_id, or
+        # content_view_id then audited changes look like
+        # ```ruby
+        # {
+        #  content_view_id: [10, 11],
+        #  kickstart_repository_id: [1000, 1200]
+        #  ......
+        # }
+        # ```
+        # So the code says "if the attribute value is an array pick the last value else just keep the value as it is "
+
+        # Further along it is to be noted that `hostgroup.audits` returns the audits ordered by the version number in ascending order, so the latest audit will be `hostgroup.audits.last`
+
+        # We want to iterate though each audit from latest audit to start, and as soon as we find a  content_view_id key or kickstart_repository_id key or lifecycle environment_id key  or content_source_id key we want it to be set once.
+
+        # So if I had an audit history like
+        # ``` ruby
+        # {
+        #  content_view_id: 10,
+        #  kickstart_repository_id: 1000,
+        #  version:1
+        #  ......
+        # },
+        # {
+        #  content_view_id: [10, 11],
+        #  kickstart_repository_id: [1000, 1200],
+        #   version: 2
+        #  ......
+        # }
+        # ```
+
+        # The code would start at version 2, notice that cv_id and ks_repo were set there
+        # and keep them as the final.
+        # So when it goes to version 1 since cv_id and ks_repo are already set,
+        # it will ignore. It will finally
+        # return {content_view_id: 11, kickstart_repository_id: 1200}
+        facet_values = {}
+        hg.audits.reverse_each do |audit|
+          hg_changes = audit.audited_changes.slice("lifecycle_environment_id",
+                                                   "kickstart_repository_id",
+                                                   "content_view_id",
+                                                   "content_source_id")
+          facet_values = hg_changes.merge(facet_values)
+        end
+
+        values = facet_values.map do |k, v|
+          v = v[-1] if v.is_a? Array
+          [k, v]
+        end
+        values.to_h.with_indifferent_access
+      end
+
+      def main
+        bad_hgs = []
+        good_hgs = []
+
+        groups = interested_hostgroups.each do |hg|
+          facet = hg.content_facet
+          values = pick_facet_values(hg)
+          if !values.empty? && facet.update(values)
+            good_hgs << { hostgroup: hg, facet_values: values }
+          else
+            bad_hgs << { hostgroup: hg, facet_values: values }
+            facet.save(validate: false)
+          end
+        end
+
+        unless bad_hgs.empty?
+          @logger.warn "Some of the hostgroups reported a validation error. "\
+                        "The hostgroups have been updated. "\
+                        "Check via the Web UI."
+
+          bad_hgs.each do |bad_group|
+            @logger.warn "Hostgroup #{bad_group[:hostgroup]}"
+            @logger.warn "Facet Values #{bad_group[:facet_values]}"
+          end
+        end
+        unless good_hgs.empty?
+          @logger.info "Following hostgroups were succesfully updated."
+          good_hgs.each do |good_group|
+            @logger.info "Hostgroup #{good_group[:hostgroup]}"
+            @logger.info "Facet Values #{good_group[:facet_values]}"
+          end
+        end
+        @logger.info("#{groups.count} Hostgroup(s) were updated.")
+      end
+    end
+  end
+end

data/app/models/katello/authorization/content_view.rb

@@ -24,10 +24,6 @@ module Katello
       authorized?(:promote_or_remove_content_views) && Katello::KTEnvironment.any_promotable?
     end
 
-    def exportable?
-      authorized?(:export_content_views)
-    end
-
     module ClassMethods
       def readable
         authorized(:view_content_views)
@@ -37,10 +33,6 @@ module Katello
         authorized(:view_content_views)
       end
 
-      def importable?
-        creatable? && publishable?
-      end
-
       def readable_as(user)
         authorized_as(user, :view_content_views)
       end
@@ -75,7 +67,11 @@ module Katello
       end
 
       def exportable
-        authorized(:export_content_views)
+        where(organization: Organization.authorized(:export_content))
+      end
+
+      def importable
+        where(organization: Organization.authorized(:import_content))
       end
 
       def readable_repositories(repo_ids = nil)

data/app/models/katello/authorization/organization.rb

@@ -12,12 +12,12 @@ module Katello
       authorized?(:import_manifest)
     end
 
-    def can_import_library_content?
-      authorized?(:import_library_content)
+    def can_import_content?
+      authorized?(:import_content)
     end
 
-    def can_export_library_content?
-      authorized?(:export_library_content)
+    def can_export_content?
+      authorized?(:export_content)
     end
 
     def readable_promotion_paths

data/app/models/katello/concerns/hostgroup_extensions.rb

@@ -102,7 +102,9 @@ module Katello
         return true unless operatingsystem
 
         if operatingsystem.respond_to? :kickstart_repos
-          return operatingsystem.kickstart_repos(self).any? { |repo| repo[:id] == (content_facet&.kickstart_repository_id || content_facet&.kickstart_repository&.id) }
+          operatingsystem.kickstart_repos(self, content_facet: content_facet).any? do |repo|
+            repo[:id] == (content_facet&.kickstart_repository_id || content_facet&.kickstart_repository&.id)
+          end
         end
       end
 

data/app/models/katello/concerns/pulp_database_unit.rb

@@ -62,6 +62,48 @@ module Katello
         [Katello::Rpm, Katello::Srpm]
       end
 
+      def content_unit_association_id
+        "#{self.name.demodulize.underscore}_id".to_sym #Rpm => rpm_id
+      end
+
+      def repository_association_units
+        repository_association_class.name.demodulize.pluralize.underscore.to_sym
+      end
+
+      def content_units_name
+        self.name.demodulize.pluralize.underscore.to_sym
+      end
+
+      def installable_for_content_facet(facet, env = nil, content_view = nil)
+        repos = if env && content_view
+                  Katello::Repository.in_environment(env).in_content_views([content_view])
+                else
+                  facet.bound_repositories.pluck(:id)
+                end
+        facet.send("applicable_#{content_units_name}".to_sym).in_repositories(repos)
+      end
+
+      def installable_for_hosts(hosts = nil)
+        # Main goal of this query
+        # 1) Get me the applicable content units for these set of hosts
+        # 2) Now further prune this list. Only include units from repos that have been "enabled" on those hosts.
+        #    In other words, prune the list to only include the units in the "bound" repositories signified by
+        #    the inner join between ContentFacetRepository and Repository<Unit>
+
+        facet_repos = Katello::ContentFacetRepository.joins(:content_facet => :host).select(:repository_id)
+        facet_content_units = content_facet_association_class.joins(:content_facet => :host).select(content_unit_association_id)
+
+        if hosts
+          hosts = ::Host.where(id: hosts) if hosts.is_a?(Array)
+          facet_repos = facet_repos.merge(hosts).reorder(nil)
+          facet_content_units = facet_content_units.merge(hosts).reorder(nil)
+        end
+
+        self.joins(repository_association_units).
+            where(repository_association_class.table_name => {:repository_id => facet_repos,
+                                                              content_unit_association_id => facet_content_units}).distinct
+      end
+
       def with_identifiers(ids)
         ids = [ids] unless ids.is_a?(Array)
         ids.map!(&:to_s)