katello 3.8.1 → 3.9.0.rc1

data/app/lib/actions/katello/event_queue/suspended_action.rb

@@ -6,8 +6,8 @@ module Actions
           @suspended_action = suspended_action
         end
 
-        def notify_queue_item(event_type, object_id, created_at)
-          @suspended_action << Monitor::Event[event_type, object_id, created_at]
+        def notify_count(processed_count)
+          @suspended_action << Monitor::Count[processed_count, Time.now]
         end
 
         def notify_ready

data/app/lib/actions/katello/host/hypervisors_update.rb

@@ -13,37 +13,86 @@ module Actions
         end
 
         def finalize
-          hypervisors = input[:hypervisors]
+          @hypervisors = input[:hypervisors]
+          return unless @hypervisors
 
-          if hypervisors
-            User.as_anonymous_admin do
-              hypervisors.each { |hypervisor| update_or_create_hypervisor(hypervisor) }
+          User.as_anonymous_admin do
+            load_resources
+
+            @hosts.each do |uuid, host|
+              update_subscription_facet(uuid, host)
             end
           end
         end
 
-        def update_or_create_hypervisor(hypervisor_json)
-          organization = ::Organization.find_by(:label => hypervisor_json[:organization_label])
+        # Loads all resources needed for refreshing subscription facet
+        def load_resources
+          @organizations = ::Organization.where(label: hypervisors_field(:organization_label)).map { |org| [org.label, org] }.to_h
+          @hosts = {}
+
+          load_hosts_by_uuid
+          load_hosts_by_duplicate_name
+          create_missing_hosts
+
+          candlepin_data = ::Katello::Resources::Candlepin::Consumer.get_all(@hosts.keys)
+          @candlepin_attributes = candlepin_data.map { |consumer| [consumer[:uuid], consumer] }.to_h
+        end
+
+        def load_hosts_by_uuid
+          hosts_by_uuid = ::Host.eager_load(:subscription_facet).where(katello_subscription_facets: { uuid: hypervisors_field(:uuid) })
+          @hosts.merge(hosts_by_uuid.map { |host| [host.subscription_facet.uuid, host] }.to_h)
+        end
+
+        def load_hosts_by_duplicate_name
+          duplicate_names, duplicate_name_orgs = generate_duplicates_list
+
+          hosts_by_dup_name = ::Host.preload(:subscription_facet).where(name: duplicate_names.keys)
+
+          hosts_by_dup_name.each do |host|
+            validate_host_organization(host, duplicate_name_orgs[host.name].try(:id))
+          end
+
+          @hosts.merge!(hosts_by_dup_name.map { |host| [duplicate_names[host.name], host] }.to_h)
+        end
+
+        def create_missing_hosts
+          # remaining hypervisors
+          @hypervisors.each do |hypervisor|
+            next if @hosts.key?(hypervisor[:uuid])
+            duplicate_name, org = duplicate_name(hypervisor)
+            @hosts[hypervisor[:uuid]] = create_host_for_hypervisor(duplicate_name, org)
+          end
+        end
+
+        def generate_duplicates_list
+          duplicate_names = {}
+          duplicate_name_orgs = {}
+          @hypervisors.each do |hypervisor|
+            next if @hosts.key?(hypervisor[:uuid])
+
+            duplicate_name, org = duplicate_name(hypervisor)
+            duplicate_names[duplicate_name] = hypervisor[:uuid]
+            duplicate_name_orgs[duplicate_name] = org
+          end
 
-          # Since host names must be unique yet hypervisors may have unique subscription
-          # facets in different orgs
-          sanitized_name = ::Katello::Host::SubscriptionFacet.sanitize_name(hypervisor_json[:name])
-          duplicate_name = "virt-who-#{sanitized_name}-#{organization.id}"
-          host = ::Katello::Host::SubscriptionFacet.find_by(:uuid => hypervisor_json[:uuid]).try(:host)
-          host ||= ::Host.find_by(:name => duplicate_name)
-          if host && host.organization.try(:id) != organization.id
+          [duplicate_names, duplicate_name_orgs]
+        end
+
+        def validate_host_organization(host, organization)
+          if host.organization_id.nil? || host.organization_id != organization
             fail _("Host '%{name}' does not belong to an organization") % {:name => host.name} unless host.organization
-            host = nil
           end
+        end
 
-          host ||= create_host_for_hypervisor(duplicate_name, organization)
-          host.subscription_facet ||= ::Katello::Host::SubscriptionFacet.new
-          host.subscription_facet.host_id = host.id
-          host.subscription_facet.uuid = hypervisor_json[:uuid]
-          host.subscription_facet.import_database_attributes(host.subscription_facet.candlepin_consumer.consumer_attributes)
-          host.subscription_facet.save!
-          host.subscription_facet.update_subscription_status
-          host.save!
+        # extracts a single field from a given list og hypervisors data.
+        def hypervisors_field(field, hypervisors = @hypervisors)
+          hypervisors.map { |h| h[field] }.uniq
+        end
+
+        def duplicate_name(hypervisor)
+          organization = @organizations[hypervisor[:organization_label]]
+          sanitized_name = ::Katello::Host::SubscriptionFacet.sanitize_name(hypervisor[:name])
+          ["virt-who-#{sanitized_name}-#{organization.id}", organization]
         end
 
         def create_host_for_hypervisor(name, organization, location = nil)
@@ -54,6 +103,17 @@ module Actions
           host
         end
 
+        def update_subscription_facet(uuid, host)
+          host.subscription_facet ||= host.build_subscription_facet(uuid: uuid)
+          if @candlepin_attributes.key?(uuid)
+            host.subscription_facet.candlepin_consumer.consumer_attributes = @candlepin_attributes[uuid]
+            host.subscription_facet.import_database_attributes
+            host.subscription_facet.save!
+            host.subscription_facet.update_subscription_status(@candlepin_attributes[uuid].try(:[], :entitlementStatus))
+          end
+          host.save!
+        end
+
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end

data/app/lib/actions/katello/organization/manifest_refresh.rb

@@ -29,7 +29,7 @@ module Actions
             import_products = plan_action(Candlepin::Owner::ImportProducts, :organization_id => organization.id, :dependency => owner_import.output)
 
             if manifest_update && SETTINGS[:katello][:use_pulp]
-              repositories = ::Katello::Repository.in_default_view.where(:product_id => ::Katello::Product.redhat.in_org(organization))
+              repositories = ::Katello::Repository.in_default_view.in_product(::Katello::Product.redhat.in_org(organization))
               repositories.each do |repo|
                 plan_action(Katello::Repository::RefreshRepository, repo, :dependency => import_products.output)
               end

data/app/lib/actions/katello/product/content_create.rb

@@ -4,63 +4,55 @@ module Actions
       class ContentCreate < Actions::Base
         middleware.use Actions::Middleware::RemoteAction
 
-        def plan(repository)
+        def plan(root)
           sequence do
-            if repository.content_id.nil?
+            if root.content_id.nil?
               content_create = plan_action(Candlepin::Product::ContentCreate,
-                                           owner:       repository.product.organization.label,
-                                           name:        repository.name,
-                                           type:        repository.content_type,
-                                           arches:      repository.arch == "noarch" ? nil : repository.arch,
-                                           label:       repository.custom_content_label,
-                                           content_url: content_url(repository))
+                                           owner:       root.product.organization.label,
+                                           name:        root.name,
+                                           type:        root.content_type,
+                                           arches:      root.arch == "noarch" ? nil : root.arch,
+                                           label:       root.custom_content_label,
+                                           content_url: root.custom_content_path)
               content_id = content_create.output[:response][:id]
-              plan_action(Candlepin::Product::ContentAdd, owner: repository.product.organization.label,
-                                    product_id: repository.product.cp_id,
+              plan_action(Candlepin::Product::ContentAdd, owner: root.product.organization.label,
+                                    product_id: root.product.cp_id,
                                     content_id: content_id)
 
             else
-              content_id = repository.content_id
+              content_id = root.content_id
             end
 
-            if repository.gpg_key
+            if root.gpg_key
               plan_action(Candlepin::Product::ContentUpdate,
-                          owner:       repository.organization.label,
+                          owner:       root.organization.label,
                           content_id:  content_id,
-                          name:        repository.name,
-                          type:        repository.content_type,
-                          arches:      repository.arch == "noarch" ? "" : repository.arch,
-                          label:       repository.custom_content_label,
-                          content_url: content_url(repository),
-                          gpg_key_url: repository.yum_gpg_key_url)
+                          name:        root.name,
+                          type:        root.content_type,
+                          arches:      root.arch == "noarch" ? "" : root.arch,
+                          label:       root.custom_content_label,
+                          content_url: root.custom_content_path,
+                          gpg_key_url: root.library_instance.yum_gpg_key_url)
             end
 
-            plan_self(repository_id: repository.id, content_id: content_id)
+            plan_self(root_repository_id: root.id, content_id: content_id)
           end
         end
 
         def finalize
-          repository = ::Katello::Repository.find(input[:repository_id])
-          repository.content_id = input[:content_id]
-          repository.save!
-
-          content = ::Katello::Content.create!(name: repository.name,
-                                               organization_id: repository.product.organization_id,
-                                               cp_content_id: repository.content_id,
-                                               content_type: repository.content_type,
-                                               label: repository.custom_content_label,
-                                               content_url: content_url(repository),
+          root = ::Katello::RootRepository.find(input[:root_repository_id])
+          root.update_attributes(:content_id => input[:content_id])
+
+          content = ::Katello::Content.create!(name: root.name,
+                                               organization_id: root.product.organization_id,
+                                               cp_content_id: root.content_id,
+                                               content_type: root.content_type,
+                                               label: root.custom_content_label,
+                                               content_url: root.custom_content_path,
                                                vendor: ::Katello::Provider::CUSTOM)
 
           #custom product content is always enabled by default
-          ::Katello::ProductContent.create!(product: repository.product, content: content, enabled: true)
-        end
-
-        private
-
-        def content_url(repository)
-          ::Katello::Glue::Pulp::Repos.custom_content_path(repository.product,
-                                                           repository.label)
+          ::Katello::ProductContent.create!(product: root.product, content: content, enabled: true)
         end
       end
     end

data/app/lib/actions/katello/product/content_destroy.rb

@@ -2,25 +2,24 @@ module Actions
   module Katello
     module Product
       class ContentDestroy < Actions::Base
-        def plan(repository)
-          fail _("Cannot delete redhat product content") if repository.product.redhat?
+        def plan(root_repository)
+          fail _("Cannot delete redhat product content") if root_repository.product.redhat?
           sequence do
             plan_action(Candlepin::Product::ContentRemove,
-                        owner: repository.product.organization.label,
-                        product_id: repository.product.cp_id,
-                        content_id: repository.content_id)
+                        owner: root_repository.product.organization.label,
+                        product_id: root_repository.product.cp_id,
+                        content_id: root_repository.content_id)
 
-            katello_content_id = repository.product.product_content_by_id(repository.content_id).content_id
-
-            ::Katello::ProductContent.where(product_id: repository.product_id,
+            katello_content_id = root_repository.content&.id
+            ::Katello::ProductContent.where(product_id: root_repository.product_id,
                                             content_id: katello_content_id).destroy_all
 
-            if repository.other_repos_with_same_content.empty?
+            if root_repository.repositories.count <= 1
               plan_action(Candlepin::Product::ContentDestroy,
-                          owner: repository.product.organization.label,
-                          content_id: repository.content_id)
+                          owner: root_repository.product.organization.label,
+                          content_id: root_repository.content_id)
 
-              ::Katello::Content.find(katello_content_id).destroy!
+              ::Katello::Content.find(katello_content_id)&.destroy!
             end
           end
         end

data/app/lib/actions/katello/product/create.rb

@@ -25,7 +25,6 @@ module Actions
             action_subject product, :cp_id => product.cp_id
 
             plan_self
-            plan_action ::Actions::Pulp::Repository::EnsureSyncNotification
             plan_action Katello::Product::ReindexSubscriptions, product
           end
         end

data/app/lib/actions/katello/product/update.rb

@@ -13,9 +13,7 @@ module Actions
               product.previous_changes.key?('ssl_client_key_id'))
             plan_action(::Actions::Katello::Product::RepositoriesCertsReset, product)
           end
-
           product.reload
-          plan_action(::Actions::Pulp::Repos::Update, product) if ::SETTINGS[:katello][:use_pulp]
         end
       end
     end

data/app/lib/actions/katello/repository/clear.rb

@@ -8,6 +8,8 @@ module Actions
            Pulp::Repository::RemovePackageGroup,
            Pulp::Repository::RemoveDistribution,
            Pulp::Repository::RemoveYumMetadataFile,
+           Pulp::Repository::RemoveModuleDefault,
+           Pulp::Repository::RemoveModuleStream,
            Pulp::Repository::RemoveFile,
            Pulp::Repository::RemovePuppetModule,
            Pulp::Repository::RemoveDockerManifest,

data/app/lib/actions/katello/repository/clone_to_environment.rb

@@ -13,8 +13,6 @@ module Actions
             else
               #only clear if it should be empty, but its not
               plan_action(Repository::Clear, clone) if (!clone.yum? || !clone.empty_in_pulp?)
-              clone.copy_library_instance_attributes
-              clone.save!
 
               if ::Katello::Repository.needs_distributor_updates([clone], ::Katello::CapsuleContent.new(::SmartProxy.default_capsule)).first
                 plan_action(Pulp::Repository::Refresh, clone)

data/app/lib/actions/katello/repository/clone_to_version.rb

@@ -5,13 +5,16 @@ module Actions
         # allows accessing the build object from the superior action
         attr_accessor :new_repository
 
-        def plan(repositories, content_view_version, incremental = false)
+        def plan(repositories, content_view_version, options = {})
+          incremental = options.fetch(:incremental, false)
           content_view = content_view_version.content_view
           filters = incremental ? [] : content_view.filters.applicable(repositories.first)
 
           self.new_repository = repositories.first.build_clone(content_view: content_view,
                                                                version: content_view_version)
 
+          rpm_filenames = extract_rpm_filenames(options.fetch(:repos_units, nil), repositories.first.label)
+
           sequence do
             plan_action(Repository::Create, new_repository, true, false)
 
@@ -22,8 +25,8 @@ module Actions
             else
               repositories.each do |repository|
                 if new_repository.yum?
-                  plan_action(Repository::CloneYumContent, repository, new_repository, filters, !incremental,
-                              :generate_metadata => !incremental, :index_content => !incremental, :simple_clone => incremental)
+                  plan_action(Repository::CloneYumContent, repository, new_repository, filters, :purge_empty_units => !incremental,
+                              :generate_metadata => !incremental, :index_content => !incremental, :simple_clone => incremental, :rpm_filenames => rpm_filenames)
                 elsif new_repository.deb?
                   plan_action(Repository::CloneDebContent, repository, new_repository, filters, !incremental,
                               :generate_metadata => !incremental, :index_content => !incremental, :simple_clone => incremental)
@@ -38,6 +41,15 @@ module Actions
             end
           end
         end
+
+        def extract_rpm_filenames(repos_units, repo_label)
+          return if repos_units.blank?
+
+          repo_units = repos_units.detect { |r| r[:label] == repo_label }
+          return if repo_units.blank?
+
+          repo_units.fetch(:rpm_filenames, nil)
+        end
       end
     end
   end

data/app/lib/actions/katello/repository/clone_yum_content.rb

@@ -3,9 +3,13 @@ module Actions
     module Repository
       class CloneYumContent < Actions::Base
         # rubocop:disable MethodLength
-        def plan(source_repo, target_repo, filters, purge_empty_units, options = {})
+        # rubocop:disable Metrics/CyclomaticComplexity TODO: refactor to push everything into options hash
+        # rubocop:disable Metrics/PerceivedComplexity
+        def plan(source_repo, target_repo, filters, options = {})
           generate_metadata = options.fetch(:generate_metadata, true)
           index_content = options.fetch(:index_content, true)
+          rpm_filenames = options.fetch(:rpm_filenames, {})
+          purge_empty_units = options.fetch(:purge_empty_units, {})
 
           copy_clauses = nil
           remove_clauses = nil
@@ -19,6 +23,19 @@ module Actions
             remove_clauses = clause_gen.remove_clause
           end
 
+          # if we are providing a list of files, process that here and override filters
+          if rpm_filenames.present?
+            # log a warning if we are overriding the list of RPMs and using a filter
+            # ensure we have all the files we want to copy
+            rpms_available = source_repo.rpms.pluck(:filename)
+            rpm_filenames.each do |filename|
+              fail "%s not available in repository %s" % [filename, source_repo.label] unless rpms_available.include? filename
+            end
+            copy_clauses = { 'filename' => { '$in' => rpm_filenames } }
+            remove_clauses = nil
+            Rails.logger.warn("Filters on content view have been overridden by passed-in filename list during publish") if filters.any?
+          end
+
           sequence do
             plan_copy(Pulp::Repository::CopySrpm, source_repo, target_repo)
 
@@ -39,6 +56,8 @@ module Actions
             end
             plan_copy(Pulp::Repository::CopyYumMetadataFile, source_repo, target_repo)
             plan_copy(Pulp::Repository::CopyDistribution, source_repo, target_repo)
+            plan_copy(Pulp::Repository::CopyModuleStream, source_repo, target_repo, nil)
+            plan_copy(Pulp::Repository::CopyModuleDefault, source_repo, target_repo)
 
             # Check for matching content before indexing happens, the content in pulp is
             # actually updated, but it is not reflected in the database yet.
@@ -58,7 +77,7 @@ module Actions
               plan_action(Katello::Repository::IndexPackageGroups, target_repo)
             end
 
-            source_repository = filters.empty? ? source_repo : nil
+            source_repository = filters.empty? && rpm_filenames.empty? ? source_repo : nil
 
             if generate_metadata
               plan_action(Katello::Repository::MetadataGenerate,

data/app/lib/actions/katello/repository/clone_yum_metadata.rb

@@ -7,7 +7,7 @@ module Actions
             # Check for matching content before indexing happens, the content in pulp is
             # actually updated, but it is not reflected in the database yet.
             output = {}
-            if !target_repo.previous_changes.include?(:unprotected) &&
+            if !target_repo.root.previous_changes.include?(:unprotected) &&
                 target_repo.environment && !options[:force_yum_metadata_regeneration]
               output = plan_action(Katello::Repository::CheckMatchingContent,
                                    :source_repo_id => source_repo.id,

data/app/lib/actions/katello/repository/correct_checksum.rb

@@ -8,12 +8,12 @@ module Actions
 
         def finalize
           ::User.current = ::User.anonymous_admin
-          repo = ::Katello::Repository.find(input[:repo_id])
+          root = ::Katello::Repository.find(input[:repo_id]).root
 
-          if repo.pulp_scratchpad_checksum_type &&
-              repo.pulp_scratchpad_checksum_type != repo.source_repo_checksum_type
-            repo.source_repo_checksum_type = repo.pulp_scratchpad_checksum_type
-            repo.save!
+          if root.pulp_scratchpad_checksum_type &&
+              root.pulp_scratchpad_checksum_type != root.source_repo_checksum_type
+            root.source_repo_checksum_type = root.pulp_scratchpad_checksum_type
+            root.save!
           end
         ensure
           ::User.current = nil