katello 3.16.0.rc3.1 → 3.16.0

data/app/lib/actions/katello/content_view/presenters/incremental_updates_presenter.rb

@@ -5,6 +5,7 @@ module Actions
         class IncrementalUpdatesPresenter < Helpers::Presenter::Base
           HUMANIZED_TYPES = {
             ::Katello::Erratum::CONTENT_TYPE => "Errata",
+            ::Katello::ModuleStream::CONTENT_TYPE => "Module Streams",
             ::Katello::Rpm::CONTENT_TYPE => "Packages",
             ::Katello::PuppetModule::CONTENT_TYPE => "Puppet Modules"
           }.freeze
@@ -25,7 +26,7 @@ module Actions
               if cvv
                 humanized_lines << "Content View: #{cvv.content_view.name} version #{cvv.version}"
                 humanized_lines << _("Added Content:")
-                [::Katello::Erratum, ::Katello::Rpm, ::Katello::PuppetModule].each do |content_type|
+                [::Katello::Erratum, ::Katello::ModuleStream, ::Katello::Rpm, ::Katello::PuppetModule].each do |content_type|
                   unless output[:added_units][content_type::CONTENT_TYPE].blank?
                     humanized_lines << "  #{HUMANIZED_TYPES[content_type::CONTENT_TYPE]}:"
                     humanized_lines += output[:added_units][content_type::CONTENT_TYPE].sort.map { |unit| "        #{unit}" }

data/app/lib/actions/katello/content_view/publish.rb

@@ -3,6 +3,8 @@ module Actions
   module Katello
     module ContentView
       class Publish < Actions::EntryAction
+        include ::Katello::ContentViewHelper
+        attr_accessor :version
         # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
         def plan(content_view, description = "", options = {})
           action_subject(content_view)
@@ -21,7 +23,11 @@ module Actions
             end
           end
 
+          # Add non-override components back in
+          options[:override_components] = include_other_components(options[:override_components], content_view)
+
           version = version_for_publish(content_view, options)
+          self.version = version
           library = content_view.organization.library
           history = ::Katello::ContentViewHistory.create!(:content_view_version => version,
                                                           :user => ::User.current.login,
@@ -32,20 +38,30 @@ module Actions
                                                           :triggered_by => options[:triggered_by]
                                                          )
           source_repositories = []
-          content_view.publish_repositories do |repositories|
+          content_view.publish_repositories(options[:override_cvvs]) do |repositories|
             source_repositories += [repositories]
           end
 
           sequence do
-            plan_action(ContentView::AddToEnvironment, version, library)
+            plan_action(ContentView::AddToEnvironment, version, library) unless options[:skip_promotion]
             repository_mapping = plan_action(ContentViewVersion::CreateRepos, version, source_repositories).repository_mapping
 
+            # Split Pulp 3 Yum repos out of the repository_mapping.  Only Pulp 3 RPM plugin has multi repo copy support.
+            separated_repo_map = separated_repo_mapping(repository_mapping)
+
+            if separated_repo_map[:pulp3_yum].keys.flatten.present? &&
+                SmartProxy.pulp_master.pulp3_support?(separated_repo_map[:pulp3_yum].keys.flatten.first)
+              plan_action(Repository::MultiCloneToVersion, separated_repo_map[:pulp3_yum], version)
+            end
+
             concurrence do
               source_repositories.each do |repositories|
                 sequence do
-                  plan_action(Repository::CloneToVersion, repositories, version, repository_mapping[repositories],
-                                                 :repos_units => options[:repos_units])
-                  plan_action(Repository::CloneToEnvironment, repository_mapping[repositories], library)
+                  if repositories.present? && separated_repo_map[:other].keys.include?(repositories)
+                    plan_action(Repository::CloneToVersion, repositories, version, repository_mapping[repositories],
+                                :repos_units => options[:repos_units])
+                  end
+                  plan_action(Repository::CloneToEnvironment, repository_mapping[repositories], library) unless options[:skip_promotion]
                 end
               end
 
@@ -55,16 +71,18 @@ module Actions
             end
             has_modules = content_view.publish_puppet_environment?
             plan_action(ContentViewPuppetEnvironment::CreateForVersion, version)
-            plan_action(ContentViewPuppetEnvironment::Clone, version, :environment => library,
-                :puppet_modules_present => has_modules)
-            plan_action(Candlepin::Environment::SetContent, content_view, library, content_view.content_view_environment(library))
-            plan_action(Katello::Foreman::ContentUpdate, library, content_view)
-            plan_action(ContentView::ErrataMail, content_view, library)
+            unless options[:skip_promotion]
+              plan_action(ContentViewPuppetEnvironment::Clone, version, :environment => library,
+                  :puppet_modules_present => has_modules)
+            end
+            plan_action(Candlepin::Environment::SetContent, content_view, library, content_view.content_view_environment(library)) unless options[:skip_promotion]
+            plan_action(Katello::Foreman::ContentUpdate, library, content_view) unless options[:skip_promotion]
+            plan_action(ContentView::ErrataMail, content_view, library) unless options[:skip_promotion]
             plan_self(history_id: history.id, content_view_id: content_view.id,
                       auto_publish_composite_ids: auto_publish_composite_ids(content_view),
                       content_view_version_name: version.name,
                       content_view_version_id: version.id,
-                      environment_id: library.id, user_id: ::User.current.id)
+                      environment_id: library.id, user_id: ::User.current.id, skip_promotion: options[:skip_promotion])
           end
         end
 
@@ -85,6 +103,7 @@ module Actions
 
           output[:content_view_id] = input[:content_view_id]
           output[:content_view_version_id] = input[:content_view_version_id]
+          output[:skip_promotion] = input[:skip_promotion]
         end
 
         def rescue_strategy_for_self
@@ -95,9 +114,11 @@ module Actions
           version = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
           version.update_content_counts!
           # update errata applicability counts for all hosts in the CV & Library
-          ::Katello::Host::ContentFacet.where(:content_view_id => input[:content_view_id],
-                                              :lifecycle_environment_id => input[:environment_id]).each do |facet|
-            facet.update_applicability_counts
+          unless input[:skip_promotion]
+            ::Katello::Host::ContentFacet.where(:content_view_id => input[:content_view_id],
+                                                :lifecycle_environment_id => input[:environment_id]).each do |facet|
+              facet.update_applicability_counts
+            end
           end
 
           history = ::Katello::ContentViewHistory.find(input[:history_id])
@@ -105,7 +126,7 @@ module Actions
           history.save!
           environment = ::Katello::KTEnvironment.find(input[:environment_id])
           view = ::Katello::ContentView.find(input[:content_view_id])
-          if SmartProxy.sync_needed?(environment) && Setting[:foreman_proxy_content_auto_sync]
+          if SmartProxy.sync_needed?(environment) && Setting[:foreman_proxy_content_auto_sync] && !input[:skip_promotion]
             ForemanTasks.async_task(ContentView::CapsuleSync,
                                     view,
                                     environment)
@@ -115,6 +136,20 @@ module Actions
 
         private
 
+        def include_other_components(override_components, content_view)
+          if override_components.present?
+            content_view.components.each do |component|
+              component_has_override = override_components.detect do |override_component|
+                component.content_view_id == override_component.content_view_id
+              end
+              unless component_has_override
+                override_components << component
+              end
+            end
+            override_components
+          end
+        end
+
         def repos_to_delete(content_view)
           if content_view.composite?
             library_instances = content_view.repositories_to_publish.map(&:library_instance_id)
@@ -132,7 +167,11 @@ module Actions
 
         def version_for_publish(content_view, options)
           if options[:minor] && options[:major]
-            content_view.create_new_version(options[:major], options[:minor])
+            if options[:override_components]
+              content_view.create_new_version(options[:major], options[:minor], options[:override_components])
+            else
+              content_view.create_new_version(options[:major], options[:minor])
+            end
           else
             content_view.create_new_version
           end

data/app/lib/actions/katello/content_view_version/incremental_update.rb

@@ -1,7 +1,9 @@
 module Actions
   module Katello
     module ContentViewVersion
+      # rubocop:disable Metrics/ClassLength
       class IncrementalUpdate < Actions::EntryAction
+        include ::Katello::ContentViewHelper
         attr_accessor :new_content_view_version
 
         HUMANIZED_TYPES = {
@@ -30,6 +32,19 @@ module Actions
           validate_environments(environments, old_version)
 
           new_minor = old_version.content_view.versions.where(:major => old_version.major).maximum(:minor) + 1
+          if SmartProxy.pulp_master.pulp3_repository_type_support?("yum") && is_composite
+            sequence do
+              publish_action = plan_action(::Actions::Katello::ContentView::Publish, old_version.content_view, description,
+                          :major => old_version.major, :minor => new_minor,
+                          :override_components => new_components, :skip_promotion => true)
+              if old_version.environments.present?
+                plan_action(::Actions::Katello::ContentView::Promote, publish_action.version,
+                            old_version.environments, true, description)
+              end
+            end
+            return
+          end
+
           self.new_content_view_version = old_version.content_view.create_new_version(old_version.major, new_minor, all_components)
           history = ::Katello::ContentViewHistory.create!(:content_view_version => new_content_view_version, :user => ::User.current.login,
                                                           :action => ::Katello::ContentViewHistory.actions[:publish],
@@ -41,6 +56,7 @@ module Actions
 
           sequence do
             repository_mapping = plan_action(ContentViewVersion::CreateRepos, new_content_view_version, repos_to_clone).repository_mapping
+            separated_repo_map = separated_repo_mapping(repository_mapping)
 
             repos_to_clone.each do |source_repos|
               plan_action(Repository::CloneToVersion,
@@ -51,11 +67,25 @@ module Actions
             end
 
             concurrence do
-              repos_to_clone.each do |source_repos|
-                copy_action_outputs += copy_repos(repository_mapping[source_repos],
-                                                  new_content_view_version,
-                                                  content,
-                                                  dep_solve)
+              if separated_repo_map[:pulp3_yum].keys.flatten.present?
+                extended_repo_mapping = pulp3_repo_mapping(separated_repo_map[:pulp3_yum], old_version)
+                unit_map = pulp3_content_mapping(content)
+
+                unless extended_repo_mapping.empty? || unit_map.values.flatten.empty?
+                  copy_action_outputs << plan_action(Pulp3::Repository::MultiCopyUnits, extended_repo_mapping, unit_map,
+                                                     dependency_solving: true).output
+                end
+              end
+
+              if separated_repo_map[:other].keys.flatten.present?
+                repos_to_clone.each do |source_repos|
+                  if separated_repo_map[:other].keys.include?(source_repos)
+                    copy_action_outputs += copy_repos(repository_mapping[source_repos],
+                                                      new_content_view_version,
+                                                      content,
+                                                      dep_solve)
+                  end
+                end
               end
 
               sequence do
@@ -68,11 +98,47 @@ module Actions
             plan_self(:content_view_id => old_version.content_view.id,
                       :new_content_view_version_id => self.new_content_view_version.id,
                       :environment_ids => environments.map(&:id), :user_id => ::User.current.id,
-                      :history_id => history.id, :copy_action_outputs => copy_action_outputs)
+                      :history_id => history.id, :copy_action_outputs => copy_action_outputs,
+                      :old_version => old_version.id)
             promote(new_content_view_version, environments)
           end
         end
 
+        def pulp3_content_mapping(content)
+          units = ::Katello::Erratum.with_identifiers(content[:errata_ids]) +
+            ::Katello::Rpm.with_identifiers(content[:package_ids])
+          unit_map = { :errata => [], :rpms => [] }
+          units.each do |unit|
+            if unit.class.name == "Katello::Erratum"
+              unit_map[:errata] << unit.id
+            elsif unit.class.name == "Katello::Rpm"
+              unit_map[:rpms] << unit.id
+            end
+          end
+          unit_map
+        end
+
+        def pulp3_repo_mapping(repo_mapping, old_version)
+          pulp3_repo_mapping = {}
+          repo_mapping.each do |source_repos, dest_repo|
+            old_version_repo = old_version.repositories.archived.find_by(root_id: dest_repo.root_id)
+
+            next if old_version_repo.version_href == old_version_repo.library_instance.version_href
+
+            source_library_repo = source_repos.first.library_instance? ? source_repos.first : source_repos.first.library_instance
+
+            source_repos = [source_library_repo]
+            if old_version_repo.version_href
+              base_version = old_version_repo.version_href.split("/")[-1].to_i
+            else
+              base_version = 0
+            end
+
+            pulp3_repo_mapping[source_repos.map(&:id)] = { dest_repo: dest_repo.id, base_version: base_version }
+          end
+          pulp3_repo_mapping
+        end
+
         def repos_to_copy(old_version, new_components)
           old_version.archived_repos.map do |source_repo|
             components_repo_instances(source_repo, new_components)
@@ -142,22 +208,48 @@ module Actions
                       ::Katello::PuppetModule::CONTENT_TYPE => []
                     }
 
-          input[:copy_action_outputs].each do |copy_output|
-            copy_output[:pulp_tasks].each do |pulp_task|
-              pulp_task[:result][:units_successful].each do |unit|
-                type = unit['type_id']
-                unit = unit['unit_key']
-                case type
-                when ::Katello::Erratum::CONTENT_TYPE
-                  content[::Katello::Erratum::CONTENT_TYPE] << unit['id']
-                when ::Katello::ModuleStream::CONTENT_TYPE
-                  content[::Katello::ModuleStream::CONTENT_TYPE] << "#{unit['name']}:#{unit['stream']}:#{unit['version']}"
-                when ::Katello::Rpm::CONTENT_TYPE
-                  content[::Katello::Rpm::CONTENT_TYPE] << ::Katello::Util::Package.build_nvra(unit)
-                when ::Katello::Deb::CONTENT_TYPE
-                  content[::Katello::Deb::CONTENT_TYPE] << "#{unit['name']}_#{unit['version']}_#{unit['architecture']}"
-                when ::Katello::PuppetModule::CONTENT_TYPE
-                  content[::Katello::PuppetModule::CONTENT_TYPE] << "#{unit['author']}-#{unit['name']}-#{unit['version']}"
+          base_repos = ::Katello::ContentViewVersion.find(input[:old_version]).repositories
+          new_repos = ::Katello::ContentViewVersion.find(input[:new_content_view_version_id]).repositories
+
+          if input[:copy_action_outputs].present? && input[:copy_action_outputs].last[:pulp_tasks].present?
+            if input[:copy_action_outputs].last[:pulp_tasks].last[:pulp_href]&.include?("/pulp/api/v3/")
+              new_repos.each do |new_repo|
+                matched_old_repo = base_repos.where(root_id: new_repo.root_id).first
+
+                new_errata = new_repo.errata - matched_old_repo.errata
+                new_module_streams = new_repo.module_streams - matched_old_repo.module_streams
+                new_rpms = new_repo.rpms - matched_old_repo.rpms
+
+                new_errata.each do |erratum|
+                  content[::Katello::Erratum::CONTENT_TYPE] << erratum.errata_id
+                end
+                new_module_streams.each do |module_stream|
+                  content[::Katello::ModuleStream::CONTENT_TYPE] <<
+                    "#{module_stream.name}:#{module_stream.stream}:#{module_stream.version}"
+                end
+                new_rpms.each do |rpm|
+                  content[::Katello::Rpm::CONTENT_TYPE] << rpm.nvra
+                end
+              end
+            else
+              input[:copy_action_outputs].each do |copy_output|
+                copy_output[:pulp_tasks].each do |pulp_task|
+                  pulp_task[:result][:units_successful].each do |unit|
+                    type = unit['type_id']
+                    unit = unit['unit_key']
+                    case type
+                    when ::Katello::Erratum::CONTENT_TYPE
+                      content[::Katello::Erratum::CONTENT_TYPE] << unit['id']
+                    when ::Katello::ModuleStream::CONTENT_TYPE
+                      content[::Katello::ModuleStream::CONTENT_TYPE] << "#{unit['name']}:#{unit['stream']}:#{unit['version']}"
+                    when ::Katello::Rpm::CONTENT_TYPE
+                      content[::Katello::Rpm::CONTENT_TYPE] << ::Katello::Util::Package.build_nvra(unit)
+                    when ::Katello::Deb::CONTENT_TYPE
+                      content[::Katello::Deb::CONTENT_TYPE] << "#{unit['name']}_#{unit['version']}_#{unit['architecture']}"
+                    when ::Katello::PuppetModule::CONTENT_TYPE
+                      content[::Katello::PuppetModule::CONTENT_TYPE] << "#{unit['author']}-#{unit['name']}-#{unit['version']}"
+                    end
+                  end
                 end
               end
             end
@@ -174,10 +266,12 @@ module Actions
           history.status = ::Katello::ContentViewHistory::SUCCESSFUL
           history.save!
 
-          version.repositories.each do |repo|
-            SmartProxy.pulp_master.pulp_api.extensions.send(:module_default).
-              copy(repo.library_instance.pulp_id,
-              repo.pulp_id)
+          unless SmartProxy.pulp_master.pulp3_support?(version.repositories.first)
+            version.repositories.each do |repo|
+              SmartProxy.pulp_master.pulp_api.extensions.send(:module_default).
+                copy(repo.library_instance.pulp_id,
+                repo.pulp_id)
+            end
           end
         end
 

data/app/lib/actions/katello/host/attach_subscriptions.rb

@@ -8,12 +8,16 @@ module Actions
 
         def plan(host, pools_with_quantities_params)
           action_subject(host)
+
+          subscription_facet = host.subscription_facet
+          fail _("Register host '%s' before attaching subscriptions") % host.name unless subscription_facet
+
           sequence do
             pool_ids = []
             pools_with_quantities = pools_with_quantities_params.map do |pool_with_quantity|
               ::Katello::PoolWithQuantities.fetch(pool_with_quantity)
             end
-            existing_pool_ids = host.subscription_facet.candlepin_consumer.pool_ids
+            existing_pool_ids = subscription_facet.candlepin_consumer.pool_ids
             pools_with_quantities.each do |pool_with_quantities|
               unless existing_pool_ids.include?(pool_with_quantities.pool.cp_id.to_s)
                 pool_ids << pool_with_quantities.pool.id

data/app/lib/actions/katello/repository/multi_clone_contents.rb

@@ -0,0 +1,66 @@
+module Actions
+  module Katello
+    module Repository
+      class MultiCloneContents < Actions::Base
+        include Actions::Katello::PulpSelector
+        def plan(extended_repo_mapping, options)
+          generate_metadata = options.fetch(:generate_metadata, true)
+          copy_contents = options.fetch(:copy_contents, true)
+          solve_dependencies = options.fetch(:solve_dependencies, false)
+
+          sequence do
+            if copy_contents
+              plan_action(Pulp3::Orchestration::Repository::MultiCopyAllUnits,
+                          extended_repo_mapping,
+                          SmartProxy.pulp_master,
+                          solve_dependencies: solve_dependencies)
+            end
+
+            extended_repo_mapping.each do |source_repos, dest_repo_map|
+              if generate_metadata
+                metadata_generate(source_repos, dest_repo_map[:dest_repo], dest_repo_map[:filters])
+              end
+            end
+
+            extended_repo_mapping.values.each do |dest_repo_map|
+              plan_action(Katello::Repository::IndexContent, id: dest_repo_map[:dest_repo].id)
+            end
+          end
+        end
+
+        def metadata_generate(source_repositories, new_repository, filters)
+          metadata_options = {}
+
+          if source_repositories.count == 1 && filters.empty?
+            metadata_options[:source_repository] = source_repositories.first
+          end
+
+          check_matching_content = ::Katello::RepositoryTypeManager.find(new_repository.content_type).metadata_publish_matching_check
+          if new_repository.environment && source_repositories.count == 1 && check_matching_content
+            match_check_output = plan_action(Katello::Repository::CheckMatchingContent,
+                                :source_repo_id => source_repositories.first.id,
+                                :target_repo_id => new_repository.id).output
+
+            metadata_options[:matching_content] = match_check_output[:matching_content]
+          end
+
+          plan_action(Katello::Repository::MetadataGenerate, new_repository, metadata_options)
+          unless source_repositories.first.saved_checksum_type == new_repository.saved_checksum_type
+            checksum_mapping = {}
+            repository_mapping.each do |source_repos, dest_repo|
+              checksum_mapping[dest_repo.id] = source_repos.first.saved_checksum_type
+            end
+            plan_self(:checksum_mapping => checksum_mapping)
+          end
+        end
+
+        def finalize
+          input[:checksum_mapping].each do |repo_id, checksum_type|
+            repository = ::Katello::Repository.find(repo_id)
+            repository.update!(saved_checksum_type: checksum_type) if (repository && checksum_type)
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/katello/repository/multi_clone_to_version.rb

@@ -0,0 +1,30 @@
+module Actions
+  module Katello
+    module Repository
+      class MultiCloneToVersion < Actions::Base
+        def plan(repository_mapping, content_view_version, options = {})
+          incremental = options.fetch(:incremental, false)
+          content_view = content_view_version.content_view
+          extended_repo_map = extended_repo_mapping(repository_mapping, content_view, incremental)
+          sequence do
+            plan_action(::Actions::Katello::Repository::MultiCloneContents, extended_repo_map,
+                        copy_contents: true,
+                        solve_dependencies: content_view.solve_dependencies,
+                        metadata_generate: !incremental)
+          end
+        end
+
+        def extended_repo_mapping(repo_map, content_view, incremental)
+          # Example: {[source_repos] => {dest_repo: dest_repo, filters: filters}}
+          extended_repo_map = {}
+          repo_map.each do |source_repos, dest_repo|
+            filters = incremental ? [] : content_view.filters.applicable(source_repos.first)
+            extended_repo_map[source_repos] = { :dest_repo => dest_repo,
+                                                :filters => filters }
+          end
+          extended_repo_map
+        end
+      end
+    end
+  end
+end