katello 3.16.0.rc4 → 3.16.1

data/app/lib/actions/katello/repository/destroy.rb

@@ -7,9 +7,11 @@ module Actions
 
         # options:
         #   skip_environment_update - defaults to false. skips updating the CP environment
+        #   destroy_content - can be disabled to skip Candlepin remove_content
         def plan(repository, options = {})
           skip_environment_update = options.fetch(:skip_environment_update, false) ||
               options.fetch(:organization_destroy, false)
+          destroy_content = options.fetch(:destroy_content, true)
           action_subject(repository)
 
           unless repository.destroyable?
@@ -29,7 +31,9 @@ module Actions
             if repository.redhat?
               handle_redhat_content(repository) unless skip_environment_update
             else
-              handle_custom_content(repository) unless skip_environment_update
+              if destroy_content && !skip_environment_update
+                handle_custom_content(repository)
+              end
             end
           end
         end

data/app/lib/actions/katello/repository/multi_clone_contents.rb

@@ -0,0 +1,62 @@
+module Actions
+  module Katello
+    module Repository
+      class MultiCloneContents < Actions::Base
+        include Actions::Katello::PulpSelector
+        def plan(extended_repo_mapping, options)
+          generate_metadata = options.fetch(:generate_metadata, true)
+          copy_contents = options.fetch(:copy_contents, true)
+          solve_dependencies = options.fetch(:solve_dependencies, false)
+
+          sequence do
+            if copy_contents
+              plan_action(Pulp3::Orchestration::Repository::MultiCopyAllUnits,
+                          extended_repo_mapping,
+                          SmartProxy.pulp_master,
+                          solve_dependencies: solve_dependencies)
+            end
+
+            extended_repo_mapping.each do |source_repos, dest_repo_map|
+              if generate_metadata
+                metadata_generate(source_repos, dest_repo_map[:dest_repo], dest_repo_map[:filters])
+              end
+            end
+
+            extended_repo_mapping.values.each do |dest_repo_map|
+              plan_action(Katello::Repository::IndexContent, id: dest_repo_map[:dest_repo].id)
+            end
+          end
+        end
+
+        def metadata_generate(source_repositories, new_repository, filters)
+          metadata_options = {}
+
+          if source_repositories.count == 1 && filters.empty?
+            metadata_options[:source_repository] = source_repositories.first
+          end
+
+          check_matching_content = ::Katello::RepositoryTypeManager.find(new_repository.content_type).metadata_publish_matching_check
+          if new_repository.environment && source_repositories.count == 1 && check_matching_content
+            match_check_output = plan_action(Katello::Repository::CheckMatchingContent,
+                                :source_repo_id => source_repositories.first.id,
+                                :target_repo_id => new_repository.id).output
+
+            metadata_options[:matching_content] = match_check_output[:matching_content]
+          end
+
+          plan_action(Katello::Repository::MetadataGenerate, new_repository, metadata_options)
+          unless source_repositories.first.saved_checksum_type == new_repository.saved_checksum_type
+            plan_self(:source_checksum_type => source_repositories.first.saved_checksum_type,
+                      :target_repo_id => new_repository.id)
+          end
+        end
+
+        def finalize
+          repository = ::Katello::Repository.find(input[:target_repo_id])
+          source_checksum_type = input[:source_checksum_type]
+          repository.update!(saved_checksum_type: source_checksum_type) if (repository && source_checksum_type)
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/katello/repository/multi_clone_to_version.rb

@@ -0,0 +1,30 @@
+module Actions
+  module Katello
+    module Repository
+      class MultiCloneToVersion < Actions::Base
+        def plan(repository_mapping, content_view_version, options = {})
+          incremental = options.fetch(:incremental, false)
+          content_view = content_view_version.content_view
+          extended_repo_map = extended_repo_mapping(repository_mapping, content_view, incremental)
+          sequence do
+            plan_action(::Actions::Katello::Repository::MultiCloneContents, extended_repo_map,
+                        copy_contents: true,
+                        solve_dependencies: content_view.solve_dependencies,
+                        metadata_generate: !incremental)
+          end
+        end
+
+        def extended_repo_mapping(repo_map, content_view, incremental)
+          # Example: {[source_repos] => {dest_repo: dest_repo, filters: filters}}
+          extended_repo_map = {}
+          repo_map.each do |source_repos, dest_repo|
+            filters = incremental ? [] : content_view.filters.applicable(source_repos.first)
+            extended_repo_map[source_repos] = { :dest_repo => dest_repo,
+                                                :filters => filters }
+          end
+          extended_repo_map
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/katello/repository/sync.rb

@@ -34,38 +34,47 @@ module Actions
 
           pulp_sync_options = {}
           pulp_sync_options[:download_policy] = ::Runcible::Models::YumImporter::DOWNLOAD_ON_DEMAND if validate_contents && repo.yum?
+
+          #pulp2 options
           pulp_sync_options[:force_full] = true if skip_metadata_check && repo.yum?
           pulp_sync_options[:repair_sync] = true if validate_contents && repo.deb?
           pulp_sync_options[:remove_missing] = false if incremental
+          pulp_sync_options[:source_url] = source_url
+
+          #pulp3 options
+          pulp_sync_options[:optimize] = false if skip_metadata_check && repo.yum?
 
           sequence do
-            plan_action(Pulp::Repository::RemoveUnits, :repo_id => repo.id, :content_unit_type => ::Katello::YumMetadataFile::CONTENT_TYPE) if validate_contents && repo.yum?
-            sync_args = {:smart_proxy_id => SmartProxy.pulp_master.id, :repo_id => repo.id, :source_url => source_url, :options => pulp_sync_options}
-            sync_action = plan_pulp_action([Actions::Pulp::Orchestration::Repository::Sync,
-                                            Actions::Pulp3::Orchestration::Repository::Sync],
-                                           repo,
-                                           SmartProxy.pulp_master,
-                                           sync_args)
-            output = sync_action.output
-
-            contents_changed = skip_metadata_check || output[:contents_changed]
-
-            plan_action(Katello::Repository::IndexContent, :id => repo.id, :contents_changed => contents_changed)
-            plan_action(Katello::Foreman::ContentUpdate, repo.environment, repo.content_view, repo)
-            plan_action(Katello::Repository::FetchPxeFiles, :id => repo.id)
-            plan_action(Katello::Repository::CorrectChecksum, repo)
-            concurrence do
-              plan_action(Pulp::Repository::Download, :pulp_id => repo.pulp_id, :options => {:verify_all_units => true}) if validate_contents && repo.yum?
-              plan_action(Katello::Repository::MetadataGenerate, repo, :force => true) if skip_metadata_check && repo.yum?
-              plan_action(Katello::Repository::ErrataMail, repo, nil, contents_changed)
-              if generate_applicability
-                regenerate_applicability(repo, contents_changed)
+            if SmartProxy.pulp_master.pulp3_support?(repo) && validate_contents
+              plan_action(Katello::Repository::VerifyChecksum, repo)
+            else
+              plan_action(Pulp::Repository::RemoveUnits, :repo_id => repo.id, :content_unit_type => ::Katello::YumMetadataFile::CONTENT_TYPE) if validate_contents && repo.yum?
+              sync_action = plan_pulp_action([Actions::Pulp::Orchestration::Repository::Sync,
+                                              Actions::Pulp3::Orchestration::Repository::Sync],
+                                             repo,
+                                             SmartProxy.pulp_master,
+                                             pulp_sync_options)
+              output = sync_action.output
+
+              contents_changed = skip_metadata_check || output[:contents_changed]
+
+              plan_action(Katello::Repository::IndexContent, :id => repo.id, :contents_changed => contents_changed)
+              plan_action(Katello::Foreman::ContentUpdate, repo.environment, repo.content_view, repo)
+              plan_action(Katello::Repository::FetchPxeFiles, :id => repo.id)
+              plan_action(Katello::Repository::CorrectChecksum, repo)
+              concurrence do
+                plan_action(Pulp::Repository::Download, :pulp_id => repo.pulp_id, :options => {:verify_all_units => true}) if validate_contents && repo.yum?
+                plan_action(Katello::Repository::MetadataGenerate, repo, :force => true) if skip_metadata_check && repo.yum?
+                plan_action(Katello::Repository::ErrataMail, repo, nil, contents_changed)
+                if generate_applicability
+                  regenerate_applicability(repo, contents_changed)
+                end
               end
+              plan_self(:id => repo.id, :sync_result => output, :skip_metadata_check => skip_metadata_check, :validate_contents => validate_contents,
+                        :contents_changed => contents_changed)
+              plan_action(Katello::Repository::ImportApplicability, :repo_id => repo.id, :contents_changed => contents_changed) if generate_applicability
+              plan_action(Katello::Repository::SyncHook, :id => repo.id)
             end
-            plan_self(:id => repo.id, :sync_result => output, :skip_metadata_check => skip_metadata_check, :validate_contents => validate_contents,
-                      :contents_changed => contents_changed)
-            plan_action(Katello::Repository::ImportApplicability, :repo_id => repo.id, :contents_changed => contents_changed) if generate_applicability
-            plan_action(Katello::Repository::SyncHook, :id => repo.id)
           end
         end
 
@@ -91,6 +100,7 @@ module Actions
         def presenter
           found = all_planned_actions(Pulp::Repository::Sync)
           found = all_planned_actions(Pulp3::Repository::Sync) if found.empty?
+          found = all_planned_actions(Pulp3::Repository::Repair) if found.empty?
           Helpers::Presenter::Delegated.new(self, found)
         end
 

data/app/lib/actions/katello/repository/update.rb

@@ -10,8 +10,7 @@ module Actions
           action_subject root.library_instance
 
           repo_params[:url] = nil if repo_params[:url] == ''
-          update_cv_cert_protected = (repo_params[:unprotected] != repository.unprotected)
-
+          update_cv_cert_protected = repo_params.key?(:unprotected) && (repo_params[:unprotected] != repository.unprotected)
           root.update!(repo_params)
 
           if root.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_BACKGROUND
@@ -42,28 +41,24 @@ module Actions
                                        gpg_url: repository.yum_gpg_key_url)
           end
           if root.pulp_update_needed?
-            plan_pulp_action([::Actions::Pulp::Orchestration::Repository::Refresh,
-                              ::Actions::Pulp3::Orchestration::Repository::Update],
-                             repository,
-                             SmartProxy.pulp_master)
-            plan_self(:repository_id => root.library_instance.id, :update_cv_cert_protected => update_cv_cert_protected)
+            sequence do
+              plan_pulp_action([::Actions::Pulp::Orchestration::Repository::Refresh,
+                                ::Actions::Pulp3::Orchestration::Repository::Update],
+                               repository,
+                               SmartProxy.pulp_master)
+              plan_self(:repository_id => root.library_instance.id)
+              if update_cv_cert_protected
+                plan_optional_pulp_action([::Actions::Pulp3::Orchestration::Repository::TriggerUpdateRepoCertGuard], repository, ::SmartProxy.pulp_master)
+              end
+            end
           end
         end
 
         def run
           repository = ::Katello::Repository.find(input[:repository_id])
-          output[:repository_id] = input[:repository_id]
-          output[:update_cv_cert_protected] = input[:update_cv_cert_protected]
           ForemanTasks.async_task(Katello::Repository::MetadataGenerate, repository)
         end
 
-        def finalize
-          repository = ::Katello::Repository.find(output[:repository_id])
-          if output[:update_cv_cert_protected]
-            ForemanTasks.async_task(::Actions::Katello::Repository::UpdateCVRepoCertGuard, repository, SmartProxy.pulp_master)
-          end
-        end
-
         private
 
         def update_content?(repository)

data/app/lib/actions/katello/repository/verify_checksum.rb

@@ -0,0 +1,28 @@
+module Actions
+  module Katello
+    module Repository
+      class VerifyChecksum < Actions::EntryAction
+        include Helpers::Presenter
+        include Actions::Katello::PulpSelector
+
+        def plan(repo)
+          action_subject(repo)
+
+          if SmartProxy.pulp_master.pulp3_support?(repo)
+            plan_action(Actions::Pulp3::Repository::Repair, repo.id, SmartProxy.pulp_master)
+          else
+            options = {}
+            options[:validate_contents] = true
+            plan_action(Actions::Katello::Repository::Sync, repo, nil, options)
+          end
+        end
+
+        def presenter
+          found = all_planned_actions(Katello::Repository::Sync)
+          found = all_planned_actions(Pulp3::Repository::Repair) if found.empty?
+          Helpers::Presenter::Delegated.new(self, found)
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/katello/sync_plan/run.rb

@@ -19,7 +19,7 @@ module Actions
             syncable_products = sync_plan.products.syncable
             syncable_roots = ::Katello::RootRepository.where(:product_id => syncable_products).has_url
 
-            plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance)) unless syncable_roots.empty?
+            plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance).compact) unless syncable_roots.empty?
             plan_self(:sync_plan_name => sync_plan.name)
           end
         end

data/app/lib/actions/pulp/abstract_async_task.rb

@@ -74,6 +74,7 @@ module Actions
       end
 
       def cancel!
+        output[:cancelled] = true
         cancel
         self.external_task = poll_external_task
         # We suspend the action and the polling will take care of finding

data/app/lib/actions/pulp/consumer/sync_capsule.rb

@@ -26,6 +26,14 @@ module Actions
           self.done? ? 1 : 0.1
         end
 
+        def rescue_strategy_for_self
+          if output[:cancelled]
+            Dynflow::Action::Rescue::Fail
+          else
+            Dynflow::Action::Rescue::Skip
+          end
+        end
+
         def run_progress_weight
           100
         end

data/app/lib/actions/pulp/orchestration/repository/sync.rb

@@ -4,8 +4,9 @@ module Actions
       module Repository
         class Sync < Pulp::Abstract
           include Actions::Helpers::OutputPropagator
-          def plan(_repository, _smart_proxy, options)
+          def plan(repository, smart_proxy, options)
             sequence do
+              options.merge!(:repo_id => repository.id, :smart_proxy_id => smart_proxy.id)
               action_output = plan_action(Actions::Pulp::Repository::Sync, options).output
               plan_self(:subaction_output => action_output)
             end

data/app/lib/actions/pulp/repository/presenters/deb_presenter.rb

@@ -54,11 +54,11 @@ module Actions
           end
 
           def items_done
-            task_details&.inject(0) { |sum, details| sum + details[:num_success].to_i }
+            task_details&.inject(0) { |sum, details| sum + details[:num_success].to_i } || 0
           end
 
           def items_total
-            task_details&.inject(0) { |sum, details| sum + details[:items_total].to_i }
+            task_details&.inject(0) { |sum, details| sum + details[:items_total].to_i } || 0
           end
 
           def size_done

data/app/lib/actions/pulp/repository/sync.rb

@@ -14,8 +14,9 @@ module Actions
           overrides = {}
           overrides[:feed] = input[:source_url] if input[:source_url]
           overrides[:validate] = !(SETTINGS[:katello][:pulp][:skip_checksum_validation])
-          overrides[:options] = input[:options] if input[:options]
+          overrides.merge(input[:options]) if input[:options]
           repo = ::Katello::Repository.find(input[:repo_id])
+
           output[:pulp_tasks] = repo.backend_service(::SmartProxy.pulp_master).sync(overrides)
         end
 

data/app/lib/actions/pulp3/abstract_async_task.rb

@@ -4,41 +4,6 @@ module Actions
       include Actions::Base::Polling
       include ::Dynflow::Action::Cancellable
 
-      WAITING = ['waiting',
-                 SKIPPED = 'skipped'.freeze,
-                 RUNNING = 'running'.freeze,
-                 COMPLETED = 'completed'.freeze,
-                 FAILED = 'failed'.freeze,
-                 CANCELED = 'canceled'.freeze].freeze
-
-      FINISHED_STATES = [COMPLETED, FAILED, CANCELED, SKIPPED].freeze
-
-      # A call report Looks like:  {"task":"/pulp/api/v3/tasks/5/"}
-      # {
-      #    "pulp_href":"/pulp/api/v3/tasks/4/",
-      #    "pulp_created":"2019-02-21T19:50:40.476767Z",
-      #    "job_id":"d0359658-d926-47a2-b430-1b2092b3bd86",
-      #    "state":"completed",
-      #    "name":"pulp_file.app.tasks.publishing.publish",
-      #    "started_at":"2019-02-21T19:50:40.556002Z",
-      #    "finished_at":"2019-02-21T19:50:40.618397Z",
-      #    "non_fatal_errors":[
-      #
-      #    ],
-      #    "error":null,
-      #    "worker":"/pulp/api/v3/workers/1/",
-      #    "parent":null,
-      #    "spawned_tasks":[
-      #
-      #    ],
-      #    "progress_reports":[
-      #
-      #    ],
-      #    "created_resources":[
-      #       "/pulp/api/v3/publications/1/"
-      #    ]
-      # }
-
       def run(event = nil)
         # do nothing when the action is being skipped
         unless event == Dynflow::Action::Skip
@@ -49,16 +14,16 @@ module Actions
       def humanized_state
         case state
         when :running
-          if self.external_task.nil?
+          if self.combined_tasks.empty?
             _("initiating Pulp task")
           else
             _("checking Pulp task status")
           end
         when :suspended
-          if external_task&.all? { |task| task[:start_time].nil? }
-            _("waiting for Pulp to start the task")
-          else
+          if combined_tasks.any?(&:started?)
             _("waiting for Pulp to finish the task")
+          else
+            _("waiting for Pulp to start the task")
           end
         else
           super
@@ -66,28 +31,49 @@ module Actions
       end
 
       def done?
-        external_task&.all? { |task| task[:finish_time] || FINISHED_STATES.include?(task[:state]) }
+        combined_tasks&.all? { |task| task.done? }
       end
 
       def external_task
-        output[:pulp_tasks]
+        #this must return nil until external_task= is called
+        combined_tasks
+      end
+
+      def combined_tasks
+        return nil if pulp_tasks.nil? || task_groups.nil?
+        pulp_tasks + task_groups
+      end
+
+      def pulp_tasks
+        return nil if output[:pulp_tasks].nil?
+        output[:pulp_tasks] = new_or_existing_objects(::Katello::Pulp3::Task, output[:pulp_tasks])
+      end
+
+      def task_groups
+        return nil if output[:task_groups].nil?
+        output[:task_groups] = new_or_existing_objects(::Katello::Pulp3::TaskGroup, output[:task_groups])
+      end
+
+      def new_or_existing_objects(object_class, objects)
+        objects.map do |object|
+          if object.is_a?(object_class)
+            object
+          else
+            object_class.new(smart_proxy, object)
+          end
+        end
       end
 
       def cancel!
         cancel
-        self.external_task = poll_external_task
+        poll_external_task
         # We suspend the action and the polling will take care of finding
         # out if the cancelling was successful
         suspend unless done?
       end
 
       def cancel
-        output[:pulp_tasks].each do |pulp_task|
-          data = PulpcoreClient::Task.new(state: 'canceled')
-          tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
-          #the main task may have completed, so cancel spawned tasks too
-          pulp_task['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
-        end
+        pulp_tasks.each { |task| task.cancel }
       end
 
       def rescue_external_task(error)
@@ -109,24 +95,42 @@ module Actions
         response
       end
 
+      def check_for_errors
+        combined_tasks.each do |task|
+          if (message = task.error)
+            fail ::Katello::Errors::Pulp3Error, message
+          end
+        end
+      end
+
       def external_task=(external_task_data)
-        output[:pulp_tasks] = transform_task_response(external_task_data)
-        output[:pulp_tasks].each do |pulp_task|
-          if (pulp_exception = ::Katello::Errors::Pulp3Error.from_task(pulp_task))
-            fail pulp_exception
+        #currently we assume everything coming from invoke_external_task_methods are tasks
+        tasks = transform_task_response(external_task_data)
+        output[:pulp_tasks] = new_or_existing_objects(::Katello::Pulp3::Task, tasks)
+
+        add_task_groups
+        check_for_errors
+      end
+
+      def add_task_groups
+        output[:task_groups] ||= []
+        pulp_tasks.each do |task|
+          if task.task_group_href && !tracking_task_group?(task.task_group_href)
+            output[:task_groups] << ::Katello::Pulp3::TaskGroup.new_from_href(smart_proxy, task.task_group_href)
           end
         end
       end
 
-      def tasks_api
-        ::Katello::Pulp3::Api::Core.new(smart_proxy).tasks_api
+      def tracking_task_group?(href)
+        task_groups&.any? { |group| group.href == href }
       end
 
       def poll_external_task
-        external_task.map do |task|
-          task = tasks_api.read(task['pulp_href'] || task['task'])
-          task.as_json
-        end
+        pulp_tasks.each(&:poll)
+        output[:task_groups] = task_groups.each(&:poll) if task_groups
+        add_task_groups
+        check_for_errors
+        pulp_tasks
       end
     end
   end