katello 3.16.0.rc3.1 → 3.16.0

data/app/lib/actions/katello/sync_plan/run.rb

@@ -19,7 +19,7 @@ module Actions
             syncable_products = sync_plan.products.syncable
             syncable_roots = ::Katello::RootRepository.where(:product_id => syncable_products).has_url
 
-            plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance)) unless syncable_roots.empty?
+            plan_action(::Actions::BulkAction, ::Actions::Katello::Repository::Sync, syncable_roots.map(&:library_instance).compact) unless syncable_roots.empty?
             plan_self(:sync_plan_name => sync_plan.name)
           end
         end

data/app/lib/actions/pulp/abstract_async_task.rb

@@ -74,6 +74,7 @@ module Actions
       end
 
       def cancel!
+        output[:cancelled] = true
         cancel
         self.external_task = poll_external_task
         # We suspend the action and the polling will take care of finding

data/app/lib/actions/pulp/consumer/sync_capsule.rb

@@ -26,6 +26,14 @@ module Actions
           self.done? ? 1 : 0.1
         end
 
+        def rescue_strategy_for_self
+          if output[:cancelled]
+            Dynflow::Action::Rescue::Fail
+          else
+            Dynflow::Action::Rescue::Skip
+          end
+        end
+
         def run_progress_weight
           100
         end

data/app/lib/actions/pulp/repository/presenters/deb_presenter.rb

@@ -54,11 +54,11 @@ module Actions
           end
 
           def items_done
-            task_details&.inject(0) { |sum, details| sum + details[:num_success].to_i }
+            task_details&.inject(0) { |sum, details| sum + details[:num_success].to_i } || 0
           end
 
           def items_total
-            task_details&.inject(0) { |sum, details| sum + details[:items_total].to_i }
+            task_details&.inject(0) { |sum, details| sum + details[:items_total].to_i } || 0
           end
 
           def size_done

data/app/lib/actions/pulp3/abstract_async_task.rb

@@ -4,41 +4,6 @@ module Actions
       include Actions::Base::Polling
       include ::Dynflow::Action::Cancellable
 
-      WAITING = ['waiting',
-                 SKIPPED = 'skipped'.freeze,
-                 RUNNING = 'running'.freeze,
-                 COMPLETED = 'completed'.freeze,
-                 FAILED = 'failed'.freeze,
-                 CANCELED = 'canceled'.freeze].freeze
-
-      FINISHED_STATES = [COMPLETED, FAILED, CANCELED, SKIPPED].freeze
-
-      # A call report Looks like:  {"task":"/pulp/api/v3/tasks/5/"}
-      # {
-      #    "pulp_href":"/pulp/api/v3/tasks/4/",
-      #    "pulp_created":"2019-02-21T19:50:40.476767Z",
-      #    "job_id":"d0359658-d926-47a2-b430-1b2092b3bd86",
-      #    "state":"completed",
-      #    "name":"pulp_file.app.tasks.publishing.publish",
-      #    "started_at":"2019-02-21T19:50:40.556002Z",
-      #    "finished_at":"2019-02-21T19:50:40.618397Z",
-      #    "non_fatal_errors":[
-      #
-      #    ],
-      #    "error":null,
-      #    "worker":"/pulp/api/v3/workers/1/",
-      #    "parent":null,
-      #    "spawned_tasks":[
-      #
-      #    ],
-      #    "progress_reports":[
-      #
-      #    ],
-      #    "created_resources":[
-      #       "/pulp/api/v3/publications/1/"
-      #    ]
-      # }
-
       def run(event = nil)
         # do nothing when the action is being skipped
         unless event == Dynflow::Action::Skip
@@ -49,16 +14,16 @@ module Actions
       def humanized_state
         case state
         when :running
-          if self.external_task.nil?
+          if self.combined_tasks.empty?
             _("initiating Pulp task")
           else
             _("checking Pulp task status")
           end
         when :suspended
-          if external_task&.all? { |task| task[:start_time].nil? }
-            _("waiting for Pulp to start the task")
-          else
+          if combined_tasks.any?(&:started?)
             _("waiting for Pulp to finish the task")
+          else
+            _("waiting for Pulp to start the task")
           end
         else
           super
@@ -66,28 +31,49 @@ module Actions
       end
 
       def done?
-        external_task&.all? { |task| task[:finish_time] || FINISHED_STATES.include?(task[:state]) }
+        combined_tasks&.all? { |task| task.done? }
       end
 
       def external_task
-        output[:pulp_tasks]
+        #this must return nil until external_task= is called
+        combined_tasks
+      end
+
+      def combined_tasks
+        return nil if pulp_tasks.nil? || task_groups.nil?
+        pulp_tasks + task_groups
+      end
+
+      def pulp_tasks
+        return nil if output[:pulp_tasks].nil?
+        output[:pulp_tasks] = new_or_existing_objects(::Katello::Pulp3::Task, output[:pulp_tasks])
+      end
+
+      def task_groups
+        return nil if output[:task_groups].nil?
+        output[:task_groups] = new_or_existing_objects(::Katello::Pulp3::TaskGroup, output[:task_groups])
+      end
+
+      def new_or_existing_objects(object_class, objects)
+        objects.map do |object|
+          if object.is_a?(object_class)
+            object
+          else
+            object_class.new(smart_proxy, object)
+          end
+        end
       end
 
       def cancel!
         cancel
-        self.external_task = poll_external_task
+        poll_external_task
         # We suspend the action and the polling will take care of finding
         # out if the cancelling was successful
         suspend unless done?
       end
 
       def cancel
-        output[:pulp_tasks].each do |pulp_task|
-          data = PulpcoreClient::Task.new(state: 'canceled')
-          tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
-          #the main task may have completed, so cancel spawned tasks too
-          pulp_task['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
-        end
+        pulp_tasks.each { |task| task.cancel }
       end
 
       def rescue_external_task(error)
@@ -109,24 +95,42 @@ module Actions
         response
       end
 
+      def check_for_errors
+        combined_tasks.each do |task|
+          if (message = task.error)
+            fail ::Katello::Errors::Pulp3Error, message
+          end
+        end
+      end
+
       def external_task=(external_task_data)
-        output[:pulp_tasks] = transform_task_response(external_task_data)
-        output[:pulp_tasks].each do |pulp_task|
-          if (pulp_exception = ::Katello::Errors::Pulp3Error.from_task(pulp_task))
-            fail pulp_exception
+        #currently we assume everything coming from invoke_external_task_methods are tasks
+        tasks = transform_task_response(external_task_data)
+        output[:pulp_tasks] = new_or_existing_objects(::Katello::Pulp3::Task, tasks)
+
+        add_task_groups
+        check_for_errors
+      end
+
+      def add_task_groups
+        output[:task_groups] ||= []
+        pulp_tasks.each do |task|
+          if task.task_group_href && !tracking_task_group?(task.task_group_href)
+            output[:task_groups] << ::Katello::Pulp3::TaskGroup.new_from_href(smart_proxy, task.task_group_href)
           end
         end
       end
 
-      def tasks_api
-        ::Katello::Pulp3::Api::Core.new(smart_proxy).tasks_api
+      def tracking_task_group?(href)
+        task_groups&.any? { |group| group.href == href }
       end
 
       def poll_external_task
-        external_task.map do |task|
-          task = tasks_api.read(task['pulp_href'] || task['task'])
-          task.as_json
-        end
+        pulp_tasks.each(&:poll)
+        output[:task_groups] = task_groups.each(&:poll) if task_groups
+        add_task_groups
+        check_for_errors
+        pulp_tasks
       end
     end
   end

data/app/lib/actions/pulp3/content_migration.rb

@@ -12,6 +12,10 @@ module Actions
         migration_service = ::Katello::Pulp3::Migration.new(smart_proxy)
         migration_service.create_and_run_migrations
       end
+
+      def rescue_strategy
+        Dynflow::Action::Rescue::Skip
+      end
     end
   end
 end

data/app/lib/actions/pulp3/orchestration/repository/copy_all_units.rb

@@ -21,8 +21,7 @@ module Actions
                 else
                   #if we are not filtering, copy the version to the cv repository, and the units for each additional repo
                   action = plan_action(Actions::Pulp3::Repository::CopyVersion, source_repositories.first, smart_proxy, target_repo)
-                  plan_action(Actions::Pulp3::Repository::SaveVersion, target_repo,
-                              repository_details: { latest_version_href: action.output[:latest_version_output] }, tasks: action.output[:pulp_tasks])
+                  plan_action(Actions::Pulp3::Repository::SaveVersion, target_repo, tasks: action.output[:pulp_tasks])
                   copy_actions = []
                   #since we're creating a new version from the first repo, start copying at the 2nd
                   source_repositories[1..-1].each do |source_repo|

data/app/lib/actions/pulp3/orchestration/repository/multi_copy_all_units.rb

@@ -0,0 +1,36 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module Repository
+        class MultiCopyAllUnits < Pulp3::Abstract
+          def plan(extended_repo_map, smart_proxy, options = {})
+            solve_dependencies = options.fetch(:solve_dependencies, false)
+            if extended_repo_map.values.pluck(:filters).flatten.present? ||
+                extended_repo_map.keys.detect { |source_repos| source_repos.length > 1 }
+              sequence do
+                copy_action = plan_action(Actions::Pulp3::Repository::MultiCopyContent, extended_repo_map, smart_proxy,
+                                          solve_dependencies: solve_dependencies)
+                plan_action(Actions::Pulp3::Repository::SaveVersions, extended_repo_map.values.pluck(:dest_repo),
+                            tasks: copy_action.output[:pulp_tasks])
+              end
+            else
+              repo_id_map = {}
+              extended_repo_map.each do |source_repos, dest_repo_map|
+                repo_id_map[source_repos.first.id] = dest_repo_map[:dest_repo].id
+              end
+              plan_self(repo_id_map: repo_id_map)
+            end
+          end
+
+          def run
+            input[:repo_id_map].each do |source_repo_id, dest_repo_id|
+              dest_repo = ::Katello::Repository.find(dest_repo_id)
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              dest_repo.update!(version_href: source_repo.version_href)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/multi_copy_content.rb

@@ -0,0 +1,28 @@
+module Actions
+  module Pulp3
+    module Repository
+      class MultiCopyContent < Pulp3::AbstractAsyncTask
+        def plan(extended_repo_map, smart_proxy, options)
+          repo_id_map = {}
+
+          extended_repo_map.each do |source_repos, dest_repo_map|
+            repo_id_map[source_repos&.map(&:id)] = { :dest_repo => dest_repo_map[:dest_repo].id,
+                                                     :filter_ids => dest_repo_map[:filters]&.map(&:id) }
+          end
+
+          plan_self(options.merge(:repo_id_map => repo_id_map, :smart_proxy_id => smart_proxy.id))
+        end
+
+        def invoke_external_task
+          repo_id_map = {}
+
+          input[:repo_id_map].each do |source_repo_ids, dest_repo_map|
+            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map
+          end
+
+          output[:pulp_tasks] = ::Katello::Repository.find(repo_id_map.values.first[:dest_repo]).backend_service(smart_proxy).copy_content_from_mapping(repo_id_map, input)
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/multi_copy_units.rb

@@ -0,0 +1,55 @@
+module Actions
+  module Pulp3
+    module Repository
+      class MultiCopyUnits < Pulp3::AbstractAsyncTask
+        # repo_map example: {
+        #   [<source_repo_ids>]: {
+        #     dest_repo: <dest_repo_id>,
+        #     base_version: <base_version>
+        #   }
+        # }
+        def plan(repo_map, unit_map, options = {})
+          if unit_map.values.flatten.any?
+            action_output = plan_self(repo_map: repo_map,
+                      unit_map: unit_map,
+                      dependency_solving: options[:dependency_solving],
+                      incremental_update: options[:incremental_update],
+                      smart_proxy_id: SmartProxy.pulp_master.id).output
+            plan_action(Pulp3::Repository::SaveVersions, repo_map.values.pluck(:dest_repo),
+                        tasks: action_output[:pulp_tasks]).output
+          end
+        end
+
+        def invoke_external_task
+          unit_hrefs = []
+          repo_map = {}
+
+          input[:repo_map].each do |source_repo_ids, dest_repo_map|
+            repo_map[JSON.parse(source_repo_ids)] = dest_repo_map
+          end
+
+          if input[:unit_map][:errata].any?
+            unit_hrefs << ::Katello::RepositoryErratum.
+              joins("inner join katello_errata on katello_repository_errata.erratum_id = katello_errata.id").
+              where("katello_repository_errata.repository_id in (#{repo_map.keys.join(',')}) and
+                    katello_errata.id in (#{input[:unit_map][:errata].join(",")})").map(&:erratum_pulp3_href)
+          end
+
+          if input[:unit_map][:rpms].any?
+            unit_hrefs << ::Katello::Rpm.where(:id => input[:unit_map][:rpms]).map(&:pulp_id)
+          end
+          unit_hrefs.flatten!
+
+          repo_map.each do |_source_repos, dest_repo_map|
+            dest_repo_map[:content_unit_hrefs] = unit_hrefs
+          end
+
+          target_repo = ::Katello::Repository.find(repo_map.values.first[:dest_repo])
+          unless unit_hrefs.flatten.empty?
+            output[:pulp_tasks] = target_repo.backend_service(SmartProxy.pulp_master).multi_copy_units(repo_map, input[:dependency_solving])
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/presenters/content_unit_presenter.rb

@@ -12,7 +12,7 @@ module Actions
           def humanized_details
             ret = []
             ret << _("Cancelled.") if cancelled?
-            ret << _("Total tasks: ") + ": #{finished_units}/#{total_units}"
+            ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
             ret << "--------------------------------"
             progress_reports = sync_task.try(:[], 'progress_reports') || []
             progress_reports = progress_reports.sort_by { |pr| pr.try(:[], 'message') }

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -9,12 +9,20 @@ module Actions
         def run
           repo = ::Katello::Repository.find(input[:repository_id])
 
-          if input[:tasks]
+          if input[:tasks].present?
             version_href = input[:tasks].last[:created_resources].first
           end
 
-          if !version_href && input[:repository_details]
-            version_href = input[:repository_details][:latest_version_href]
+          unless version_href
+            if input[:repository_details]
+              version_href = input[:repository_details][:latest_version_href]
+            elsif repo.version_href.nil?
+              # Fetch latest Pulp 3 repo version
+              repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+              version_href ||= repo_backend_service.api.
+                repositories_api.read(repo_backend_service.
+                repository_reference.repository_href).latest_version_href
+            end
           end
 
           if version_href

data/app/lib/actions/pulp3/repository/save_versions.rb

@@ -0,0 +1,73 @@
+module Actions
+  module Pulp3
+    module Repository
+      class SaveVersions < Pulp3::Abstract
+        def plan(repository_ids, options)
+          plan_self(:repository_ids => repository_ids, :tasks => options[:tasks])
+        end
+
+        def run
+          return if input[:tasks].empty?
+          version_hrefs = input[:tasks].last[:created_resources]
+          repositories = find_repositories(input[:repository_ids])
+
+          output.merge!(contents_changed: false, updated_repositories: [])
+          repositories.each do |repo|
+            repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+            if repo.version_href
+              # Chop off the version number to compare base repo strings
+              unversioned_href = repo.version_href[0..-2].rpartition('/').first
+              # Could have multiple version_hrefs for the same repo depending on the copy task
+              new_version_hrefs = version_hrefs.collect do |version_href|
+                version_href if unversioned_href == version_href[0..-2].rpartition('/').first
+              end
+
+              new_version_hrefs.compact!
+              if new_version_hrefs.size > 1
+                # Find latest version_href by its version number
+                new_version_href = version_map(new_version_hrefs).max_by { |_href, version| version }.first
+              else
+                new_version_href = new_version_hrefs.first
+              end
+
+              # Successive incremental updates won't generate a new repo version, so fetch the latest Pulp 3 repo version
+              new_version_href ||= latest_version_href(repo_backend_service)
+            else
+              new_version_href = latest_version_href(repo_backend_service)
+            end
+
+            unless new_version_href == repo.version_href
+              repo.update(version_href: new_version_href)
+              repo.index_content
+              output[:contents_changed] = true
+              output[:updated_repositories] << repo.id
+            end
+          end
+        end
+
+        def version_map(version_hrefs)
+          version_map = {}
+          version_hrefs.each do |href|
+            version_map[href] = href.split("/")[-1].to_i
+          end
+          version_map
+        end
+
+        def latest_version_href(repo_backend_service)
+          repo_backend_service.api.repositories_api.
+            read(repo_backend_service.repository_reference.repository_href).latest_version_href
+        end
+
+        def find_repositories(repository_ids)
+          repository_ids.collect do |repo_id|
+            if repo_id.is_a?(Hash)
+              ::Katello::Repository.find(repo_id.with_indifferent_access[:id])
+            else
+              ::Katello::Repository.find(repo_id)
+            end
+          end
+        end
+      end
+    end
+  end
+end