katello 3.16.0.rc5.1 → 3.16.0

data/app/lib/actions/katello/repository/multi_clone_contents.rb

@@ -0,0 +1,66 @@
+module Actions
+  module Katello
+    module Repository
+      class MultiCloneContents < Actions::Base
+        include Actions::Katello::PulpSelector
+        def plan(extended_repo_mapping, options)
+          generate_metadata = options.fetch(:generate_metadata, true)
+          copy_contents = options.fetch(:copy_contents, true)
+          solve_dependencies = options.fetch(:solve_dependencies, false)
+
+          sequence do
+            if copy_contents
+              plan_action(Pulp3::Orchestration::Repository::MultiCopyAllUnits,
+                          extended_repo_mapping,
+                          SmartProxy.pulp_master,
+                          solve_dependencies: solve_dependencies)
+            end
+
+            extended_repo_mapping.each do |source_repos, dest_repo_map|
+              if generate_metadata
+                metadata_generate(source_repos, dest_repo_map[:dest_repo], dest_repo_map[:filters])
+              end
+            end
+
+            extended_repo_mapping.values.each do |dest_repo_map|
+              plan_action(Katello::Repository::IndexContent, id: dest_repo_map[:dest_repo].id)
+            end
+          end
+        end
+
+        def metadata_generate(source_repositories, new_repository, filters)
+          metadata_options = {}
+
+          if source_repositories.count == 1 && filters.empty?
+            metadata_options[:source_repository] = source_repositories.first
+          end
+
+          check_matching_content = ::Katello::RepositoryTypeManager.find(new_repository.content_type).metadata_publish_matching_check
+          if new_repository.environment && source_repositories.count == 1 && check_matching_content
+            match_check_output = plan_action(Katello::Repository::CheckMatchingContent,
+                                :source_repo_id => source_repositories.first.id,
+                                :target_repo_id => new_repository.id).output
+
+            metadata_options[:matching_content] = match_check_output[:matching_content]
+          end
+
+          plan_action(Katello::Repository::MetadataGenerate, new_repository, metadata_options)
+          unless source_repositories.first.saved_checksum_type == new_repository.saved_checksum_type
+            checksum_mapping = {}
+            repository_mapping.each do |source_repos, dest_repo|
+              checksum_mapping[dest_repo.id] = source_repos.first.saved_checksum_type
+            end
+            plan_self(:checksum_mapping => checksum_mapping)
+          end
+        end
+
+        def finalize
+          input[:checksum_mapping].each do |repo_id, checksum_type|
+            repository = ::Katello::Repository.find(repo_id)
+            repository.update!(saved_checksum_type: checksum_type) if (repository && checksum_type)
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/katello/repository/multi_clone_to_version.rb

@@ -0,0 +1,30 @@
+module Actions
+  module Katello
+    module Repository
+      class MultiCloneToVersion < Actions::Base
+        def plan(repository_mapping, content_view_version, options = {})
+          incremental = options.fetch(:incremental, false)
+          content_view = content_view_version.content_view
+          extended_repo_map = extended_repo_mapping(repository_mapping, content_view, incremental)
+          sequence do
+            plan_action(::Actions::Katello::Repository::MultiCloneContents, extended_repo_map,
+                        copy_contents: true,
+                        solve_dependencies: content_view.solve_dependencies,
+                        metadata_generate: !incremental)
+          end
+        end
+
+        def extended_repo_mapping(repo_map, content_view, incremental)
+          # Example: {[source_repos] => {dest_repo: dest_repo, filters: filters}}
+          extended_repo_map = {}
+          repo_map.each do |source_repos, dest_repo|
+            filters = incremental ? [] : content_view.filters.applicable(source_repos.first)
+            extended_repo_map[source_repos] = { :dest_repo => dest_repo,
+                                                :filters => filters }
+          end
+          extended_repo_map
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/abstract_async_task.rb

@@ -4,41 +4,6 @@ module Actions
       include Actions::Base::Polling
       include ::Dynflow::Action::Cancellable
 
-      WAITING = ['waiting',
-                 SKIPPED = 'skipped'.freeze,
-                 RUNNING = 'running'.freeze,
-                 COMPLETED = 'completed'.freeze,
-                 FAILED = 'failed'.freeze,
-                 CANCELED = 'canceled'.freeze].freeze
-
-      FINISHED_STATES = [COMPLETED, FAILED, CANCELED, SKIPPED].freeze
-
-      # A call report Looks like:  {"task":"/pulp/api/v3/tasks/5/"}
-      # {
-      #    "pulp_href":"/pulp/api/v3/tasks/4/",
-      #    "pulp_created":"2019-02-21T19:50:40.476767Z",
-      #    "job_id":"d0359658-d926-47a2-b430-1b2092b3bd86",
-      #    "state":"completed",
-      #    "name":"pulp_file.app.tasks.publishing.publish",
-      #    "started_at":"2019-02-21T19:50:40.556002Z",
-      #    "finished_at":"2019-02-21T19:50:40.618397Z",
-      #    "non_fatal_errors":[
-      #
-      #    ],
-      #    "error":null,
-      #    "worker":"/pulp/api/v3/workers/1/",
-      #    "parent":null,
-      #    "spawned_tasks":[
-      #
-      #    ],
-      #    "progress_reports":[
-      #
-      #    ],
-      #    "created_resources":[
-      #       "/pulp/api/v3/publications/1/"
-      #    ]
-      # }
-
       def run(event = nil)
         # do nothing when the action is being skipped
         unless event == Dynflow::Action::Skip
@@ -49,16 +14,16 @@ module Actions
       def humanized_state
         case state
         when :running
-          if self.external_task.nil?
+          if self.combined_tasks.empty?
             _("initiating Pulp task")
           else
             _("checking Pulp task status")
           end
         when :suspended
-          if external_task&.all? { |task| task[:start_time].nil? }
-            _("waiting for Pulp to start the task")
-          else
+          if combined_tasks.any?(&:started?)
             _("waiting for Pulp to finish the task")
+          else
+            _("waiting for Pulp to start the task")
           end
         else
           super
@@ -66,28 +31,49 @@ module Actions
       end
 
       def done?
-        external_task&.all? { |task| task[:finish_time] || FINISHED_STATES.include?(task[:state]) }
+        combined_tasks&.all? { |task| task.done? }
       end
 
       def external_task
-        output[:pulp_tasks]
+        #this must return nil until external_task= is called
+        combined_tasks
+      end
+
+      def combined_tasks
+        return nil if pulp_tasks.nil? || task_groups.nil?
+        pulp_tasks + task_groups
+      end
+
+      def pulp_tasks
+        return nil if output[:pulp_tasks].nil?
+        output[:pulp_tasks] = new_or_existing_objects(::Katello::Pulp3::Task, output[:pulp_tasks])
+      end
+
+      def task_groups
+        return nil if output[:task_groups].nil?
+        output[:task_groups] = new_or_existing_objects(::Katello::Pulp3::TaskGroup, output[:task_groups])
+      end
+
+      def new_or_existing_objects(object_class, objects)
+        objects.map do |object|
+          if object.is_a?(object_class)
+            object
+          else
+            object_class.new(smart_proxy, object)
+          end
+        end
       end
 
       def cancel!
         cancel
-        self.external_task = poll_external_task
+        poll_external_task
         # We suspend the action and the polling will take care of finding
         # out if the cancelling was successful
         suspend unless done?
       end
 
       def cancel
-        output[:pulp_tasks].each do |pulp_task|
-          data = PulpcoreClient::Task.new(state: 'canceled')
-          tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
-          #the main task may have completed, so cancel spawned tasks too
-          pulp_task['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
-        end
+        pulp_tasks.each { |task| task.cancel }
       end
 
       def rescue_external_task(error)
@@ -109,24 +95,42 @@ module Actions
         response
       end
 
+      def check_for_errors
+        combined_tasks.each do |task|
+          if (message = task.error)
+            fail ::Katello::Errors::Pulp3Error, message
+          end
+        end
+      end
+
       def external_task=(external_task_data)
-        output[:pulp_tasks] = transform_task_response(external_task_data)
-        output[:pulp_tasks].each do |pulp_task|
-          if (pulp_exception = ::Katello::Errors::Pulp3Error.from_task(pulp_task))
-            fail pulp_exception
+        #currently we assume everything coming from invoke_external_task_methods are tasks
+        tasks = transform_task_response(external_task_data)
+        output[:pulp_tasks] = new_or_existing_objects(::Katello::Pulp3::Task, tasks)
+
+        add_task_groups
+        check_for_errors
+      end
+
+      def add_task_groups
+        output[:task_groups] ||= []
+        pulp_tasks.each do |task|
+          if task.task_group_href && !tracking_task_group?(task.task_group_href)
+            output[:task_groups] << ::Katello::Pulp3::TaskGroup.new_from_href(smart_proxy, task.task_group_href)
           end
         end
       end
 
-      def tasks_api
-        ::Katello::Pulp3::Api::Core.new(smart_proxy).tasks_api
+      def tracking_task_group?(href)
+        task_groups&.any? { |group| group.href == href }
       end
 
       def poll_external_task
-        external_task.map do |task|
-          task = tasks_api.read(task['pulp_href'] || task['task'])
-          task.as_json
-        end
+        pulp_tasks.each(&:poll)
+        output[:task_groups] = task_groups.each(&:poll) if task_groups
+        add_task_groups
+        check_for_errors
+        pulp_tasks
       end
     end
   end

data/app/lib/actions/pulp3/content_migration.rb

@@ -12,6 +12,10 @@ module Actions
         migration_service = ::Katello::Pulp3::Migration.new(smart_proxy)
         migration_service.create_and_run_migrations
       end
+
+      def rescue_strategy
+        Dynflow::Action::Rescue::Skip
+      end
     end
   end
 end

data/app/lib/actions/pulp3/orchestration/repository/copy_all_units.rb

@@ -21,8 +21,7 @@ module Actions
                 else
                   #if we are not filtering, copy the version to the cv repository, and the units for each additional repo
                   action = plan_action(Actions::Pulp3::Repository::CopyVersion, source_repositories.first, smart_proxy, target_repo)
-                  plan_action(Actions::Pulp3::Repository::SaveVersion, target_repo,
-                              repository_details: { latest_version_href: action.output[:latest_version_output] }, tasks: action.output[:pulp_tasks])
+                  plan_action(Actions::Pulp3::Repository::SaveVersion, target_repo, tasks: action.output[:pulp_tasks])
                   copy_actions = []
                   #since we're creating a new version from the first repo, start copying at the 2nd
                   source_repositories[1..-1].each do |source_repo|

data/app/lib/actions/pulp3/orchestration/repository/multi_copy_all_units.rb

@@ -0,0 +1,36 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module Repository
+        class MultiCopyAllUnits < Pulp3::Abstract
+          def plan(extended_repo_map, smart_proxy, options = {})
+            solve_dependencies = options.fetch(:solve_dependencies, false)
+            if extended_repo_map.values.pluck(:filters).flatten.present? ||
+                extended_repo_map.keys.detect { |source_repos| source_repos.length > 1 }
+              sequence do
+                copy_action = plan_action(Actions::Pulp3::Repository::MultiCopyContent, extended_repo_map, smart_proxy,
+                                          solve_dependencies: solve_dependencies)
+                plan_action(Actions::Pulp3::Repository::SaveVersions, extended_repo_map.values.pluck(:dest_repo),
+                            tasks: copy_action.output[:pulp_tasks])
+              end
+            else
+              repo_id_map = {}
+              extended_repo_map.each do |source_repos, dest_repo_map|
+                repo_id_map[source_repos.first.id] = dest_repo_map[:dest_repo].id
+              end
+              plan_self(repo_id_map: repo_id_map)
+            end
+          end
+
+          def run
+            input[:repo_id_map].each do |source_repo_id, dest_repo_id|
+              dest_repo = ::Katello::Repository.find(dest_repo_id)
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              dest_repo.update!(version_href: source_repo.version_href)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/multi_copy_content.rb

@@ -0,0 +1,28 @@
+module Actions
+  module Pulp3
+    module Repository
+      class MultiCopyContent < Pulp3::AbstractAsyncTask
+        def plan(extended_repo_map, smart_proxy, options)
+          repo_id_map = {}
+
+          extended_repo_map.each do |source_repos, dest_repo_map|
+            repo_id_map[source_repos&.map(&:id)] = { :dest_repo => dest_repo_map[:dest_repo].id,
+                                                     :filter_ids => dest_repo_map[:filters]&.map(&:id) }
+          end
+
+          plan_self(options.merge(:repo_id_map => repo_id_map, :smart_proxy_id => smart_proxy.id))
+        end
+
+        def invoke_external_task
+          repo_id_map = {}
+
+          input[:repo_id_map].each do |source_repo_ids, dest_repo_map|
+            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map
+          end
+
+          output[:pulp_tasks] = ::Katello::Repository.find(repo_id_map.values.first[:dest_repo]).backend_service(smart_proxy).copy_content_from_mapping(repo_id_map, input)
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/multi_copy_units.rb

@@ -3,7 +3,7 @@ module Actions
     module Repository
       class MultiCopyUnits < Pulp3::AbstractAsyncTask
         # repo_map example: {
-        #   <source_repo_id>: {
+        #   [<source_repo_ids>]: {
         #     dest_repo: <dest_repo_id>,
         #     base_version: <base_version>
         #   }
@@ -22,7 +22,11 @@ module Actions
 
         def invoke_external_task
           unit_hrefs = []
-          repo_map = input[:repo_map]
+          repo_map = {}
+
+          input[:repo_map].each do |source_repo_ids, dest_repo_map|
+            repo_map[JSON.parse(source_repo_ids)] = dest_repo_map
+          end
 
           if input[:unit_map][:errata].any?
             unit_hrefs << ::Katello::RepositoryErratum.
@@ -34,13 +38,16 @@ module Actions
           if input[:unit_map][:rpms].any?
             unit_hrefs << ::Katello::Rpm.where(:id => input[:unit_map][:rpms]).map(&:pulp_id)
           end
+          unit_hrefs.flatten!
+
+          repo_map.each do |_source_repos, dest_repo_map|
+            dest_repo_map[:content_unit_hrefs] = unit_hrefs
+          end
 
-          # TODO: Fix this workaround by refactoring copy_units after general content view dep solving is refactored
-          source_repo = ::Katello::Repository.find(repo_map.keys.first)
           target_repo = ::Katello::Repository.find(repo_map.values.first[:dest_repo])
-          dest_base_version = repo_map.values.first[:base_version]
-          repo_map.delete(repo_map.keys.first)
-          output[:pulp_tasks] = target_repo.backend_service(SmartProxy.pulp_master).copy_units(source_repo, unit_hrefs.flatten, input[:dependency_solving], dest_base_version, repo_map)
+          unless unit_hrefs.flatten.empty?
+            output[:pulp_tasks] = target_repo.backend_service(SmartProxy.pulp_master).multi_copy_units(repo_map, input[:dependency_solving])
+          end
         end
       end
     end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -9,12 +9,20 @@ module Actions
         def run
           repo = ::Katello::Repository.find(input[:repository_id])
 
-          if input[:tasks]
+          if input[:tasks].present?
             version_href = input[:tasks].last[:created_resources].first
           end
 
-          if !version_href && input[:repository_details]
-            version_href = input[:repository_details][:latest_version_href]
+          unless version_href
+            if input[:repository_details]
+              version_href = input[:repository_details][:latest_version_href]
+            elsif repo.version_href.nil?
+              # Fetch latest Pulp 3 repo version
+              repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+              version_href ||= repo_backend_service.api.
+                repositories_api.read(repo_backend_service.
+                repository_reference.repository_href).latest_version_href
+            end
           end
 
           if version_href

data/app/lib/actions/pulp3/repository/save_versions.rb

@@ -7,23 +7,34 @@ module Actions
         end
 
         def run
+          return if input[:tasks].empty?
           version_hrefs = input[:tasks].last[:created_resources]
-          repositories = input[:repository_ids].collect do |repo_id|
-            ::Katello::Repository.find(repo_id)
-          end
+          repositories = find_repositories(input[:repository_ids])
 
-          output[:contents_changed] = false
-          output[:updated_repositories] = []
+          output.merge!(contents_changed: false, updated_repositories: [])
           repositories.each do |repo|
-            # Chop off the version number to compare base repo strings
-            unversioned_href = repo.version_href[0..-2].rpartition('/').first
-            new_version_href = version_hrefs.detect do |version_href|
-              unversioned_href == version_href[0..-2].rpartition('/').first
+            repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+            if repo.version_href
+              # Chop off the version number to compare base repo strings
+              unversioned_href = repo.version_href[0..-2].rpartition('/').first
+              # Could have multiple version_hrefs for the same repo depending on the copy task
+              new_version_hrefs = version_hrefs.collect do |version_href|
+                version_href if unversioned_href == version_href[0..-2].rpartition('/').first
+              end
+
+              new_version_hrefs.compact!
+              if new_version_hrefs.size > 1
+                # Find latest version_href by its version number
+                new_version_href = version_map(new_version_hrefs).max_by { |_href, version| version }.first
+              else
+                new_version_href = new_version_hrefs.first
+              end
+
+              # Successive incremental updates won't generate a new repo version, so fetch the latest Pulp 3 repo version
+              new_version_href ||= latest_version_href(repo_backend_service)
+            else
+              new_version_href = latest_version_href(repo_backend_service)
             end
-            # Successive incremental updates won't generate a new repo version, so fetch the latest Pulp 3 repo version
-            new_version_href ||= ::Katello::Pulp3::Api::Yum.new(SmartProxy.pulp_master!).
-              repositories_api.read(repo.backend_service(SmartProxy.pulp_master).
-              repository_reference.repository_href).latest_version_href
 
             unless new_version_href == repo.version_href
               repo.update(version_href: new_version_href)
@@ -33,6 +44,29 @@ module Actions
             end
           end
         end
+
+        def version_map(version_hrefs)
+          version_map = {}
+          version_hrefs.each do |href|
+            version_map[href] = href.split("/")[-1].to_i
+          end
+          version_map
+        end
+
+        def latest_version_href(repo_backend_service)
+          repo_backend_service.api.repositories_api.
+            read(repo_backend_service.repository_reference.repository_href).latest_version_href
+        end
+
+        def find_repositories(repository_ids)
+          repository_ids.collect do |repo_id|
+            if repo_id.is_a?(Hash)
+              ::Katello::Repository.find(repo_id.with_indifferent_access[:id])
+            else
+              ::Katello::Repository.find(repo_id)
+            end
+          end
+        end
       end
     end
   end