katello 3.13.4 → 3.14.0.rc1

data/app/lib/actions/katello/repository/import_upload.rb

@@ -3,6 +3,7 @@ module Actions
   module Katello
     module Repository
       class ImportUpload < Actions::EntryAction
+        include Actions::Katello::PulpSelector
         def plan(repository, uploads, options = {})
           action_subject(repository)
           repo_service = repository.backend_service(::SmartProxy.pulp_master)
@@ -11,24 +12,24 @@ module Actions
           unit_keys = repo_service.unit_keys(uploads)
           generate_metadata = options.fetch(:generate_metadata, true)
           sync_capsule = options.fetch(:sync_capsule, true)
-          repo_type = repository.content_type
 
-          options[:content_type] ||= ::Katello::RepositoryTypeManager.find(repo_type).default_managed_content_type.label
+          options[:content_type] ||= ::Katello::RepositoryTypeManager.find(repository.content_type).default_managed_content_type.label
           unit_type_id = SmartProxy.pulp_master.content_service(options[:content_type])::CONTENT_TYPE
 
           sequence do
             upload_results = concurrence do
               upload_ids.zip(unit_keys).collect do |upload_id, unit_key|
-                if unit_type_id == 'docker_tag'
-                  unit_metadata = unit_key
-                end
-                import_upload = plan_action(Pulp::Repository::ImportUpload,
-                                            pulp_id: repository.pulp_id,
-                                            unit_type_id: unit_type_id,
-                                            unit_key: unit_key,
-                                            upload_id: upload_id,
-                                            unit_metadata: unit_metadata)
-
+                unit_metadata = unit_key if unit_type_id == 'docker_tag'
+                import_upload_args = {
+                  pulp_id: repository.pulp_id,
+                  unit_type_id: unit_type_id,
+                  unit_key: unit_key,
+                  upload_id: upload_id,
+                  unit_metadata: unit_metadata
+                }
+                import_upload = plan_pulp_action([Actions::Pulp::Repository::ImportUpload,
+                                                  Actions::Pulp3::Orchestration::Repository::ImportUpload],
+                                  repository, SmartProxy.pulp_master, import_upload_args)
                 plan_action(FinishUpload, repository, :dependency => import_upload.output,
                             generate_metadata: false, content_type: options[:content_type])
                 import_upload.output

data/app/lib/actions/katello/repository/metadata_generate.rb

@@ -10,7 +10,7 @@ module Actions
           repository_creation = options.fetch(:repository_creation, false)
           source_repository = options.fetch(:source_repository, nil)
           source_repository ||= repository.target_repository if repository.link?
-          smart_proxy_id = options.fetch(:capsule_id, SmartProxy.pulp_master.id)
+          smart_proxy = options.fetch(:smart_proxy, SmartProxy.pulp_master)
           if repository_creation
             matching_content = false
           else
@@ -18,7 +18,7 @@ module Actions
           end
 
           plan_pulp_action([Pulp::Repository::DistributorPublish, Pulp3::Orchestration::Repository::GenerateMetadata],
-                        repository, SmartProxy.find(smart_proxy_id),
+                        repository, smart_proxy,
                         :force => force,
                         :source_repository => source_repository,
                         :matching_content => matching_content,

data/app/lib/actions/katello/repository/update.rb

@@ -4,6 +4,7 @@ module Actions
       class Update < Actions::EntryAction
         include Actions::Katello::PulpSelector
 
+        # rubocop:disable Metrics/MethodLength
         def plan(root, repo_params)
           repository = root.library_instance
           action_subject root.library_instance
@@ -11,6 +12,14 @@ module Actions
           repo_params[:url] = nil if repo_params[:url] == ''
           root.update_attributes!(repo_params)
 
+          if root.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_BACKGROUND
+            ::Foreman::Deprecation.api_deprecation_warning("Background download_policy will be removed in Katello 3.16.  Any background repositories will be converted to Immediate")
+          end
+
+          if root['content_type'] == 'puppet' || root['content_type'] == 'ostree'
+            ::Foreman::Deprecation.api_deprecation_warning("Repository types of 'Puppet' and 'OSTree' will no longer be supported in Katello 3.16.")
+          end
+
           if update_content?(repository)
             content = root.content
 

data/app/lib/actions/katello/repository/upload_files.rb

@@ -6,6 +6,7 @@ module Actions
   module Katello
     module Repository
       class UploadFiles < Actions::EntryAction
+        include Actions::Katello::PulpSelector
         def plan(repository, files, content_type = nil)
           action_subject(repository)
           tmp_files = prepare_tmp_files(files)
@@ -17,17 +18,9 @@ module Actions
             concurrence do
               tmp_files.each do |file|
                 sequence do
-                  upload_request = plan_action(Pulp::Repository::CreateUploadRequest)
-                  plan_action(Pulp::Repository::UploadFile,
-                              upload_id: upload_request.output[:upload_id],
-                              file: file[:path])
-                  plan_action(Pulp::Repository::ImportUpload,
-                              pulp_id: repository.pulp_id,
-                              unit_type_id: unit_type_id,
-                              unit_key: unit_key(file, repository),
-                              upload_id: upload_request.output[:upload_id])
-                  plan_action(Pulp::Repository::DeleteUploadRequest,
-                              upload_id: upload_request.output[:upload_id])
+                  plan_pulp_action([Pulp::Orchestration::Repository::UploadContent,
+                                    Pulp3::Orchestration::Repository::UploadContent],
+                                                   repository, SmartProxy.pulp_master!, file, unit_type_id)
                 end
               end
             end

data/app/lib/actions/pulp/consumer/sync_capsule.rb

@@ -8,6 +8,10 @@ module Actions
           param :sync_options
         end
 
+        def plan(repository, smart_proxy, options)
+          plan_self(:capsule_id => smart_proxy.id, :repo_pulp_id => repository.pulp_id, :sync_options => options)
+        end
+
         def humanized_name
           _("Synchronize capsule content")
         end

data/app/lib/actions/pulp/orchestration/orphan_cleanup/remove_orphans.rb

@@ -0,0 +1,16 @@
+module Actions
+  module Pulp
+    module Orchestration
+      module OrphanCleanup
+        class RemoveOrphans < Pulp::Abstract
+          def plan(proxy)
+            sequence do
+              plan_action(Actions::Pulp::OrphanCleanup::RemoveUnneededRepos, proxy) unless proxy.pulp_master?
+              plan_action(Actions::Pulp::OrphanCleanup::RemoveOrphans, proxy)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp/orchestration/repository/refresh_repos.rb

@@ -0,0 +1,25 @@
+module Actions
+  module Pulp
+    module Orchestration
+      module Repository
+        class RefreshRepos < Pulp::AbstractAsyncTask
+          include ::Actions::Katello::CapsuleContent::RefreshRepos
+          input_format do
+            param :smart_proxy_id
+            param :environment_id
+            param :content_view_id
+            param :repository_id
+          end
+
+          def fetch_proxy_service(smart_proxy)
+            ::Katello::Pulp::SmartProxyRepository.new(smart_proxy)
+          end
+
+          def act_on_repo?(repo, smart_proxy)
+            !smart_proxy.pulp3_support?(repo)
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp/orchestration/repository/smart_proxy_sync.rb

@@ -0,0 +1,34 @@
+module Actions
+  module Pulp
+    module Orchestration
+      module Repository
+        class SmartProxySync < Pulp::Abstract
+          # rubocop:disable MethodLength
+          def plan(repository, smart_proxy, options)
+            skip_metadata_check = options.fetch(:skip_metadata_check, false)
+            sequence do
+              if repository && ['yum', 'puppet'].exclude?(repository.content_type)
+                # we unassociate units in non-yum/puppet repos in order to avoid version conflicts
+                # during publish. (i.e. two versions of a unit in the same repository)
+                plan_action(Pulp::Consumer::UnassociateUnits,
+                             capsule_id: smart_proxy.id,
+                             repo_pulp_id: repository.pulp_id)
+              end
+              pulp_options = { remove_missing: repository && ["deb", "puppet", "yum"].include?(repository.content_type) }
+              pulp_options[:force_full] = true if skip_metadata_check && repository.content_type == "yum"
+
+              plan_action(Pulp::Consumer::SyncCapsule,
+                               repository, smart_proxy, pulp_options)
+              if skip_metadata_check
+                plan_action(Katello::Repository::MetadataGenerate,
+                            repository,
+                            smart_proxy: smart_proxy,
+                            force: true)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp/orchestration/repository/upload_content.rb

@@ -0,0 +1,35 @@
+module Actions
+  module Pulp
+    module Orchestration
+      module Repository
+        class UploadContent < Pulp::Abstract
+          def plan(repository, smart_proxy, file, unit_type_id)
+            sequence do
+              upload_request = plan_action(Pulp::Repository::CreateUploadRequest)
+              plan_action(Pulp::Repository::UploadFile,
+                          upload_id: upload_request.output[:upload_id],
+                          file: file[:path])
+              plan_action(Pulp::Repository::ImportUpload,
+                          repository, smart_proxy,
+                          pulp_id: repository.pulp_id,
+                          unit_type_id: unit_type_id,
+                          unit_key: unit_key(file, repository),
+                          upload_id: upload_request.output[:upload_id])
+              plan_action(Pulp::Repository::DeleteUploadRequest,
+                          upload_id: upload_request.output[:upload_id])
+            end
+          end
+
+          def unit_key(file, repository)
+            return {} unless repository.file?
+            {
+              :checksum => Digest::SHA256.hexdigest(File.read(file[:path])),
+              :name => file[:filename],
+              :size => File.size(file[:path])
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp/orphan_cleanup/remove_orphans.rb

@@ -0,0 +1,15 @@
+module Actions
+  module Pulp
+    module OrphanCleanup
+      class RemoveOrphans < Pulp::AbstractAsyncTask
+        def plan(proxy)
+          plan_self(:capsule_id => proxy.id)
+        end
+
+        def invoke_external_task
+          pulp_resources.content.remove_orphans
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/{katello/capsule_content → pulp/orphan_cleanup}/remove_unneeded_repos.rb

@@ -1,6 +1,6 @@
 module Actions
-  module Katello
-    module CapsuleContent
+  module Pulp
+    module OrphanCleanup
       class RemoveUnneededRepos < Pulp::AbstractAsyncTask
         def plan(smart_proxy)
           plan_self(:smart_proxy_id => smart_proxy.id)

data/app/lib/actions/pulp/repository/import_upload.rb

@@ -2,20 +2,16 @@ module Actions
   module Pulp
     module Repository
       class ImportUpload < Pulp::AbstractAsyncTask
-        input_format do
-          param :pulp_id
-          param :unit_type_id
-          param :upload_id
-          param :unit_key
-          param :unit_metadata
+        def plan(_repo, _smart_proxy, options)
+          plan_self(:options => options)
         end
 
         def invoke_external_task
-          output[:pulp_tasks] = [pulp_resources.content.import_into_repo(input[:pulp_id],
-                                                   input[:unit_type_id],
-                                                   input[:upload_id],
-                                                   input[:unit_key],
-                                                   unit_metadata: input[:unit_metadata] || {})]
+          output[:pulp_tasks] = [pulp_resources.content.import_into_repo(input[:options][:pulp_id],
+                                                   input[:options][:unit_type_id],
+                                                   input[:options][:upload_id],
+                                                   input[:options][:unit_key],
+                                                   unit_metadata: input[:options][:unit_metadata] || {})]
         end
       end
     end

data/app/lib/actions/pulp3/abstract_async_task.rb

@@ -15,8 +15,8 @@ module Actions
 
       # A call report Looks like:  {"task":"/pulp/api/v3/tasks/5/"}
       # {
-      #    "_href":"/pulp/api/v3/tasks/4/",
-      #    "_created":"2019-02-21T19:50:40.476767Z",
+      #    "pulp_href":"/pulp/api/v3/tasks/4/",
+      #    "pulp_created":"2019-02-21T19:50:40.476767Z",
       #    "job_id":"d0359658-d926-47a2-b430-1b2092b3bd86",
       #    "state":"completed",
       #    "name":"pulp_file.app.tasks.publishing.publish",
@@ -84,16 +84,16 @@ module Actions
       def cancel
         output[:pulp_tasks].each do |pulp_task|
           data = PulpcoreClient::Task.new(state: 'canceled')
-          tasks_api.cancel(pulp_task['_href'], data)
+          tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
           if pulp_task['spawned_tasks']
             #the main task may have completed, so cancel spawned tasks too
-            pulp_task['spawned_tasks'].each { |spawned| tasks_api.cancel(spawned['_href'], data) }
+            pulp_task['spawned_tasks'].each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
           end
         end
       end
 
       def rescue_external_task(error)
-        if error.is_a?(::Katello::Errors::PulpError)
+        if error.is_a?(::Katello::Errors::Pulp3Error)
           fail error
         else
           super
@@ -114,7 +114,7 @@ module Actions
       def external_task=(external_task_data)
         output[:pulp_tasks] = transform_task_response(external_task_data)
         output[:pulp_tasks].each do |pulp_task|
-          if (pulp_exception = ::Katello::Errors::PulpError.from_task(pulp_task))
+          if (pulp_exception = ::Katello::Errors::Pulp3Error.from_task(pulp_task))
             fail pulp_exception
           end
         end
@@ -127,7 +127,7 @@ module Actions
 
       def poll_external_task
         external_task.map do |task|
-          task = tasks_api.read(task['_href'] || task['task'])
+          task = tasks_api.read(task['pulp_href'] || task['task'])
           task.as_json
         end
       end

data/app/lib/actions/pulp3/capsule_content/generate_metadata.rb

@@ -0,0 +1,32 @@
+module Actions
+  module Pulp3
+    module CapsuleContent
+      class GenerateMetadata < Pulp3::AbstractAsyncTask
+        middleware.use Actions::Middleware::ExecuteIfContentsChanged
+        def plan(repository, smart_proxy, options = {})
+          options[:contents_changed] = (options && options.key?(:contents_changed)) ? options[:contents_changed] : true
+          sequence do
+            if !::Katello::RepositoryTypeManager.find(repository.content_type).pulp3_skip_publication
+              action_output = plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id,
+                                 :options => options).output
+              plan_action(RefreshDistribution, repository, smart_proxy,
+                            :tasks => action_output,
+                            :use_repository_version => false,
+                            :contents_changed => options[:contents_changed])
+            else
+              plan_action(RefreshDistribution, repository, smart_proxy,
+                            :use_repository_version => true,
+                            :contents_changed => options[:contents_changed])
+            end
+          end
+        end
+
+        def invoke_external_task
+          repository = ::Katello::Repository.find(input[:repository_id])
+          smart_proxy = ::SmartProxy.unscoped.find(input[:smart_proxy_id])
+          output[:response] = repository.backend_service(smart_proxy).create_mirror_publication
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/capsule_content/refresh_distribution.rb

@@ -0,0 +1,31 @@
+module Actions
+  module Pulp3
+    module CapsuleContent
+      class RefreshDistribution < Pulp3::AbstractAsyncTask
+        include Helpers::Presenter
+        middleware.use Actions::Middleware::ExecuteIfContentsChanged
+
+        def plan(repository, smart_proxy, options = {})
+          plan_self(:repository_id => repository.id,
+                             :smart_proxy_id => smart_proxy.id,
+                             :options => options)
+        end
+
+        def run
+          smart_proxy = ::SmartProxy.unscoped.find(input[:smart_proxy_id])
+          options = input[:options]
+          tasks = options[:tasks]
+          repo = ::Katello::Repository.find(input[:repository_id])
+          if options[:use_repository_version]
+            output[:response] = repo.backend_service(smart_proxy).refresh_mirror_distributions(:use_repository_version => true)
+          elsif tasks && tasks[:pulp_tasks] && tasks[:pulp_tasks].first
+            publication_href = tasks[:pulp_tasks].first[:created_resources].first
+            if publication_href
+              output[:response] = repo.backend_service(smart_proxy).refresh_mirror_distributions(:publication => publication_href)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/capsule_content/sync.rb

@@ -0,0 +1,26 @@
+module Actions
+  module Pulp3
+    module CapsuleContent
+      class Sync < Pulp3::AbstractAsyncTask
+        def plan(repository, smart_proxy, options = {})
+          sequence do
+            plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :options => options)
+            plan_action(GenerateMetadata, repository, smart_proxy, options)
+          end
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync_mirror
+        end
+
+        def rescue_strategy_for_self
+          # There are various reasons the syncing fails, not all of them are
+          # fatal: when fail on syncing, we continue with the task ending up
+          # in the warning state, but not locking further syncs
+          Dynflow::Action::Rescue::Skip
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/orphan_cleanup/remove_orphans.rb

@@ -0,0 +1,20 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module OrphanCleanup
+        class RemoveOrphans < Pulp::Abstract
+          def plan(proxy)
+            if proxy.pulp3_enabled?
+              sequence do
+                if proxy.pulp_mirror?
+                  plan_action(Actions::Pulp3::OrphanCleanup::RemoveUnneededRepos, proxy)
+                end
+                plan_action(Actions::Pulp3::OrphanCleanup::DeleteOrphanRepositoryVersions, proxy)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end