katello 3.17.0 → 3.18.0

data/app/lib/actions/pulp3/content_view/delete_repository_references.rb

@@ -14,7 +14,7 @@ module Actions
           content_view.repository_references.each do |repository_reference|
             repo = repository_reference.root_repository.library_instance
             #force pulp3 in case we've done migrations, but haven't switched over yet
-            tasks << repo.backend_service(smart_proxy, true).delete(repository_reference.repository_href)
+            tasks << repo.backend_service(smart_proxy, true).delete_repository(repository_reference)
           end
           content_view.repository_references.destroy_all
 

data/app/lib/actions/pulp3/content_view_version/create_importer.rb

@@ -6,13 +6,17 @@ module Actions
           param :smart_proxy_id, Integer
           param :content_view_version_id, Integer
           param :path, String
+          param :metadata, Hash
         end
 
         def run
           cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
-          output[:importer_data] = ::Katello::Pulp3::ContentViewVersion::Import.new(smart_proxy: smart_proxy,
-                                                                            content_view_version: cvv,
-                                                                            path: input[:path]).create_importer
+          output[:importer_data] = ::Katello::Pulp3::ContentViewVersion::Import.new(
+            smart_proxy: smart_proxy,
+            content_view_version: cvv,
+            path: input[:path],
+            metadata: input[:metadata]
+          ).create_importer
         end
       end
     end

data/app/lib/actions/pulp3/content_view_version/export.rb

@@ -4,14 +4,20 @@ module Actions
       class Export < Pulp3::AbstractAsyncTask
         input_format do
           param :content_view_version_id, Integer
+          param :from_content_view_version_id, Integer
           param :smart_proxy_id, Integer
           param :exporter_data, Hash
+          param :chunk_size, Integer
         end
 
         def invoke_external_task
           cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+          from_cvv = ::Katello::ContentViewVersion.find(input[:from_content_view_version_id]) unless input[:from_content_view_version_id].blank?
           ::Katello::Pulp3::ContentViewVersion::Export.new(smart_proxy: smart_proxy,
-                                                   content_view_version: cvv).create_export(input[:exporter_data][:pulp_href])
+                                                   content_view_version: cvv,
+                                                   from_content_view_version: from_cvv)
+                                                   .create_export(input[:exporter_data][:pulp_href],
+                                                                        chunk_size: input[:chunk_size])
         end
       end
     end

data/app/lib/actions/pulp3/content_view_version/import.rb

@@ -7,13 +7,17 @@ module Actions
           param :smart_proxy_id, Integer
           param :importer_data, Hash
           param :path, String
+          param :metadata, Hash
         end
 
         def invoke_external_task
           cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
-          output[:pulp_tasks] = ::Katello::Pulp3::ContentViewVersion::Import.new(smart_proxy: smart_proxy,
-                                                   content_view_version: cvv,
-                                                   path: input[:path]).create_import(input[:importer_data][:pulp_href])
+          output[:pulp_tasks] = ::Katello::Pulp3::ContentViewVersion::Import.new(
+            smart_proxy: smart_proxy,
+            content_view_version: cvv,
+            path: input[:path],
+            metadata: input[:metadata]
+          ).create_import(input[:importer_data][:pulp_href])
         end
       end
     end

data/app/lib/actions/pulp3/import_migration.rb

@@ -6,9 +6,14 @@ module Actions
       end
 
       def run
-        migration_service = ::Katello::Pulp3::Migration.new(SmartProxy.pulp_primary)
+        task_id = ForemanTasks::Task.find_by(external_id: self.execution_plan_id)&.id
+        migration_service = ::Katello::Pulp3::Migration.new(SmartProxy.pulp_primary, input.merge(task_id: task_id))
         migration_service.import_pulp3_content
       end
+
+      def humanized_output
+        output[:status]
+      end
     end
   end
 end

data/app/lib/actions/pulp3/orchestration/content_view_version/copy_version_units_to_library.rb

@@ -9,7 +9,8 @@ module Actions
                 sequence do
                   copy_action = plan_action(Actions::Pulp3::Repository::CopyContent, repo, SmartProxy.pulp_primary!,
                                                     repo.library_instance,
-                                                    copy_all: true)
+                                                    copy_all: true,
+                                                    mirror: content_view_version.content_view.library_import?)
                   plan_action(Actions::Pulp3::Repository::SaveVersion, repo.library_instance,
                                 tasks: copy_action.output[:pulp_tasks])
                   plan_action(Katello::Repository::IndexContent, id: repo.library_instance_id)

data/app/lib/actions/pulp3/orchestration/content_view_version/export.rb

@@ -6,17 +6,22 @@ module Actions
           input_format do
             param :smart_proxy_id, Integer
             param :content_view_version_id, Integer
+            param :from_content_view_version_id, Integer
             param :export_history_id, Integer
             param :exporter_data, Hash
             param :destination_server, String
           end
 
           output_format do
-            param :exported_file_name, String
-            param :exported_file_checksum, String
+            param :exported_file_checksum, Hash
+            param :export_path, String
           end
 
-          def plan(content_view_version, destination_server:)
+          # rubocop:disable Metrics/MethodLength
+          def plan(content_view_version:, destination_server: nil,
+                   chunk_size: nil, from_history: nil,
+                   validate_incremental: true,
+                   fail_on_missing_content: false)
             action_subject(content_view_version)
             unless File.directory?(Setting['pulpcore_export_destination'])
               fail ::Foreman::Exception, N_("Unable to export. 'pulpcore_export_destination' setting is not set to a valid directory.")
@@ -24,6 +29,15 @@ module Actions
 
             sequence do
               smart_proxy = SmartProxy.pulp_primary!
+              from_content_view_version = from_history&.content_view_version
+              export_service = ::Katello::Pulp3::ContentViewVersion::Export.new(
+                                                     smart_proxy: smart_proxy,
+                                                     content_view_version: content_view_version,
+                                                     destination_server: destination_server,
+                                                     from_content_view_version: from_content_view_version)
+              export_service.validate!(fail_on_missing_content: fail_on_missing_content,
+                                       validate_incremental: validate_incremental)
+
               action_output = plan_action(::Actions::Pulp3::ContentViewVersion::CreateExporter,
                                      content_view_version_id: content_view_version.id,
                                      smart_proxy_id: smart_proxy.id,
@@ -32,12 +46,14 @@ module Actions
               plan_action(::Actions::Pulp3::ContentViewVersion::Export,
                                      content_view_version_id: content_view_version.id,
                                      smart_proxy_id: smart_proxy.id,
-                                     exporter_data: action_output[:exporter_data]
-                                     )
+                                     exporter_data: action_output[:exporter_data],
+                                     chunk_size: chunk_size,
+                                     from_content_view_version_id: from_content_view_version&.id)
 
               plan_self(exporter_data: action_output[:exporter_data], smart_proxy_id: smart_proxy.id,
                         destination_server: destination_server,
-                        content_view_version_id: content_view_version.id)
+                        content_view_version_id: content_view_version.id,
+                        from_content_view_version_id: from_content_view_version&.id)
 
               plan_action(::Actions::Pulp3::ContentViewVersion::DestroyExporter,
                             smart_proxy_id: smart_proxy.id,
@@ -49,18 +65,26 @@ module Actions
             smart_proxy = ::SmartProxy.find(input[:smart_proxy_id])
             api = ::Katello::Pulp3::Api::Core.new(smart_proxy)
             export_data = api.export_api.list(input[:exporter_data][:pulp_href]).results.first
-            output[:exported_file_name], output[:exported_file_checksum] = export_data.output_file_info.first
-            path = File.dirname(output[:exported_file_name].to_s)
-            ::Katello::ContentViewVersionExportHistory.create!(content_view_version_id: input[:content_view_version_id],
-                                                               destination_server: input[:destination_server],
-                                                               path: path)
+            output[:exported_file_checksum] = export_data.output_file_info
+            file_name = output[:exported_file_checksum].first&.first
+            path = File.dirname(file_name.to_s)
+            output[:export_path] = path
             cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+            from_cvv = ::Katello::ContentViewVersion.find(input[:from_content_view_version_id]) unless input[:from_content_view_version_id].blank?
 
-            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(:content_view_version => cvv,
-                                                               :smart_proxy => smart_proxy).generate_metadata
+            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(
+                                                       content_view_version: cvv,
+                                                       smart_proxy: smart_proxy,
+                                                       from_content_view_version: from_cvv).generate_metadata
             toc = Dir.glob("#{path}/*toc.json").first
             export_metadata[:toc] = File.basename(toc) if toc
-            File.write("#{path}/#{::Katello::Pulp3::ContentViewVersion::Export::METADATA_FILE}", export_metadata.to_json)
+            history = ::Katello::ContentViewVersionExportHistory.create!(
+              content_view_version_id: input[:content_view_version_id],
+              destination_server: input[:destination_server],
+              path: path,
+              metadata: export_metadata
+            )
+            output[:export_history_id] = history.id
           end
 
           def humanized_name

data/app/lib/actions/pulp3/orchestration/content_view_version/export_library.rb

@@ -0,0 +1,60 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module ContentViewVersion
+        class ExportLibrary < Actions::EntryAction
+          def plan(organization, destination_server: nil,
+                                 chunk_size: nil,
+                                 from_history: nil,
+                                 fail_on_missing_content: false)
+            action_subject(organization)
+            validate_repositories_immediate!(organization) if fail_on_missing_content
+
+            content_view = ::Katello::Pulp3::ContentViewVersion::Export.find_library_export_view(destination_server: destination_server,
+                                                                           organization: organization,
+                                                                           create_by_default: true)
+            repo_ids_in_library = organization.default_content_view_version.repositories.yum_type.immediate.pluck(:id)
+            content_view.update!(repository_ids: repo_ids_in_library)
+
+            sequence do
+              publish_action = plan_action(::Actions::Katello::ContentView::Publish, content_view, '')
+              export_action = plan_action(::Actions::Pulp3::Orchestration::ContentViewVersion::Export,
+                                          content_view_version: publish_action.version,
+                                          destination_server: destination_server,
+                                          chunk_size: chunk_size,
+                                          from_history: from_history,
+                                          validate_incremental: false,
+                                          fail_on_missing_content: fail_on_missing_content)
+              plan_self(export_action_output: export_action.output)
+            end
+          end
+
+          def run
+            output[:export_history_id] = input[:export_action_output][:export_history_id]
+          end
+
+          def humanized_name
+            _("Export Library")
+          end
+
+          def rescue_strategy
+            Dynflow::Action::Rescue::Skip
+          end
+
+          def validate_repositories_immediate!(organization)
+            non_immediate_repos = organization.default_content_view_version.repositories.yum_type.non_immediate
+            if non_immediate_repos.any?
+              fail _("NOTE: Unable to fully export '%{organization}' organization's library because"\
+                     " it contains repositories without the 'immediate' download policy."\
+                     " Update the download policy and sync affected repositories to include them in the export."\
+                     " \n %{repos}" %
+                     { organization: organization.name,
+                       repos: ::Katello::Pulp3::ContentViewVersion::Export
+                              .generate_product_repo_strings(repositories: non_immediate_repos)})
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/content_view_version/import.rb

@@ -3,20 +3,26 @@ module Actions
     module Orchestration
       module ContentViewVersion
         class Import < Actions::EntryAction
-          def plan(content_view_version, path:)
+          def plan(content_view_version, path:, metadata:)
             action_subject(content_view_version)
             sequence do
               smart_proxy = SmartProxy.pulp_primary!
-              importer_output = plan_action(::Actions::Pulp3::ContentViewVersion::CreateImporter,
-                                           content_view_version_id: content_view_version.id,
-                                           smart_proxy_id: smart_proxy.id,
-                                           path: path).output
+              importer_output = plan_action(
+                ::Actions::Pulp3::ContentViewVersion::CreateImporter,
+                content_view_version_id: content_view_version.id,
+                smart_proxy_id: smart_proxy.id,
+                path: path,
+                metadata: metadata
+              ).output
 
-              import_output = plan_action(::Actions::Pulp3::ContentViewVersion::Import,
-                                           content_view_version_id: content_view_version.id,
-                                           smart_proxy_id: smart_proxy.id,
-                                           importer_data: importer_output[:importer_data],
-                                           path: path).output
+              import_output = plan_action(
+                ::Actions::Pulp3::ContentViewVersion::Import,
+                content_view_version_id: content_view_version.id,
+                smart_proxy_id: smart_proxy.id,
+                importer_data: importer_output[:importer_data],
+                path: path,
+                metadata: metadata
+              ).output
 
               plan_action(Actions::Pulp3::Repository::SaveVersions, content_view_version.importable_repositories.pluck(:id),
                           tasks: import_output[:pulp_tasks])

data/app/lib/actions/pulp3/orchestration/repository/generate_metadata.rb

@@ -8,7 +8,7 @@ module Actions
             publication_content_type = !::Katello::RepositoryTypeManager.find(repository.content_type).pulp3_skip_publication
             sequence do
               if options[:source_repository] && publication_content_type
-                plan_self(source_repository_id: options[:source_repository].id, target_repository_id: repository.id)
+                plan_self(source_repository_id: options[:source_repository].id, target_repository_id: repository.id, smart_proxy_id: smart_proxy.id)
               elsif publication_content_type
                 plan_action(Actions::Pulp3::Repository::CreatePublication, repository, smart_proxy, options)
               end
@@ -21,6 +21,9 @@ module Actions
             #we don't have to actually generate a publication, we can reuse the old one
             target_repo = ::Katello::Repository.find(input[:target_repository_id])
             source_repo = ::Katello::Repository.find(input[:source_repository_id])
+            if (target_repo.publication_href != source_repo.publication_href && smart_proxy.pulp_primary?)
+              target_repo.clear_smart_proxy_sync_histories
+            end
             target_repo.update!(publication_href: source_repo.publication_href)
           end
         end

data/app/lib/actions/pulp3/orchestration/repository/import_upload.rb

@@ -1,3 +1,4 @@
+# rubocop:disable Metrics/MethodLength
 module Actions
   module Pulp3
     module Orchestration
@@ -6,9 +7,19 @@ module Actions
           def plan(repository, smart_proxy, args)
             file = {:filename => args.dig(:unit_key, :name)}
             content_unit_href = args.dig(:unit_key, :content_unit_id)
+            docker_tag = (args.dig(:unit_type_id) == "docker_tag")
             sequence do
               if content_unit_href
                 plan_self(:commit_output => [], :content_unit_href => content_unit_href)
+                action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
+                plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
+              elsif docker_tag
+                tag_manifest_output = plan_action(Pulp3::Repository::UploadTag,
+                            repository,
+                            smart_proxy,
+                            args).output
+                plan_self(:commit_output => tag_manifest_output[:pulp_tasks])
+                plan_action(Pulp3::Repository::SaveVersion, repository, {force_fetch_version: true, tasks: tag_manifest_output[:pulp_tasks]})
               else
                 commit_output = plan_action(Pulp3::Repository::CommitUpload,
                                             repository,
@@ -24,9 +35,9 @@ module Actions
                                               args.dig(:unit_type_id)).output
                 content_unit_href = artifact_output[:pulp_tasks]
                 plan_self(:commit_output => commit_output[:pulp_tasks], :artifact_output => artifact_output[:pulp_tasks])
+                action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
+                plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
               end
-              action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
-              plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
             end
           end
 
@@ -34,8 +45,10 @@ module Actions
             output[:pulp_tasks] = input[:commit_output]
             if input[:content_unit_href]
               output[:content_unit_href] = input[:content_unit_href]
-            else
+            elsif input[:artifact_output]
               output[:content_unit_href] = input[:artifact_output].last[:created_resources].first
+            else
+              output[:content_unit_href] = nil
             end
           end
         end

data/app/lib/actions/pulp3/repository/commit_upload.rb

@@ -10,7 +10,8 @@ module Actions
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
           uploads_api = repo_backend_service.core_api.uploads_api
-          output[:pulp_tasks] = [uploads_api.commit(input[:upload_href], sha256: input[:sha256])]
+          upload_commit = repo_backend_service.core_api.upload_commit_class.new(sha256: input[:sha256])
+          output[:pulp_tasks] = [uploads_api.commit(input[:upload_href], upload_commit)]
         end
       end
     end

data/app/lib/actions/pulp3/repository/copy_content.rb

@@ -13,7 +13,7 @@ module Actions
           target = ::Katello::Repository.find(input[:target_repository_id] || input[:target_repository])
           service = target.backend_service(smart_proxy)
           output[:pulp_tasks] = if input[:copy_all]
-                                  service.copy_all(source)
+                                  service.copy_all(source, mirror: input[:mirror] || false)
                                 else
                                   service.copy_content_for_source(source, input)
                                 end

data/app/lib/actions/pulp3/repository/delete.rb

@@ -8,7 +8,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
-          output[:response] = repo.backend_service(smart_proxy).delete
+          output[:response] = repo.backend_service(smart_proxy).delete_repository
         end
       end
     end

data/app/lib/actions/pulp3/repository/save_artifact.rb

@@ -11,7 +11,7 @@ module Actions
           artifact_href = input[:tasks].last[:created_resources].first
           content_type = input[:unit_type_id]
           content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
-          output[:pulp_tasks] = [content_backend_service.content_api.create(input[:options][:file_name], artifact: artifact_href)]
+          output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name], artifact: artifact_href)]
         end
       end
     end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -2,7 +2,7 @@ module Actions
   module Pulp3
     module Repository
       class SaveVersion < Pulp3::Abstract
-        def plan(repository, options)
+        def plan(repository, options = {})
           fail "Cannot accept tasks and repository_details into Save Version." if options[:tasks].present? && options[:repository_details].present?
           plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details], :force_fetch_version => options.fetch(:force_fetch_version, false))
         end

data/app/lib/actions/pulp3/repository/upload_tag.rb

@@ -0,0 +1,18 @@
+module Actions
+  module Pulp3
+    module Repository
+      class UploadTag < Pulp3::AbstractAsyncTask
+        def plan(repository, smart_proxy, args)
+          plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :args => args)
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          tag_name = input[:args].dig(:unit_key, :name)
+          manifest_digest = input[:args].dig(:unit_key, :digest)
+          output[:pulp_tasks] = [repo.backend_service(smart_proxy).tag_manifest(tag_name, manifest_digest)]
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/resources/candlepin/consumer.rb

@@ -53,8 +53,8 @@ module Katello
             JSON.parse(response).with_indifferent_access
           end
 
-          def async_hypervisors(owner, raw_json)
-            url = "/candlepin/hypervisors/#{owner}"
+          def async_hypervisors(owner:, reporter_id:, raw_json:)
+            url = "/candlepin/hypervisors/#{owner}?reporter_id=#{reporter_id}"
             headers = self.default_headers
             headers['content-type'] = 'text/plain'
             response = self.post(url, raw_json, headers)

data/app/lib/katello/resources/candlepin/owner.rb

@@ -5,6 +5,11 @@ module Katello
         extend OwnerResource
 
         class << self
+          def all
+            response = self.get(path, default_headers)
+            JSON.parse(response.body)
+          end
+
           # Set the contentPrefix at creation time so that the client will get
           # content only for the org it has been subscribed to
           def create(key, description)