katello 3.18.0.rc1 → 4.0.0.rc1

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -19,25 +19,15 @@ module Katello
           repo_api.read(version_href_to_repository_href(version_href))
         end
 
-        def repository_hrefs
-          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
-        end
-
-        def version_hrefs
-          repositories.pluck(:version_href).compact
-        end
-
-        def repositories
-          if @content_view_version.default?
-            @content_view_version.repositories.yum_type
-          else
-            @content_view_version.archived_repos.yum_type
-          end
-        end
-
         def version_href_to_repository_href(version_href)
           version_href.split("/")[0..-3].join("/") + "/"
         end
+
+        def zero_version_href(repository_href)
+          #  /pulp/api/v3/repositories/rpm/rpm/e59c4334-81d2-4d6b-a1a1-b61fa55ed664/versions/0/
+          repository_href += "/" unless repository_href.ends_with?('/')
+          "#{repository_href}versions/0/"
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -0,0 +1,84 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ImportValidator
+        attr_accessor :metadata, :path, :content_view
+        def initialize(content_view:, path:, metadata:)
+          self.content_view = content_view
+          self.path = path
+          self.metadata = metadata
+        end
+
+        def check!
+          unless content_view.default?
+            ensure_importing_cvv_does_not_exist!
+            ensure_from_cvv_exists!
+          end
+          ensure_repositories_metadata_are_in_the_library!
+        end
+
+        def ensure_importing_cvv_does_not_exist!
+          major = metadata[:content_view_version][:major]
+          minor = metadata[:content_view_version][:minor]
+
+          if ::Katello::ContentViewVersion.where(major: major, minor: minor, content_view: content_view).exists?
+            fail _("Content View Version specified in the metadata - '%{name}' already exists. "\
+                    "If you wish to replace the existing version, delete %{name} and try again. " % { name: "#{content_view.name} #{major}.#{minor}" })
+          end
+        end
+
+        def ensure_from_cvv_exists!
+          major = metadata[:content_view_version][:major]
+          minor = metadata[:content_view_version][:minor]
+
+          if metadata[:from_content_view_version].present?
+            from_major = metadata[:from_content_view_version][:major]
+            from_minor = metadata[:from_content_view_version][:minor]
+
+            unless ::Katello::ContentViewVersion.where(major: from_major, minor: from_minor, content_view: content_view).exists?
+              fail _("Prior Content View Version specified in the metadata - '%{name}' does not exist. "\
+                      "Please import the metadata for '%{name}' before importing '%{current}' " % { name: "#{content_view.name} #{from_major}.#{from_minor}",
+                                                                                                    current: "#{content_view.name} #{major}.#{minor}"})
+            end
+          end
+        end
+
+        def ensure_repositories_metadata_are_in_the_library!
+          repos_in_library = Katello::Repository.
+                              in_default_view.
+                              yum_type.
+                              joins(:product => :provider, :content_view_version => :content_view).
+                              joins(:root).
+                              where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                              pluck("#{::Katello::Product.table_name}.name",
+                                    "#{::Katello::RootRepository.table_name}.name",
+                                    "#{::Katello::Provider.table_name}.provider_type"
+                                    )
+
+          # repos_in_library look like [["prod1", "repo1", "Anonymous"], ["prod2", "repo2", "Red Hat"]]
+          product_repos_in_library = repos_in_library.map { |product, repo, provider| [product, repo, provider == ::Katello::Provider::REDHAT] }
+          product_repos_in_metadata = metadata[:repository_mapping].values.map { |repo| [repo[:product], repo[:repository], repo[:redhat]] }
+          # product_repos_in_library & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos_not_in_library = product_repos_in_metadata - product_repos_in_library
+          unless product_repos_not_in_library.blank?
+            repos_in_import = generate_product_repo_i18n_string(product_repos_not_in_library)
+            fail _("The following repositories provided in the import metadata are either not available in the Library or are of incorrect Respository Type. "\
+                    "Please add or enable the repositories before importing\n "\
+                    "%{repos}" % { content_view: content_view.name, repos: repos_in_import.join("")}
+                  )
+          end
+        end
+
+        def generate_product_repo_i18n_string(product_repos)
+          # product_repos look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos.map do |product, repo, redhat|
+            repo_type = redhat ? _("Red Hat") : _("Custom")
+            _("\n* Product = '%{product}', Repository = '%{repository}', Repository Type = '%{repo_type}'" % { product: product,
+                                                                                                               repository: repo,
+                                                                                                               repo_type: repo_type})
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerManifest < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_manifest".freeze
 
       def self.content_api
         PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/docker_tag.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerTag < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_tag".freeze
 
       def self.content_api
         PulpContainerClient::ContentTagsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/migration.rb

@@ -3,7 +3,7 @@ require 'pulp_2to3_migration_client'
 module Katello
   module Pulp3
     class Migration
-      attr_accessor :smart_proxy, :reimport_all
+      attr_accessor :smart_proxy, :reimport_all, :task_id
       GET_QUERY_ID_LENGTH = 90
 
       MUTABLE_CONTENT_TYPES = [
@@ -21,8 +21,10 @@ module Katello
         overridden.select { |type| SmartProxy.pulp_primary.pulp3_repository_type_support?(type.to_s, false) }.map { |t| t.to_s }
       end
 
-      def initialize(smart_proxy, repository_types = Migration.repository_types_for_migration, options = {})
-        self.reimport_all = options.fetch(:reimport, false)
+      def initialize(smart_proxy, options = {})
+        self.task_id = options.fetch(:task_id, nil)
+        self.reimport_all = options.fetch(:reimport_all, false)
+        repository_types = options.fetch(:repository_types, Migration.repository_types_for_migration)
 
         if (repository_types - smart_proxy.supported_pulp_types[:pulp3][:overriden_to_pulp2]).any?
           fail ::Katello::Errors::Pulp3MigrationError, _("Pulp 3 migration cannot run. Types %s have already been migrated.") %
@@ -62,6 +64,15 @@ module Katello
         [YumMetadataFile]
       end
 
+      def last_successful_migration_time
+        task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigration.to_s, :result => 'success').order("started_at desc").first
+        if reimport_all || task.nil?
+          0
+        else
+          task.started_at.to_i
+        end
+      end
+
       def content_types_for_migration
         content_types = @repository_types.collect do |repository_type_label|
           Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index
@@ -70,7 +81,20 @@ module Katello
         content_types.flatten - Migration.ignorable_content_types
       end
 
+      def update_import_status(message, index = nil)
+        #reduce output updating, only update every 20 items
+        if (index.nil? || index % 20 == 0) && self.task_id
+          progress = Katello::ContentMigrationProgress.find_or_create_by(:task_id => self.task_id)
+          progress.update(:progress_message => message)
+          progress.save!
+
+          fail Katello::Errors::Pulp3MigrationError, "Cancelled by user." if progress.canceled?
+        end
+      end
+
       def import_pulp3_content
+        update_import_status("Starting katello import phase.")
+
         Katello::Logging.time("CONTENT_MIGRATION - Total Import Process") do
           @repository_types.each do |repository_type_label|
             Katello::Logging.time("CONTENT_MIGRATION - Importing Repository", data: {type: repository_type_label}) do
@@ -78,7 +102,7 @@ module Katello
             end
 
             Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
-              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type}) do
+              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type.label}) do
                 import_content_type(content_type)
               end
             end
@@ -114,7 +138,9 @@ module Katello
         if repository_type_label == 'yum'
           import_yum_repos(imported, katello_repos)
         else
-          katello_repos.each do |repo|
+          repo_count = katello_repos.count
+          katello_repos.each_with_index do |repo, index|
+            update_import_status("Importing migrated content units #{repository_type_label}: #{index + 1}/#{repo_count}", index)
             found = imported.find { |migrated_repo| migrated_repo.pulp2_repo_id == repo.pulp_id }
             import_repo(repo, found) if found
           end
@@ -122,7 +148,9 @@ module Katello
       end
 
       def import_yum_repos(migrated_repo_items, repos)
-        repos.each do |yum_repo|
+        repo_count = repos.count
+        repos.each_with_index do |yum_repo, index|
+          update_import_status("Importing migrated yum repositories: #{index + 1}/#{repo_count}", index)
           to_find = nil
           if yum_repo.content_view.composite?
             if yum_repo.link?
@@ -165,10 +193,16 @@ module Katello
         process_distributions(pulp3_api, migrated_repo.pulp3_distribution_hrefs)
       end
 
+      def distribution_path_from_cache(href, pulp3_api)
+        @distribution_cache ||= {}
+        @distribution_cache[href] ||= pulp3_api.distributions_api.read(href).base_path
+        @distribution_cache[href]
+      end
+
       def process_distributions(pulp3_api, dist_hrefs_list)
         #distribution_path_hash(pulp3_api, dist_hrefs_list).each do |relative_path, href|
         dist_hrefs_list.each do |href|
-          relative_path = pulp3_api.distributions_api.read(href).base_path
+          relative_path = distribution_path_from_cache(href, pulp3_api)
 
           dist_ref = Katello::Pulp3::DistributionReference.find_or_initialize_by(:path => relative_path)
           if (distribution_repo = Katello::Repository.find_by(:relative_path => relative_path) || Katello::Repository.find_by(:container_repository_name => relative_path))
@@ -189,14 +223,16 @@ module Katello
       end
 
       def operate_on_errata
+        last_migration_time = last_successful_migration_time
         offset = 0
-        limit = 300
-        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+        limit = SETTINGS[:katello][:pulp][:bulk_load_size]
+        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
         total_count = response.count
         yield(response.results)
         until (offset + limit > total_count)
           offset += limit
-          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
+          update_import_status("Importing migrated content type erratum: #{offset + limit}/#{total_count}")
           yield(response.results)
         end
       end
@@ -236,7 +272,12 @@ module Katello
             unmigrated_units = unmigrated_units.where(:migrated_pulp3_href => nil)
           end
 
+          total_count = unmigrated_units.count
+          current_count = 0
+
           unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
+            current_count += needing_hrefs.count
+            update_import_status("Importing migrated content type #{content_type.label}: #{current_count}/#{total_count}")
             migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
             migrated_units.results.each do |migrated_unit|
               matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }

data/app/services/katello/pulp3/repository.rb

@@ -1,5 +1,5 @@
+# rubocop:disable Metrics/ClassLength
 require "pulpcore_client"
-
 module Katello
   module Pulp3
     class Repository
@@ -48,6 +48,10 @@ module Katello
         nil
       end
 
+      def published?
+        !repo.publication_href.nil?
+      end
+
       def skip_types
         nil
       end
@@ -188,8 +192,9 @@ module Katello
         api.repositories_api.read(repository_reference.try(:repository_href))
       end
 
-      def delete(href = repository_reference.try(:repository_href))
-        repository_reference.try(:destroy)
+      def delete_repository(repo_reference = repository_reference)
+        href = repo_reference.try(:repository_href)
+        repo_reference.try(:destroy)
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
@@ -257,8 +262,12 @@ module Katello
         create_version(:base_version => from_repository.version_href)
       end
 
+      def version_zero?
+        repo.version_href.ends_with?('/versions/0/')
+      end
+
       def delete_version
-        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
+        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) } unless version_zero?
       end
 
       def create_version(options = {})

data/app/services/katello/pulp3/repository/docker.rb

@@ -52,6 +52,11 @@ module Katello
                                       api.class.recursive_manage_class.new(content_units: options[:remove_content_units]))
         end
 
+        def tag_manifest(name, digest)
+          api.repositories_api.tag(repository_reference.repository_href,
+                                   api.class.tag_image_class.new(tag: name, digest: digest))
+        end
+
         def add_content(content_unit_href)
           content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
           api.repositories_api.add(repository_reference.repository_href, content_units: content_unit_href)

data/app/services/katello/pulp3/repository/yum.rb

@@ -159,8 +159,7 @@ module Katello
 
             data_dup.config[i][:content] = leftover_units.pop(copy_amount)
             unit_copy_counter += copy_amount
-            # Do copy call if limit is reached or if we're under the limit but on the last repo config.
-            if unit_copy_counter >= UNIT_LIMIT || (i == data_dup.config.size - 1 && leftover_units.empty?)
+            if unit_copy_counter != 0
               tasks << api.copy_api.copy_content(data_dup)
               unit_copy_counter = 0
             end
@@ -208,14 +207,20 @@ module Katello
           tasks
         end
 
-        def copy_all(source_repository)
-          data = PulpRpmClient::Copy.new
-          data.config = [{
-            source_repo_version: source_repository.version_href,
-            dest_repo: repository_reference.repository_href
-          }]
+        def copy_all(source_repository, mirror: false)
+          if mirror
+            data = PulpRpmClient::RepositoryAddRemoveContent.new(
+                      base_version: source_repository.version_href)
 
-          [api.copy_api.copy_content(data)]
+            [api.repositories_api.modify(repository_reference.repository_href, data)]
+          else
+            data = PulpRpmClient::Copy.new
+            data.config = [{
+              source_repo_version: source_repository.version_href,
+              dest_repo: repository_reference.repository_href
+            }]
+            [api.copy_api.copy_content(data)]
+          end
         end
 
         def remove_all_content_from_repo(repo_href)
@@ -335,7 +340,7 @@ module Katello
           multi_copy_units(repo_id_map, dependency_solving)
         end
 
-        def copy_content_for_source(source_repository, options = {})
+        def copy_content_for_source(source_repository, options = {}) # rubocop:disable Metrics/CyclomaticComplexity
           filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class|
             filter_class.where(:id => options[:filter_ids])
           end

data/app/services/katello/pulp3/task.rb

@@ -68,6 +68,10 @@ module Katello
         task_data[:finish_time] || FINISHED_STATES.include?(task_data[:state])
       end
 
+      def progress_reports
+        task_data['progress_reports']
+      end
+
       def poll
         task_data(true)
         self
@@ -79,7 +83,7 @@ module Katello
 
       def error
         if task_data[:state] == CANCELED
-          self.new(_("Task canceled"))
+          _("Task canceled")
         elsif task_data[:state] == FAILED
           if task_data[:error][:description].blank?
             _("Pulp task error")
@@ -90,8 +94,8 @@ module Katello
       end
 
       def cancel
-        data = PulpcoreClient::Task.new(state: 'canceled')
-        tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
+        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
+        tasks_api.tasks_cancel(task_data['pulp_href'], data)
         #the main task may have completed, so cancel spawned tasks too
         task_data['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
       end

data/app/services/katello/pulp3/task_group.rb

@@ -53,6 +53,10 @@ module Katello
         task_group_data['all_tasks_dispatched'] == true && IN_PROGRESS_STATES.all? { |state| task_group_data[state] == 0 }
       end
 
+      def group_progress_reports
+        task_group_data['group_progress_reports']
+      end
+
       def poll
         clear_task_group_data
         task_group_data
@@ -73,6 +77,12 @@ module Katello
       end
 
       def cancel
+        tasks_api = ::Katello::Pulp3::Api::Core.new(@smart_proxy).tasks_api
+        tasks_response = tasks_api.list(task_group: task_group_data['pulp_href'])
+        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
+        tasks_response.results.collect do |result|
+          tasks_api.tasks_cancel(result.pulp_href, data)
+        end
       end
     end
   end

data/app/services/katello/repository_type.rb

@@ -33,7 +33,7 @@ module Katello
     end
 
     def content_types_to_index
-      if SmartProxy.pulp_primary.pulp3_repository_type_support?(self)
+      if SmartProxy.pulp_primary&.pulp3_repository_type_support?(self)
         # type.index being false supersedes type.index_on_pulp3 being true
         @content_types.select { |type| type.index && type.index_on_pulp3 }
       else
@@ -70,7 +70,8 @@ module Katello
       {
         :name => self.id.to_s,
         :id => self.id,
-        :creatable => @allow_creation_by_user
+        :creatable => @allow_creation_by_user,
+        :pulp3_support => SmartProxy.pulp_primary.pulp3_repository_type_support?(self)
       }
     end