katello 3.18.0.rc2.1 → 3.18.2.1

data/app/services/katello/pulp3/docker_manifest.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerManifest < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_manifest".freeze
 
       def self.content_api
         PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/docker_tag.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerTag < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_tag".freeze
 
       def self.content_api
         PulpContainerClient::ContentTagsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/erratum.rb

@@ -35,7 +35,8 @@ module Katello
         custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])
         custom_json["updated"] = convert_date_if_epoch(custom_json["updated"]) unless custom_json["updated"].blank?
 
-        if model.updated.blank? || (custom_json['updated'].to_datetime != model.updated.to_datetime)
+        if model.updated.blank? ||
+            (custom_json['updated'] && (custom_json['updated'].to_datetime != model.updated.to_datetime))
           custom_json['errata_id'] = custom_json.delete('id')
           custom_json['errata_type'] = custom_json.delete('type')
           custom_json['updated'] = custom_json['updated'].blank? ? custom_json['issued'] : custom_json['updated']

data/app/services/katello/pulp3/migration.rb

@@ -3,7 +3,7 @@ require 'pulp_2to3_migration_client'
 module Katello
   module Pulp3
     class Migration
-      attr_accessor :smart_proxy, :reimport_all
+      attr_accessor :smart_proxy, :reimport_all, :task_id
       GET_QUERY_ID_LENGTH = 90
 
       MUTABLE_CONTENT_TYPES = [
@@ -15,14 +15,20 @@ module Katello
         Katello::Erratum
       ].freeze
 
+      CORRUPTABLE_CONTENT_TYPES = [
+        Katello::Rpm, Katello::FileUnit
+      ].freeze
+
       def self.repository_types_for_migration
         #we can migrate types that pulp3 supports, but are overridden to pulp2.  These are in 'migration mode'
         overridden = (SETTINGS[:katello][:use_pulp_2_for_content_type] || {}).keys.select { |key| SETTINGS[:katello][:use_pulp_2_for_content_type][key] }
         overridden.select { |type| SmartProxy.pulp_primary.pulp3_repository_type_support?(type.to_s, false) }.map { |t| t.to_s }
       end
 
-      def initialize(smart_proxy, repository_types = Migration.repository_types_for_migration, options = {})
-        self.reimport_all = options.fetch(:reimport, false)
+      def initialize(smart_proxy, options = {})
+        self.task_id = options.fetch(:task_id, nil)
+        self.reimport_all = options.fetch(:reimport_all, false)
+        repository_types = options.fetch(:repository_types, Migration.repository_types_for_migration)
 
         if (repository_types - smart_proxy.supported_pulp_types[:pulp3][:overriden_to_pulp2]).any?
           fail ::Katello::Errors::Pulp3MigrationError, _("Pulp 3 migration cannot run. Types %s have already been migrated.") %
@@ -62,6 +68,15 @@ module Katello
         [YumMetadataFile]
       end
 
+      def last_successful_migration_time
+        task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigration.to_s, :result => 'success').order("started_at desc").first
+        if reimport_all || task.nil?
+          0
+        else
+          task.started_at.to_i
+        end
+      end
+
       def content_types_for_migration
         content_types = @repository_types.collect do |repository_type_label|
           Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index
@@ -70,7 +85,20 @@ module Katello
         content_types.flatten - Migration.ignorable_content_types
       end
 
+      def update_import_status(message, index = nil)
+        #reduce output updating, only update every 20 items
+        if (index.nil? || index % 20 == 0) && self.task_id
+          progress = Katello::ContentMigrationProgress.find_or_create_by(:task_id => self.task_id)
+          progress.update(:progress_message => message)
+          progress.save!
+
+          fail Katello::Errors::Pulp3MigrationError, "Cancelled by user." if progress.canceled?
+        end
+      end
+
       def import_pulp3_content
+        update_import_status("Starting katello import phase.")
+
         Katello::Logging.time("CONTENT_MIGRATION - Total Import Process") do
           @repository_types.each do |repository_type_label|
             Katello::Logging.time("CONTENT_MIGRATION - Importing Repository", data: {type: repository_type_label}) do
@@ -78,8 +106,9 @@ module Katello
             end
 
             Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
-              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type}) do
+              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type.label}) do
                 import_content_type(content_type)
+                mark_missing_content(content_type)
               end
             end
           end
@@ -90,6 +119,44 @@ module Katello
         Katello::Pulp3::MigrationPlan.new(@repository_types).generate.as_json
       end
 
+      def reset
+        if @repository_types.empty?
+          fail ::Katello::Errors::Pulp3MigrationError, 'There are no Pulp 3 content types to reset'
+        end
+
+        plugins = @repository_types.sort.map do |repository_type|
+          {
+            type: ::Katello::Pulp3::MigrationPlan.pulp2_repository_type(repository_type)
+          }
+        end
+        plan = { plugins: plugins }
+
+        # TODO: Don't provide the plan as a string once this is resolved: https://pulp.plan.io/issues/8211
+        migration_plan_api.reset(migration_plan_api.create(plan: plan).pulp_href, plan.to_json)
+
+        content_types_for_migration.each do |content_type|
+          if content_type.model_class == ::Katello::Erratum
+            ::Katello::RepositoryErratum.update_all(erratum_pulp3_href: nil)
+          else
+            content_type.model_class.update_all(migrated_pulp3_href: nil)
+          end
+        end
+
+        @repository_types.each do |repo_type|
+          if repo_type == "file"
+            ::Katello::Repository.file_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+          elsif repo_type == "docker"
+            ::Katello::Repository.docker_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+          elsif repo_type == "yum"
+            ::Katello::Repository.yum_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+          end
+        end
+
+        ::Katello::Pulp3::RepositoryReference.destroy_all
+        ::Katello::Pulp3::DistributionReference.destroy_all
+        ::Katello::Pulp3::ContentGuard.destroy_all
+      end
+
       def create_migrations
         plan = migration_plan
         Rails.logger.info("Migration Plan: #{plan}")
@@ -103,7 +170,7 @@ module Katello
       end
 
       def start_migration(plan_href)
-        migration_plan_api.run(plan_href, dry_run: false, validate: true)
+        migration_plan_api.run(plan_href, dry_run: false, validate: true, skip_corrupted: true)
       end
 
       def import_repositories(repository_type_label)
@@ -114,7 +181,9 @@ module Katello
         if repository_type_label == 'yum'
           import_yum_repos(imported, katello_repos)
         else
-          katello_repos.each do |repo|
+          repo_count = katello_repos.count
+          katello_repos.each_with_index do |repo, index|
+            update_import_status("Importing migrated content units #{repository_type_label}: #{index + 1}/#{repo_count}", index)
             found = imported.find { |migrated_repo| migrated_repo.pulp2_repo_id == repo.pulp_id }
             import_repo(repo, found) if found
           end
@@ -122,7 +191,9 @@ module Katello
       end
 
       def import_yum_repos(migrated_repo_items, repos)
-        repos.each do |yum_repo|
+        repo_count = repos.count
+        repos.each_with_index do |yum_repo, index|
+          update_import_status("Importing migrated yum repositories: #{index + 1}/#{repo_count}", index)
           to_find = nil
           if yum_repo.content_view.composite?
             if yum_repo.link?
@@ -195,14 +266,16 @@ module Katello
       end
 
       def operate_on_errata
+        last_migration_time = last_successful_migration_time
         offset = 0
-        limit = 300
-        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+        limit = SETTINGS[:katello][:pulp][:bulk_load_size]
+        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
         total_count = response.count
         yield(response.results)
         until (offset + limit > total_count)
           offset += limit
-          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
+          update_import_status("Importing migrated content type erratum: #{offset + limit}/#{total_count}")
           yield(response.results)
         end
       end
@@ -231,6 +304,12 @@ module Katello
                                           on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
       end
 
+      def mark_missing_content(content_type)
+        unless [Katello::DockerTag, Katello::DockerManifest, Katello::Erratum].include?(content_type.model_class)
+          content_type.model_class.where(:migrated_pulp3_href => nil).update_all(:missing_from_migration => true)
+        end
+      end
+
       def import_content_type(content_type)
         if content_type.model_class == Katello::Erratum
           import_errata
@@ -242,11 +321,15 @@ module Katello
             unmigrated_units = unmigrated_units.where(:migrated_pulp3_href => nil)
           end
 
+          total_count = unmigrated_units.count
+          current_count = 0
+
           unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
+            current_count += needing_hrefs.count
+            update_import_status("Importing migrated content type #{content_type.label}: #{current_count}/#{total_count}")
             migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
             migrated_units.results.each do |migrated_unit|
-              matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }
-              matching_record&.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content)
+              content_type.model_class.where(pulp_id: migrated_unit.pulp2_id).update_all(migrated_pulp3_href: migrated_unit.pulp3_content)
             end
           end
         end

data/app/services/katello/pulp3/migration_plan.rb

@@ -21,13 +21,13 @@ module Katello
       def generate_plugins
         @repository_types.sort.map do |repository_type|
           {
-            type: pulp2_repository_type(repository_type),
+            type: self.class.pulp2_repository_type(repository_type),
             repositories: repository_migrations(repository_type)
           }
         end
       end
 
-      def pulp2_repository_type(repository_type)
+      def self.pulp2_repository_type(repository_type)
         if repository_type == 'yum'
           return 'rpm' #migration plugin uses rpm
         else

data/app/services/katello/pulp3/migration_switchover.rb

@@ -21,6 +21,7 @@ module Katello
           Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_content_types") { combine_duplicate_content_types }
           Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_docker_tags") { combine_duplicate_docker_tags } if docker_migration?
           Katello::Logging.time("CONTENT_SWITCHOVER - migrate_pulp3_hrefs") { migrate_pulp3_hrefs }
+          Katello::Logging.time("CONTENT_SWITCHOVER - remove_missing_content") { remove_missing_content }
         end
       end
 
@@ -106,12 +107,29 @@ module Katello
         end
       end
 
-      def migrated_content_type_check
+      def remove_missing_content
         content_types.each do |content_type|
-          if content_type.model_class == Katello::Erratum
-            migrated_errata_check
-          elsif content_type.model_class.where(migrated_pulp3_href: nil).any?
-            fail SwitchOverError, "ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n"
+          if Migration::CORRUPTABLE_CONTENT_TYPES.include?(content_type.model_class)
+            content_type.model_class.ignored_missing_migrated_content.destroy_all
+          elsif content_type.model_class == Katello::Erratum
+            Katello::RepositoryErratum.where(:erratum_pulp3_href => nil).delete_all
+          else
+            content_type.model_class.unmigrated_content.destroy_all
+          end
+        end
+      end
+
+      def migrated_content_type_check
+        content_classes = content_types.map(&:model_class)
+        migrated_errata_check if content_classes.include?(Katello::Erratum)
+
+        (content_classes & Migration::CORRUPTABLE_CONTENT_TYPES).each do |content_type|
+          if content_type.missing_migrated_content.any?
+            fail SwitchOverError, "ERROR: at least one #{content_type.table_name} record has been detected as corrupt or missing. Run 'foreman-rake katello:pulp3_migration_stats' for more information.\n"
+          end
+
+          if content_type.unmigrated_content.any?
+            fail SwitchOverError, "ERROR: at least one #{content_type.table_name} record was not able to be migrated\n"
           end
         end
       end

data/app/services/katello/pulp3/repository.rb

@@ -1,7 +1,7 @@
 require "pulpcore_client"
-
 module Katello
   module Pulp3
+    # rubocop:disable Metrics/ClassLength
     class Repository
       include Katello::Util::HttpProxy
       attr_accessor :repo
@@ -188,8 +188,9 @@ module Katello
         api.repositories_api.read(repository_reference.try(:repository_href))
       end
 
-      def delete(href = repository_reference.try(:repository_href))
-        repository_reference.try(:destroy)
+      def delete_repository(repo_reference = repository_reference)
+        href = repo_reference.try(:repository_href)
+        repo_reference.try(:destroy)
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
@@ -257,8 +258,12 @@ module Katello
         create_version(:base_version => from_repository.version_href)
       end
 
+      def version_zero?
+        repo.version_href.ends_with?('/versions/0/')
+      end
+
       def delete_version
-        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
+        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) } unless version_zero?
       end
 
       def create_version(options = {})
@@ -300,7 +305,7 @@ module Katello
         }
         remote_options[:url] = root.url unless root.url.blank?
         remote_options[:download_concurrency] = root.download_concurrency unless root.download_concurrency.blank?
-        if root.upstream_username && root.upstream_password
+        if !root.upstream_username.blank? && !root.upstream_password.blank?
           remote_options.merge!(username: root.upstream_username,
                                password: root.upstream_password)
         end

data/app/services/katello/pulp3/repository/docker.rb

@@ -52,6 +52,11 @@ module Katello
                                       api.class.recursive_manage_class.new(content_units: options[:remove_content_units]))
         end
 
+        def tag_manifest(name, digest)
+          api.repositories_api.tag(repository_reference.repository_href,
+                                   api.class.tag_image_class.new(tag: name, digest: digest))
+        end
+
         def add_content(content_unit_href)
           content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
           api.repositories_api.add(repository_reference.repository_href, content_units: content_unit_href)

data/app/services/katello/pulp3/repository/yum.rb

@@ -14,7 +14,16 @@ module Katello
           url, sles_token = extract_sles_token
           options = common_remote_options
           options.merge!(sles_auth_token: sles_token) if sles_token
-          options.merge!(url: url, policy: root.download_policy)
+
+          options.merge!(url: url, policy: translated_download_policy)
+        end
+
+        def translated_download_policy
+          if root.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_BACKGROUND
+            ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE
+          else
+            root.download_policy
+          end
         end
 
         def extract_sles_token
@@ -207,14 +216,20 @@ module Katello
           tasks
         end
 
-        def copy_all(source_repository)
-          data = PulpRpmClient::Copy.new
-          data.config = [{
-            source_repo_version: source_repository.version_href,
-            dest_repo: repository_reference.repository_href
-          }]
+        def copy_all(source_repository, mirror: false)
+          if mirror
+            data = PulpRpmClient::RepositoryAddRemoveContent.new(
+                      base_version: source_repository.version_href)
 
-          [api.copy_api.copy_content(data)]
+            [api.repositories_api.modify(repository_reference.repository_href, data)]
+          else
+            data = PulpRpmClient::Copy.new
+            data.config = [{
+              source_repo_version: source_repository.version_href,
+              dest_repo: repository_reference.repository_href
+            }]
+            [api.copy_api.copy_content(data)]
+          end
         end
 
         def remove_all_content_from_repo(repo_href)

data/app/services/katello/pulp3/rpm.rb

@@ -111,6 +111,10 @@ module Katello
         backend_data['rpm_license']
       end
 
+      def parse_filename(path)
+        File.split(path).last unless path.blank?
+      end
+
       def update_model(model)
         custom_json = {}
         custom_json['modular'] = backend_data['is_modular']
@@ -119,7 +123,7 @@ module Katello
           each { |field| custom_json[field] = backend_data[field] }
         custom_json['release_sortable'] = Util::Package.sortable_version(backend_data['release'])
         custom_json['version_sortable'] = Util::Package.sortable_version(backend_data['version'])
-        custom_json['filename'] = backend_data['location_href']
+        custom_json['filename'] = parse_filename(backend_data['location_href']) #location_href is the relative path of the rpm in the upstream repo
         custom_json['checksum'] = backend_data['pkgId']
         custom_json['sourcerpm'] = backend_data['rpm_sourcerpm']
         model.assign_attributes(custom_json)

data/app/services/katello/pulp3/task.rb

@@ -68,6 +68,10 @@ module Katello
         task_data[:finish_time] || FINISHED_STATES.include?(task_data[:state])
       end
 
+      def progress_reports
+        task_data['progress_reports']
+      end
+
       def poll
         task_data(true)
         self

data/app/services/katello/pulp3/task_group.rb

@@ -53,6 +53,10 @@ module Katello
         task_group_data['all_tasks_dispatched'] == true && IN_PROGRESS_STATES.all? { |state| task_group_data[state] == 0 }
       end
 
+      def group_progress_reports
+        task_group_data['group_progress_reports']
+      end
+
       def poll
         clear_task_group_data
         task_group_data

data/app/views/katello/api/v2/content_views/show.json.rabl

@@ -4,6 +4,12 @@ extends "katello/api/v2/content_views/base"
 
 attributes :content_host_count
 
+node :errors do
+  unless @resource.valid?
+    attribute :messages => @resource.errors.full_messages
+  end
+end
+
 child :duplicate_repositories_to_publish => :duplicate_repositories_to_publish do
   attributes :id, :name
   node :components do |repo|