katello 3.17.0.rc1 → 3.17.0.rc2

data/app/services/katello/pulp3/file_unit.rb

@@ -5,7 +5,7 @@ module Katello
       CONTENT_TYPE = "file".freeze
 
       def self.content_api
-        PulpFileClient::ContentFilesApi.new(Katello::Pulp3::Api::File.new(SmartProxy.pulp_master!).api_client)
+        PulpFileClient::ContentFilesApi.new(Katello::Pulp3::Api::File.new(SmartProxy.pulp_primary!).api_client)
       end
 
       def self.create_content(options)
@@ -14,7 +14,7 @@ module Katello
       end
 
       def self.ids_for_repository(repo_id)
-        repo = Katello::Pulp3::Repository::File.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
+        repo = Katello::Pulp3::Repository::File.new(Katello::Repository.find(repo_id), SmartProxy.pulp_primary)
         repo_content_list = repo.content_list
         repo_content_list.map { |content| content.try(:pulp_href) }
       end

data/app/services/katello/pulp3/migration.rb

@@ -3,20 +3,27 @@ require 'pulp_2to3_migration_client'
 module Katello
   module Pulp3
     class Migration
-      attr_accessor :smart_proxy
+      attr_accessor :smart_proxy, :reimport_all
       GET_QUERY_ID_LENGTH = 90
 
-      REPOSITORY_TYPES = [
-        Katello::Repository::FILE_TYPE,
-        Katello::Repository::DOCKER_TYPE
-      ].freeze
-
       MUTABLE_CONTENT_TYPES = [
         Katello::DockerTag,
         Katello::Erratum
       ].freeze
 
-      def initialize(smart_proxy, repository_types = REPOSITORY_TYPES)
+      UNIFIED_CONTENT_TYPES = [
+        Katello::Erratum
+      ].freeze
+
+      def self.repository_types_for_migration
+        #we can migrate types that pulp3 supports, but are overridden to pulp2.  These are in 'migration mode'
+        overridden = (SETTINGS[:katello][:use_pulp_2_for_content_type] || {}).keys.select { |key| SETTINGS[:katello][:use_pulp_2_for_content_type][key] }
+        overridden.select { |type| SmartProxy.pulp_primary.pulp3_repository_type_support?(type.to_s, false) }.map { |t| t.to_s }
+      end
+
+      def initialize(smart_proxy, repository_types = Migration.repository_types_for_migration, options = {})
+        self.reimport_all = options.fetch(:reimport, false)
+
         if (repository_types - smart_proxy.supported_pulp_types[:pulp3][:overriden_to_pulp2]).any?
           fail ::Katello::Errors::Pulp3MigrationError, _("Pulp 3 migration cannot run. Types %s have already been migrated.") %
             (repository_types - smart_proxy.supported_pulp_types[:pulp3][:overriden_to_pulp2]).join(', ')
@@ -47,7 +54,12 @@ module Katello
       end
 
       def create_and_run_migrations
-        create_migrations.map { |href| start_migration(href) }
+        migs = create_migrations
+        migs.map { |href| start_migration(href) }
+      end
+
+      def self.ignorable_content_types
+        [YumMetadataFile]
       end
 
       def content_types_for_migration
@@ -55,35 +67,79 @@ module Katello
           Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index
         end
 
-        content_types.flatten
+        content_types.flatten - Migration.ignorable_content_types
       end
 
       def import_pulp3_content
-        @repository_types.each do |repository_type_label|
-          Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
-            import_content_type(content_type)
+        Katello::Logging.time("CONTENT_MIGRATION - Total Import Process") do
+          @repository_types.each do |repository_type_label|
+            Katello::Logging.time("CONTENT_MIGRATION - Importing Repository", data: {type: repository_type_label}) do
+              import_repositories(repository_type_label)
+            end
+
+            Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
+              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type}) do
+                import_content_type(content_type)
+              end
+            end
           end
-          import_repositories(repository_type_label)
         end
       end
 
+      def migration_plan
+        Katello::Pulp3::MigrationPlan.new(@repository_types).generate.as_json
+      end
+
       def create_migrations
-        plan = Katello::Pulp3::MigrationPlan.new(@repository_types)
-        Rails.logger.info(plan)
-        [migration_plan_api.create(plan: plan.generate.as_json).pulp_href]
+        plan = migration_plan
+        Rails.logger.info("Migration Plan: #{plan}")
+
+        if plan['plugins'].empty?
+          Rails.logger.error("No Repositories to migrate")
+          []
+        else
+          [migration_plan_api.create(plan: plan).pulp_href]
+        end
       end
 
       def start_migration(plan_href)
-        migration_plan_api.run(plan_href, dry_run: false)
+        migration_plan_api.run(plan_href, dry_run: false, validate: true)
       end
 
       def import_repositories(repository_type_label)
         imported = Katello::Pulp3::Api::Core.fetch_from_list { |opts| pulp2_repositories_api.list(opts) }
         imported = imported.select { |migrated| !migrated.not_in_plan }
+        katello_repos = Katello::Repository.with_type(repository_type_label)
+
+        if repository_type_label == 'yum'
+          import_yum_repos(imported, katello_repos)
+        else
+          katello_repos.each do |repo|
+            found = imported.find { |migrated_repo| migrated_repo.pulp2_repo_id == repo.pulp_id }
+            import_repo(repo, found) if found
+          end
+        end
+      end
 
-        Katello::Repository.with_type(repository_type_label).each do |repo|
-          found = imported.find { |migrated_repo| migrated_repo.pulp2_repo_id == repo.pulp_id }
-          import_repo(repo, found) if found
+      def import_yum_repos(migrated_repo_items, repos)
+        repos.each do |yum_repo|
+          to_find = nil
+          if yum_repo.content_view.composite?
+            if yum_repo.link?
+              to_find = yum_repo.target_repository
+            else
+              to_find = yum_repo
+            end
+          elsif yum_repo.environment_id.nil? || yum_repo.in_default_view? #non-composite archive repo or default content view repo
+            to_find = yum_repo
+          else #non-composite env-repo
+            to_find = yum_repo.archived_instance
+          end
+
+          if to_find
+            found = migrated_repo_items.find { |migrated_repo| migrated_repo.pulp2_repo_id == to_find.pulp_id }
+            import_repo(yum_repo, found)
+          end
         end
       end
 
@@ -92,7 +148,7 @@ module Katello
       end
 
       def import_repo(katello_repo, migrated_repo)
-        pulp3_api = api_for_repository(katello_repo).new(SmartProxy.pulp_master)
+        pulp3_api = api_for_repository(katello_repo).new(SmartProxy.pulp_primary)
         katello_repo.remote_href = migrated_repo.pulp3_remote_href
         katello_repo.publication_href = migrated_repo.pulp3_publication_href
         katello_repo.version_href = migrated_repo.pulp3_repository_version
@@ -105,9 +161,7 @@ module Katello
         end
 
         repo_ref = Katello::Pulp3::RepositoryReference.find_or_initialize_by(:root_repository_id => katello_repo.root_id, :content_view_id => katello_repo.content_view.id)
-        repo_ref.repository_href = migrated_repo.pulp3_repository_href
-        repo_ref.save!
-
+        repo_ref.update!(repository_href: migrated_repo.pulp3_repository_href) if repo_ref.repository_href != migrated_repo.pulp3_repository_href
         process_distributions(pulp3_api, migrated_repo.pulp3_distribution_hrefs)
       end
 
@@ -125,18 +179,69 @@ module Katello
         end
       end
 
-      def import_content_type(content_type)
-        unmigrated_units = content_type.model_class
-        #mutable content types have to be completely re-indexed every time
-        unless MUTABLE_CONTENT_TYPES.include?(content_type.model_class)
-          unmigrated_units = unmigrated_units.where(:migrated_pulp3_href => nil)
+      def generate_repo_version_map
+        map = {}
+        Katello::Repository.select(:id, :version_href).each do |repo|
+          map[repo.version_href] ||= []
+          map[repo.version_href] << repo.id
+        end
+        map
+      end
+
+      def operate_on_errata
+        offset = 0
+        limit = 300
+        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+        total_count = response.count
+        yield(response.results)
+        until (offset + limit > total_count)
+          offset += limit
+          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+          yield(response.results)
+        end
+      end
+
+      def import_errata
+        to_import = {}
+
+        repo_version_map = generate_repo_version_map
+
+        operate_on_errata do |migrated_list|
+          katello_errata = Katello::Erratum.where(:pulp_id => migrated_list.map(&:pulp2_id).uniq)
+          migrated_list.each do |migrated_unit|
+            pulp3_href = migrated_unit.pulp3_content
+            errata_id = katello_errata.find { |erratum| erratum.pulp_id == migrated_unit.pulp2_id }&.id
+            next if errata_id.nil?
+            repo_ids = repo_version_map[migrated_unit.pulp3_repository_version]
+            #currently pulp can have duplicates, so de-duplicate with a hash
+            repo_ids&.each do |repo_id|
+              #currently pulp can have duplicates, so de-duplicate with a hash
+              to_import[[errata_id, repo_id]] ||= {erratum_id: errata_id, erratum_pulp3_href: pulp3_href, repository_id: repo_id}
+            end
+          end
         end
-        unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
-          migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
-          migrated_units.results.each do |migrated_unit|
-            matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }
 
-            matching_record&.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content)
+        Katello::RepositoryErratum.import([:erratum_id, :erratum_pulp3_href, :repository_id], to_import.values, :validate => false,
+                                          on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
+      end
+
+      def import_content_type(content_type)
+        if content_type.model_class == Katello::Erratum
+          import_errata
+        else
+          unmigrated_units = content_type.model_class
+
+          #mutable content types have to be completely re-indexed every time
+          if !MUTABLE_CONTENT_TYPES.include?(content_type.model_class) && !self.reimport_all
+            unmigrated_units = unmigrated_units.where(:migrated_pulp3_href => nil)
+          end
+
+          unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
+            migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
+            migrated_units.results.each do |migrated_unit|
+              matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }
+              matching_record&.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content)
+            end
           end
         end
       end

data/app/services/katello/pulp3/migration_plan.rb

@@ -5,19 +5,21 @@ module Katello
         @repository_types = repository_type_labels
       end
 
-      def master_proxy
-        SmartProxy.pulp_master!
+      def primary_proxy
+        SmartProxy.pulp_primary!
       end
 
       def generate
         plan = {}
-        plan[:plugins] = generate_plugins
+        Katello::Logging.time("CONTENT_MIGRATION - Generating Migration plan") do
+          plan[:plugins] = generate_plugins
+        end
         Rails.logger.error("Migration Plan: #{plan.to_json}")
         plan
       end
 
       def generate_plugins
-        @repository_types.map do |repository_type|
+        @repository_types.sort.map do |repository_type|
           {
             type: pulp2_repository_type(repository_type),
             repositories: repository_migrations(repository_type)
@@ -26,17 +28,21 @@ module Katello
       end
 
       def pulp2_repository_type(repository_type)
-        Katello::RepositoryTypeManager.repository_types[repository_type].service_class::REPOSITORY_TYPE
+        if repository_type == 'yum'
+          return 'rpm' #migration plugin uses rpm
+        else
+          Katello::RepositoryTypeManager.repository_types[repository_type].service_class::REPOSITORY_TYPE
+        end
       end
 
       def repository_migrations(repo_type)
-        roots = Katello::RootRepository.where(:content_type => repo_type)
+        roots = Katello::RootRepository.where(:content_type => repo_type).order(:label)
         plans = []
         roots.each do |root|
           plans << library_migration_for(root)
           plans += content_view_migrations_for(root)
         end
-        plans
+        plans.compact.sort_by { |plan| plan[:name] }
       end
 
       # plugins: [
@@ -73,28 +79,58 @@ module Katello
 
       def content_view_migrations_for(root)
         plans = []
-        ContentView.non_default.published_with_repositories(root).each do |cv|
+        ContentView.non_default.published_with_repositories(root).sort_by(&:label).each do |cv|
           plans << content_view_migration(cv, root)
         end
         plans
       end
 
+      #since we don't have an easy to look up all 'link' repos for a target, lets
+      # pre-generate a map of all target_repo => link_repos
+      # This is a fairly inefficient way to do this, but its quite a complex calculation to use AR
+      def target_link_map
+        return @target_link_map if @target_link_map
+
+        map = {}
+        composites_with_repo = ::Katello::ContentViewVersion.where(:content_view_id => ContentView.composite)
+        env_repos = Katello::Repository.yum_type.where(:content_view_version_id => composites_with_repo)
+        env_repos.each do |link_repo|
+          if link_repo.link?
+            target = link_repo.target_repository
+            map[target] ||= []
+            map[target] << link_repo
+          end
+        end
+        @target_link_map = map
+      end
+
+      def name_for_content_view(content_view, root_repo)
+        "#{content_view.label}-#{root_repo.label}"
+      end
+
       def content_view_migration(content_view, root)
         plan = {
-          name: "#{content_view.name}-#{root.id}-#{content_view.id}",
+          name: name_for_content_view(content_view, root),
           repository_versions: []
         }
         library_instance = root.library_instance
         content_view.versions.with_library_repo(library_instance).uniq.each do |version|
-          repo = version.archived_repos.find_by(:library_instance_id => library_instance.id)
-          env_repos = version.repositories.where(:library_instance_id => library_instance.id).where.not(:environment_id => nil)
+          archived_repo = version.archived_repos.find_by(:library_instance_id => library_instance.id)
+          next if archived_repo.link?
+
+          env_repos = version.repositories.where(:library_instance_id => library_instance.id)
+          env_repos += target_link_map[archived_repo] if target_link_map[archived_repo]
+
           plan[:repository_versions] << {
-            pulp2_repository_id: repo.pulp_id,
-            pulp2_distributor_repository_ids: env_repos.map(&:pulp_id)
+            pulp2_repository_id: archived_repo.pulp_id,
+            pulp2_distributor_repository_ids: env_repos.map(&:pulp_id).uniq.sort
           }
         end
 
-        plan
+        if plan[:repository_versions].any?
+          plan[:repository_versions].sort_by! { |v| v[:pulp2_repository_id] }
+          plan
+        end
       end
     end
   end

data/app/services/katello/pulp3/migration_switchover.rb

@@ -2,7 +2,7 @@ require 'pulp_2to3_migration_client'
 
 module Katello
   module Pulp3
-    class SwitchoverError < StandardError; end
+    class SwitchOverError < StandardError; end
 
     class MigrationSwitchover
       def initialize(*argv)
@@ -14,11 +14,19 @@ module Katello
       end
 
       def run
-        check_already_migrated_content
-        cleanup_v1_docker_tags if docker_migration?
-        migrated_content_type_check
-        combine_duplicate_docker_tags if docker_migration?
-        migrate_pulp3_hrefs
+        Katello::Logging.time("CONTENT_SWITCHOVER - Total Switchover Process") do
+          Katello::Logging.time("CONTENT_SWITCHOVER - check_already_migrated_content") { check_already_migrated_content }
+          Katello::Logging.time("CONTENT_SWITCHOVER - cleanup_v1_docker_tags") { cleanup_v1_docker_tags } if docker_migration?
+          Katello::Logging.time("CONTENT_SWITCHOVER - migrated_content_type_check") { migrated_content_type_check }
+          Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_content_types") { combine_duplicate_content_types }
+          Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_docker_tags") { combine_duplicate_docker_tags } if docker_migration?
+          Katello::Logging.time("CONTENT_SWITCHOVER - migrate_pulp3_hrefs") { migrate_pulp3_hrefs }
+        end
+      end
+
+      def deduplicated_content_types
+        #even though YumMetatadataFile is de-depulicated, we're not indexing it in pulp3
+        [Katello::PackageGroup]
       end
 
       def docker_migration?
@@ -34,7 +42,7 @@ module Katello
       end
 
       def check_already_migrated_content
-        content_types.each do |content_type|
+        (content_types - Migration.ignorable_content_types).each do |content_type|
           if content_type.model_class.where("pulp_id=migrated_pulp3_href").any?
             Rails.logger.error("Content Switchover: #{content_type.label} seems to have already migrated content, switchover may fail.  Did you already perform the switchover?")
           end
@@ -60,6 +68,22 @@ module Katello
         Katello::DockerMetaTag.cleanup_tags
       end
 
+      def combine_duplicate_content_types
+        deduplicated_content_types.each do |content_class|
+          to_delete = []
+          content_class.having("count(migrated_pulp3_href) > 1").group(:migrated_pulp3_href).pluck(:migrated_pulp3_href).each do |duplicate_href|
+            units = content_class.where(:migrated_pulp3_href => duplicate_href).to_a
+            main_unit = units.pop
+            content_class.repository_association_class.where(content_class.unit_id_field => units.map(&:id)).update_all(content_class.unit_id_field => main_unit.id)
+            to_delete += units.map(&:id)
+          end
+
+          to_delete.each_slice(10_000) do |group|
+            content_class.where(:id => group).delete_all
+          end
+        end
+      end
+
       def combine_duplicate_docker_tags
         to_delete = []
         Katello::DockerTag.having("count(migrated_pulp3_href) > 1").group(:migrated_pulp3_href).pluck(:migrated_pulp3_href).each do |duplicate_href|
@@ -84,11 +108,19 @@ module Katello
 
       def migrated_content_type_check
         content_types.each do |content_type|
-          if content_type.model_class.where(migrated_pulp3_href: nil).any?
-            fail SwitchoverError, "ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n"
+          if content_type.model_class == Katello::Erratum
+            migrated_errata_check
+          elsif content_type.model_class.where(migrated_pulp3_href: nil).any?
+            fail SwitchOverError, "ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n"
           end
         end
       end
+
+      def migrated_errata_check
+        if Katello::RepositoryErratum.where(:erratum_pulp3_href => nil).any?
+          fail SwitchOverError, "ERROR: at least one Erratum record has migrated_pulp3_href NULL value\n"
+        end
+      end
     end
   end
 end