katello 3.18.0.rc2 → 3.18.2

data/app/models/katello/subscription_status.rb

@@ -43,9 +43,10 @@ module Katello
 
     def to_status(options = {})
       return UNKNOWN unless host.subscription_facet.try(:uuid)
-      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.hypervisor && !host.subscription_facet.candlepin_consumer.entitlements?
+      return DISABLED if host.organization.simple_content_access?
+      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.unsubscribed_hypervisor?
       status_override ||= options.fetch(:status_override, nil)
-      status = status_override || Katello::Candlepin::Consumer.new(host.subscription_facet.uuid, host.organization.label).entitlement_status
+      status = status_override || host.subscription_facet.candlepin_consumer.entitlement_status
 
       case status
       when Katello::Candlepin::Consumer::ENTITLEMENTS_DISABLED

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -60,17 +60,7 @@ module Katello
       end
 
       def fetch_rpm_content_ids
-        # Query for applicable RPM ids
-        # -> Include all non-modular rpms or rpms that exist within installed module streams
-        enabled_module_stream_ids = ::Katello::ModuleStream.
-          joins("inner join katello_available_module_streams on
-            katello_module_streams.name = katello_available_module_streams.name and
-            katello_module_streams.stream = katello_available_module_streams.stream").
-          joins("inner join katello_host_available_module_streams on
-            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
-          where("katello_host_available_module_streams.host_id = :content_facet_id and
-            katello_host_available_module_streams.status = 'enabled'",
-            :content_facet_id => self.content_facet.host.id).select(:id)
+        enabled_module_stream_ids = fetch_enabled_module_stream_ids
 
         ::Katello::Rpm.
           joins("INNER JOIN katello_repository_rpms ON
@@ -78,7 +68,8 @@ module Katello
           joins("INNER JOIN katello_installed_packages ON
             katello_rpms.name = katello_installed_packages.name AND
             katello_rpms.arch = katello_installed_packages.arch AND
-            katello_rpms.evr > katello_installed_packages.evr").
+            katello_rpms.evr > katello_installed_packages.evr AND
+            katello_installed_packages.id in (#{newest_distinct_installed_packages_query})").
           joins("LEFT JOIN katello_module_stream_rpms ON
             katello_rpms.id = katello_module_stream_rpms.rpm_id").
           joins("INNER JOIN katello_host_installed_packages ON
@@ -87,9 +78,47 @@ module Katello
             :bound_library_repos => self.bound_library_instance_repos).
           where("katello_host_installed_packages.host_id = :content_facet_id",
             :content_facet_id => self.content_facet.host.id).
-          where("katello_module_stream_rpms.module_stream_id is null or
-            katello_module_stream_rpms.module_stream_id in (:enabled_module_streams)",
-            :enabled_module_streams => enabled_module_stream_ids).pluck(:id).uniq
+          where("(katello_module_stream_rpms.module_stream_id IS NULL AND
+            katello_installed_packages.id NOT IN (:locked_modular_installed_packages)) OR
+            (katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)
+            AND katello_installed_packages.id IN (:locked_modular_installed_packages))",
+            :enabled_module_streams => enabled_module_stream_ids,
+            :locked_modular_installed_packages => locked_modular_installed_packages(enabled_module_stream_ids)).pluck(:id).uniq
+      end
+
+      def fetch_enabled_module_stream_ids
+        # Query for applicable RPM ids
+        # -> Include all non-modular rpms or rpms that exist within installed module streams
+        ::Katello::ModuleStream.
+          joins("inner join katello_available_module_streams on
+            katello_module_streams.name = katello_available_module_streams.name and
+            katello_module_streams.stream = katello_available_module_streams.stream").
+          joins("inner join katello_host_available_module_streams on
+            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
+          where("katello_host_available_module_streams.host_id = :content_facet_id and
+            katello_host_available_module_streams.status = 'enabled'",
+            :content_facet_id => self.content_facet.host.id).select(:id)
+      end
+
+      # Installed packages that are locked for the host due to enabled module stream membership
+      def locked_modular_installed_packages(enabled_module_streams)
+        rpms_in_enabled_module_streams = ::Katello::Rpm.
+          joins("INNER JOIN katello_module_stream_rpms ON katello_rpms.id = katello_module_stream_rpms.rpm_id").
+          where("katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)",
+                :enabled_module_streams => enabled_module_streams).select(:nvra, :epoch)
+
+        ::Katello::InstalledPackage.where(nvra: rpms_in_enabled_module_streams.map(&:nvra),
+                                          epoch: rpms_in_enabled_module_streams.map(&:epoch)).select(:id)
+      end
+
+      def newest_distinct_installed_packages_query
+        "SELECT DISTINCT ON (katello_installed_packages.name) katello_installed_packages.id " \
+          "FROM katello_installed_packages INNER JOIN " \
+          "katello_host_installed_packages ON " \
+          "katello_installed_packages.id = " \
+          "katello_host_installed_packages.installed_package_id " \
+          "WHERE katello_host_installed_packages.host_id = " \
+          "#{content_facet.host.id} ORDER BY katello_installed_packages.name, katello_installed_packages.evr DESC"
       end
 
       def applicable_differences

data/app/services/katello/pulp3/api/docker.rb

@@ -32,6 +32,10 @@ module Katello
           PulpContainerClient::RecursiveManage
         end
 
+        def self.tag_image_class
+          PulpContainerClient::TagImage
+        end
+
         def api_client
           PulpContainerClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpContainerClient::Configuration))
         end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -4,13 +4,37 @@ module Katello
       class Export
         include ImportExportCommon
 
-        def initialize(smart_proxy:, content_view_version: nil, destination_server: nil, from_content_view_version: nil)
+        def initialize(smart_proxy:,
+                        content_view_version: nil,
+                        destination_server: nil,
+                        from_content_view_version: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
           @from_content_view_version = from_content_view_version
         end
 
+        def repository_hrefs
+          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
+        end
+
+        def version_hrefs
+          repositories.pluck(:version_href).compact
+        end
+
+        def repositories(fetch_all: false)
+          repos = if @content_view_version.default?
+                    @content_view_version.repositories.yum_type
+                  else
+                    @content_view_version.archived_repos.yum_type
+                  end
+          if fetch_all
+            repos
+          else
+            repos.immediate
+          end
+        end
+
         def generate_exporter_path
           export_path = "#{@content_view_version.content_view}/#{@content_view_version.version}/#{@destination_server}/#{date_dir}".gsub(/\s/, '_')
           "#{@content_view_version.organization.label}/#{export_path}"
@@ -66,12 +90,29 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
+        def validate!(fail_on_missing_content: true, validate_incremental: true)
+          validate_repositories_immediate! if fail_on_missing_content
+          validate_incremental_export! if validate_incremental && !@from_content_view_version.blank?
+        end
+
+        def validate_repositories_immediate!
+          non_immediate_repos = repositories(fetch_all: true).non_immediate
+          if non_immediate_repos.any?
+            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories without the 'immediate' download policy."\
+                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
+                   " and export the generated version. \n %{repos}" %
+                   { content_view: @content_view_version.content_view.name,
+                     current: @content_view_version.version,
+                     repos: self.class.generate_product_repo_strings(repositories: non_immediate_repos)})
+          end
+        end
+
         def validate_incremental_export!
-          return if @from_content_view_version.blank?
           from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
 
-          from_exporter_repos = generate_repo_mapping(from_exporter.repositories)
-          to_exporter_repos = generate_repo_mapping(repositories)
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
+          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
 
           invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
             from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
@@ -99,7 +140,8 @@ module Katello
           ret = { organization: @content_view_version.organization.name,
                   repository_mapping: {},
                   content_view: @content_view_version.content_view.name,
-                  content_view_version: @content_view_version.slice(:major, :minor)
+                  content_view_version: @content_view_version.slice(:major, :minor),
+                  incremental: @from_content_view_version.present?
           }
 
           unless @from_content_view_version.blank?
@@ -120,6 +162,22 @@ module Katello
           end
           ret
         end
+
+        def self.find_library_export_view(create_by_default: false,
+                                          destination_server:,
+                                          organization:)
+          name = "Export-Library"
+          name += "-#{destination_server}" unless destination_server.blank?
+          select_method = create_by_default ? :first_or_create : :first
+          ::Katello::ContentView.where(name: name, organization: organization).send(select_method)
+        end
+
+        def self.generate_product_repo_strings(repositories:)
+          repositories.map do |repo|
+            _("Product: '%{product}', Repository: '%{repository}'" % { product: repo.product.name,
+                                                                       repository: repo.name})
+          end
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -46,6 +46,46 @@ module Katello
         def self.check!(content_view:, metadata:, path:)
           ImportValidator.new(content_view: content_view, metadata: metadata, path: path).check!
         end
+
+        def self.reset_content_view_repositories_from_metadata!(content_view:, metadata:)
+          # Given metadata from the dump and a content view
+          # this method
+          # 1) Fetches ids of the library repos whose product name, repo name amd redhat?
+          # =>  match values provided in the metadata's repository mapping
+          # 2) Removes all the repositories associated to this content view
+          # 3) Adds the repositories matched from the dump
+          # The main intent of this method is to assume that the user intends for the
+          # content view to exaclty look like what is specified in metadata
+
+          repos_in_library = Katello::Repository.
+                    in_default_view.
+                    yum_type.
+                    joins(:product => :provider, :content_view_version => :content_view).
+                    joins(:root).
+                    where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                    pluck("#{::Katello::Repository.table_name}.id",
+                          "#{::Katello::RootRepository.table_name}.name",
+                          "#{::Katello::Product.table_name}.name",
+                          "#{::Katello::Provider.table_name}.provider_type"
+                          )
+          repos_in_library_map = {}
+          # repos_in_library_map is going to look like {['repo1', 'product1', false] => 100, ['repo1', 'product1', true] => 200 }
+          repos_in_library.each do |id, repo, product, provider_type|
+            repos_in_library_map[[repo, product, provider_type == Katello::Provider::REDHAT]] = id
+          end
+
+          repo_ids = metadata[:repository_mapping].values.map do |repo|
+            repos_in_library_map[[repo[:repository], repo[:product], repo[:redhat]]]
+          end
+          content_view.update!(repository_ids: repo_ids)
+        end
+
+        def self.find_or_create_library_import_view(organization)
+          name = ::Katello::ContentView::IMPORT_LIBRARY
+          ::Katello::ContentView.where(name: name,
+                                       organization: organization,
+                                       import_only: true).first_or_create
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -19,22 +19,6 @@ module Katello
           repo_api.read(version_href_to_repository_href(version_href))
         end
 
-        def repository_hrefs
-          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
-        end
-
-        def version_hrefs
-          repositories.pluck(:version_href).compact
-        end
-
-        def repositories
-          if @content_view_version.default?
-            @content_view_version.repositories.yum_type
-          else
-            @content_view_version.archived_repos.yum_type
-          end
-        end
-
         def version_href_to_repository_href(version_href)
           version_href.split("/")[0..-3].join("/") + "/"
         end

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -2,7 +2,6 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class ImportValidator
-        BASEDIR = '/var/lib/pulp'.freeze
         attr_accessor :metadata, :path, :content_view
         def initialize(content_view:, path:, metadata:)
           self.content_view = content_view
@@ -11,10 +10,11 @@ module Katello
         end
 
         def check!
-          check_permissions!
-          ensure_importing_cvv_does_not_exist!
-          ensure_from_cvv_exists!
-          ensure_repositories_metadata_and_content_view_match!
+          unless content_view.default?
+            ensure_importing_cvv_does_not_exist!
+            ensure_from_cvv_exists!
+          end
+          ensure_repositories_metadata_are_in_the_library!
         end
 
         def ensure_importing_cvv_does_not_exist!
@@ -43,55 +43,32 @@ module Katello
           end
         end
 
-        def ensure_repositories_metadata_and_content_view_match!
-          product_repos_in_content_view = content_view.repositories.yum_type.map { |repo| [repo.product.name, repo.name, repo.redhat?] }
+        def ensure_repositories_metadata_are_in_the_library!
+          repos_in_library = Katello::Repository.
+                              in_default_view.
+                              yum_type.
+                              joins(:product => :provider, :content_view_version => :content_view).
+                              joins(:root).
+                              where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                              pluck("#{::Katello::Product.table_name}.name",
+                                    "#{::Katello::RootRepository.table_name}.name",
+                                    "#{::Katello::Provider.table_name}.provider_type"
+                                    )
+
+          # repos_in_library look like [["prod1", "repo1", "Anonymous"], ["prod2", "repo2", "Red Hat"]]
+          product_repos_in_library = repos_in_library.map { |product, repo, provider| [product, repo, provider == ::Katello::Provider::REDHAT] }
           product_repos_in_metadata = metadata[:repository_mapping].values.map { |repo| [repo[:product], repo[:repository], repo[:redhat]] }
-
-          product_repos_in_content_view.sort!
-          product_repos_in_metadata.sort!
-          # product_repos_in_content_view & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
-
-          if product_repos_in_content_view != product_repos_in_metadata
-            repos_in_content_view = generate_product_repo_i18n_string(product_repos_in_content_view)
-            repos_in_import = generate_product_repo_i18n_string(product_repos_in_metadata)
-
-            fail _("Repositories in the importing content view do not match the repositories provided in the import metadata.\n "\
-                    "Repositories in Content View '%{content_view}': %{repos_in_content_view}\n "\
-                    "Repositories in the Import Metadata: %{repos_in_import}" % { content_view: content_view.name,
-                                                                                  repos_in_content_view: repos_in_content_view.join(""),
-                                                                                  repos_in_import: repos_in_import.join("")}
+          # product_repos_in_library & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos_not_in_library = product_repos_in_metadata - product_repos_in_library
+          unless product_repos_not_in_library.blank?
+            repos_in_import = generate_product_repo_i18n_string(product_repos_not_in_library)
+            fail _("The following repositories provided in the import metadata are either not available in the Library or are of incorrect Respository Type. "\
+                    "Please add or enable the repositories before importing\n "\
+                    "%{repos}" % { content_view: content_view.name, repos: repos_in_import.join("")}
                   )
           end
         end
 
-        def check_permissions!
-          fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
-          fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
-          fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
-
-          Dir.glob("#{path}/*").each do |file|
-            fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
-          end
-          toc_path = "#{path}/#{metadata[:toc]}"
-          fail _("The TOC file specified in the metadata does not exist. %s " % toc_path) unless File.exist?(toc_path)
-        end
-
-        def pulp_user_accessible?(path)
-          pulp_info = fetch_pulp_user_info
-          return false if pulp_info.blank?
-
-          stat = File.stat(path)
-          stat.gid.to_s == pulp_info.gid ||
-            stat.uid.to_s == pulp_info.uid ||
-            stat.mode.to_s(8)[-1].to_i >= 4
-        end
-
-        def fetch_pulp_user_info
-          pulp_user = nil
-          Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
-          pulp_user
-        end
-
         def generate_product_repo_i18n_string(product_repos)
           # product_repos look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
           product_repos.map do |product, repo, redhat|

data/app/services/katello/pulp3/docker_manifest.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerManifest < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_manifest".freeze
 
       def self.content_api
         PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/docker_tag.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerTag < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_tag".freeze
 
       def self.content_api
         PulpContainerClient::ContentTagsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/erratum.rb

@@ -35,7 +35,8 @@ module Katello
         custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])
         custom_json["updated"] = convert_date_if_epoch(custom_json["updated"]) unless custom_json["updated"].blank?
 
-        if model.updated.blank? || (custom_json['updated'].to_datetime != model.updated.to_datetime)
+        if model.updated.blank? ||
+            (custom_json['updated'] && (custom_json['updated'].to_datetime != model.updated.to_datetime))
           custom_json['errata_id'] = custom_json.delete('id')
           custom_json['errata_type'] = custom_json.delete('type')
           custom_json['updated'] = custom_json['updated'].blank? ? custom_json['issued'] : custom_json['updated']

data/app/services/katello/pulp3/migration.rb

@@ -3,7 +3,7 @@ require 'pulp_2to3_migration_client'
 module Katello
   module Pulp3
     class Migration
-      attr_accessor :smart_proxy, :reimport_all
+      attr_accessor :smart_proxy, :reimport_all, :task_id
       GET_QUERY_ID_LENGTH = 90
 
       MUTABLE_CONTENT_TYPES = [
@@ -15,14 +15,20 @@ module Katello
         Katello::Erratum
       ].freeze
 
+      CORRUPTABLE_CONTENT_TYPES = [
+        Katello::Rpm, Katello::FileUnit
+      ].freeze
+
       def self.repository_types_for_migration
         #we can migrate types that pulp3 supports, but are overridden to pulp2.  These are in 'migration mode'
         overridden = (SETTINGS[:katello][:use_pulp_2_for_content_type] || {}).keys.select { |key| SETTINGS[:katello][:use_pulp_2_for_content_type][key] }
         overridden.select { |type| SmartProxy.pulp_primary.pulp3_repository_type_support?(type.to_s, false) }.map { |t| t.to_s }
       end
 
-      def initialize(smart_proxy, repository_types = Migration.repository_types_for_migration, options = {})
-        self.reimport_all = options.fetch(:reimport, false)
+      def initialize(smart_proxy, options = {})
+        self.task_id = options.fetch(:task_id, nil)
+        self.reimport_all = options.fetch(:reimport_all, false)
+        repository_types = options.fetch(:repository_types, Migration.repository_types_for_migration)
 
         if (repository_types - smart_proxy.supported_pulp_types[:pulp3][:overriden_to_pulp2]).any?
           fail ::Katello::Errors::Pulp3MigrationError, _("Pulp 3 migration cannot run. Types %s have already been migrated.") %
@@ -62,6 +68,15 @@ module Katello
         [YumMetadataFile]
       end
 
+      def last_successful_migration_time
+        task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigration.to_s, :result => 'success').order("started_at desc").first
+        if reimport_all || task.nil?
+          0
+        else
+          task.started_at.to_i
+        end
+      end
+
       def content_types_for_migration
         content_types = @repository_types.collect do |repository_type_label|
           Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index
@@ -70,7 +85,20 @@ module Katello
         content_types.flatten - Migration.ignorable_content_types
       end
 
+      def update_import_status(message, index = nil)
+        #reduce output updating, only update every 20 items
+        if (index.nil? || index % 20 == 0) && self.task_id
+          progress = Katello::ContentMigrationProgress.find_or_create_by(:task_id => self.task_id)
+          progress.update(:progress_message => message)
+          progress.save!
+
+          fail Katello::Errors::Pulp3MigrationError, "Cancelled by user." if progress.canceled?
+        end
+      end
+
       def import_pulp3_content
+        update_import_status("Starting katello import phase.")
+
         Katello::Logging.time("CONTENT_MIGRATION - Total Import Process") do
           @repository_types.each do |repository_type_label|
             Katello::Logging.time("CONTENT_MIGRATION - Importing Repository", data: {type: repository_type_label}) do
@@ -78,8 +106,9 @@ module Katello
             end
 
             Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
-              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type}) do
+              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type.label}) do
                 import_content_type(content_type)
+                mark_missing_content(content_type)
               end
             end
           end
@@ -90,6 +119,44 @@ module Katello
         Katello::Pulp3::MigrationPlan.new(@repository_types).generate.as_json
       end
 
+      def reset
+        if @repository_types.empty?
+          fail ::Katello::Errors::Pulp3MigrationError, 'There are no Pulp 3 content types to reset'
+        end
+
+        plugins = @repository_types.sort.map do |repository_type|
+          {
+            type: ::Katello::Pulp3::MigrationPlan.pulp2_repository_type(repository_type)
+          }
+        end
+        plan = { plugins: plugins }
+
+        # TODO: Don't provide the plan as a string once this is resolved: https://pulp.plan.io/issues/8211
+        migration_plan_api.reset(migration_plan_api.create(plan: plan).pulp_href, plan.to_json)
+
+        content_types_for_migration.each do |content_type|
+          if content_type.model_class == ::Katello::Erratum
+            ::Katello::RepositoryErratum.update_all(erratum_pulp3_href: nil)
+          else
+            content_type.model_class.update_all(migrated_pulp3_href: nil)
+          end
+        end
+
+        @repository_types.each do |repo_type|
+          if repo_type == "file"
+            ::Katello::Repository.file_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+          elsif repo_type == "docker"
+            ::Katello::Repository.docker_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+          elsif repo_type == "yum"
+            ::Katello::Repository.yum_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+          end
+        end
+
+        ::Katello::Pulp3::RepositoryReference.destroy_all
+        ::Katello::Pulp3::DistributionReference.destroy_all
+        ::Katello::Pulp3::ContentGuard.destroy_all
+      end
+
       def create_migrations
         plan = migration_plan
         Rails.logger.info("Migration Plan: #{plan}")
@@ -103,7 +170,7 @@ module Katello
       end
 
       def start_migration(plan_href)
-        migration_plan_api.run(plan_href, dry_run: false, validate: true)
+        migration_plan_api.run(plan_href, dry_run: false, validate: true, skip_corrupted: true)
       end
 
       def import_repositories(repository_type_label)
@@ -114,7 +181,9 @@ module Katello
         if repository_type_label == 'yum'
           import_yum_repos(imported, katello_repos)
         else
-          katello_repos.each do |repo|
+          repo_count = katello_repos.count
+          katello_repos.each_with_index do |repo, index|
+            update_import_status("Importing migrated content units #{repository_type_label}: #{index + 1}/#{repo_count}", index)
             found = imported.find { |migrated_repo| migrated_repo.pulp2_repo_id == repo.pulp_id }
             import_repo(repo, found) if found
           end
@@ -122,7 +191,9 @@ module Katello
       end
 
       def import_yum_repos(migrated_repo_items, repos)
-        repos.each do |yum_repo|
+        repo_count = repos.count
+        repos.each_with_index do |yum_repo, index|
+          update_import_status("Importing migrated yum repositories: #{index + 1}/#{repo_count}", index)
           to_find = nil
           if yum_repo.content_view.composite?
             if yum_repo.link?
@@ -195,14 +266,16 @@ module Katello
       end
 
       def operate_on_errata
+        last_migration_time = last_successful_migration_time
         offset = 0
-        limit = 300
-        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+        limit = SETTINGS[:katello][:pulp][:bulk_load_size]
+        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
         total_count = response.count
         yield(response.results)
         until (offset + limit > total_count)
           offset += limit
-          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
+          update_import_status("Importing migrated content type erratum: #{offset + limit}/#{total_count}")
           yield(response.results)
         end
       end
@@ -231,6 +304,12 @@ module Katello
                                           on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
       end
 
+      def mark_missing_content(content_type)
+        unless [Katello::DockerTag, Katello::DockerManifest, Katello::Erratum].include?(content_type.model_class)
+          content_type.model_class.where(:migrated_pulp3_href => nil).update_all(:missing_from_migration => true)
+        end
+      end
+
       def import_content_type(content_type)
         if content_type.model_class == Katello::Erratum
           import_errata
@@ -242,11 +321,15 @@ module Katello
             unmigrated_units = unmigrated_units.where(:migrated_pulp3_href => nil)
           end
 
+          total_count = unmigrated_units.count
+          current_count = 0
+
           unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
+            current_count += needing_hrefs.count
+            update_import_status("Importing migrated content type #{content_type.label}: #{current_count}/#{total_count}")
             migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
             migrated_units.results.each do |migrated_unit|
-              matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }
-              matching_record&.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content)
+              content_type.model_class.where(pulp_id: migrated_unit.pulp2_id).update_all(migrated_pulp3_href: migrated_unit.pulp3_content)
             end
           end
         end