katello 3.18.1 → 3.18.3.1

data/app/models/katello/host/content_facet.rb

@@ -31,43 +31,21 @@ module Katello
       validates :host, :presence => true, :allow_blank => false
       validates_with Validators::ContentViewEnvironmentValidator
 
-      def bindable_types
-        [
-          {
-            type: Repository::DEB_TYPE,
-            matcher: '/pulp/deb/',
-            paths: []
-          },
-          {
-            type: Repository::YUM_TYPE,
-            matcher: '/pulp/repos/',
-            paths: []
-          }
-        ]
-      end
-
       def update_repositories_by_paths(paths)
-        bindable_paths = bindable_types
+        prefixes = %w(/pulp/deb/ /pulp/repos/ /pulp/content/)
         relative_paths = []
 
-        # paths == ["/pulp/repos/Default_Organization/Library/custom/Test_product/test2",
-        # "/pulp/repos/Default_Organization/Library/custom/Test_product/My_repo"]
-        paths.each do |absolute_path|
-          bindable_paths.each do |supported|
-            relative_path = absolute_path.gsub(supported[:matcher], '') # remove e.g. '/pulp/repos/' from beginning of string
-            relative_paths << relative_path unless relative_path == absolute_path
-            if absolute_path.starts_with?(supported[:matcher])
-              supported[:paths] << relative_path
-              break
-            end
+        # paths == ["/pulp/repos/Default_Organization/Library/custom/Test_product/test2"]
+        paths.each do |path|
+          if (prefix = prefixes.find { |pre| path.start_with?(pre) })
+            relative_paths << path.gsub(prefix, '')
+          else
+            Rails.logger.warn("System #{self.host.name} (#{self.host.id}) requested binding to repo with unknown prefix. #{path}")
           end
         end
 
-        repos = bindable_paths.flat_map do |supported|
-          repos = Repository.joins(:root).where(RootRepository.table_name => {content_type: supported[:type]}, relative_path: supported[:paths])
-          relative_paths -= repos.pluck(:relative_path) # remove relative paths that match our repos
-          repos
-        end
+        repos = Repository.where(relative_path: relative_paths)
+        relative_paths -= repos.pluck(:relative_path) # remove relative paths that match our repos
 
         # Any leftover relative paths do not match the repos we've just retrieved from the db,
         # so we should log warnings about them.

data/app/models/katello/ping.rb

@@ -23,25 +23,18 @@ module Katello
         services
       end
 
-      # Calls "status" services in all backend engines.
       def ping(services: nil, capsule_id: nil)
-        services ||= self.services(capsule_id)
-        result = {}
-        services.each { |service| result[service] = {} }
+        ping_services_for_capsule(services, capsule_id)
+      end
 
-        ping_pulp3_without_auth(result[:pulp3], capsule_id) if result.include?(:pulp3)
-        ping_pulp_without_auth(result[:pulp], capsule_id) if result.include?(:pulp)
-        ping_candlepin_without_auth(result[:candlepin]) if result.include?(:candlepin)
+      def ping!(services: nil, capsule_id: nil)
+        result = ping_services_for_capsule(services, capsule_id)
 
-        ping_pulp_with_auth(result[:pulp_auth], result[:pulp][:status]) if result.include?(:pulp_auth)
-        ping_candlepin_with_auth(result[:candlepin_auth]) if result.include?(:candlepin_auth)
-        ping_foreman_tasks(result[:foreman_tasks]) if result.include?(:foreman_tasks)
-        ping_katello_events(result[:katello_events]) if result.include?(:katello_events)
-        ping_candlepin_events(result[:candlepin_events]) if result.include?(:candlepin_events)
+        if result[:status] != OK_RETURN_CODE
+          failed_names = failed_services(result).keys
+          fail("The following services have not been started or are reporting errors: #{failed_names.join(', ')}")
+        end
 
-        # set overall status result code
-        result = {:services => result}
-        result[:status] = result[:services].each_value.any? { |v| v[:status] == FAIL_RETURN_CODE } ? FAIL_RETURN_CODE : OK_RETURN_CODE
         result
       end
 
@@ -221,6 +214,33 @@ module Katello
 
       private
 
+      def failed_services(result)
+        result[:services].reject do |_name, details|
+          details[:status] != OK_RETURN_CODE
+        end
+      end
+
+      def ping_services_for_capsule(services, capsule_id)
+        services ||= self.services(capsule_id)
+        result = {}
+        services.each { |service| result[service] = {} }
+
+        ping_pulp3_without_auth(result[:pulp3], capsule_id) if result.include?(:pulp3)
+        ping_pulp_without_auth(result[:pulp], capsule_id) if result.include?(:pulp)
+        ping_candlepin_without_auth(result[:candlepin]) if result.include?(:candlepin)
+
+        ping_pulp_with_auth(result[:pulp_auth], result[:pulp][:status]) if result.include?(:pulp_auth)
+        ping_candlepin_with_auth(result[:candlepin_auth]) if result.include?(:candlepin_auth)
+        ping_foreman_tasks(result[:foreman_tasks]) if result.include?(:foreman_tasks)
+        ping_katello_events(result[:katello_events]) if result.include?(:katello_events)
+        ping_candlepin_events(result[:candlepin_events]) if result.include?(:candlepin_events)
+
+        # set overall status result code
+        result = {:services => result}
+        result[:status] = result[:services].each_value.any? { |v| v[:status] == FAIL_RETURN_CODE } ? FAIL_RETURN_CODE : OK_RETURN_CODE
+        result
+      end
+
       def fetch_proxy(capsule_id)
         capsule_id ? SmartProxy.unscoped.find(capsule_id) : SmartProxy.pulp_primary
       end

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -60,17 +60,7 @@ module Katello
       end
 
       def fetch_rpm_content_ids
-        # Query for applicable RPM ids
-        # -> Include all non-modular rpms or rpms that exist within installed module streams
-        enabled_module_stream_ids = ::Katello::ModuleStream.
-          joins("inner join katello_available_module_streams on
-            katello_module_streams.name = katello_available_module_streams.name and
-            katello_module_streams.stream = katello_available_module_streams.stream").
-          joins("inner join katello_host_available_module_streams on
-            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
-          where("katello_host_available_module_streams.host_id = :content_facet_id and
-            katello_host_available_module_streams.status = 'enabled'",
-            :content_facet_id => self.content_facet.host.id).select(:id)
+        enabled_module_stream_ids = fetch_enabled_module_stream_ids
 
         ::Katello::Rpm.
           joins("INNER JOIN katello_repository_rpms ON
@@ -88,9 +78,37 @@ module Katello
             :bound_library_repos => self.bound_library_instance_repos).
           where("katello_host_installed_packages.host_id = :content_facet_id",
             :content_facet_id => self.content_facet.host.id).
-          where("katello_module_stream_rpms.module_stream_id is null or
-            katello_module_stream_rpms.module_stream_id in (:enabled_module_streams)",
-            :enabled_module_streams => enabled_module_stream_ids).pluck(:id).uniq
+          where("(katello_module_stream_rpms.module_stream_id IS NULL AND
+            katello_installed_packages.id NOT IN (:locked_modular_installed_packages)) OR
+            (katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)
+            AND katello_installed_packages.id IN (:locked_modular_installed_packages))",
+            :enabled_module_streams => enabled_module_stream_ids,
+            :locked_modular_installed_packages => locked_modular_installed_packages(enabled_module_stream_ids)).pluck(:id).uniq
+      end
+
+      def fetch_enabled_module_stream_ids
+        # Query for applicable RPM ids
+        # -> Include all non-modular rpms or rpms that exist within installed module streams
+        ::Katello::ModuleStream.
+          joins("inner join katello_available_module_streams on
+            katello_module_streams.name = katello_available_module_streams.name and
+            katello_module_streams.stream = katello_available_module_streams.stream").
+          joins("inner join katello_host_available_module_streams on
+            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
+          where("katello_host_available_module_streams.host_id = :content_facet_id and
+            katello_host_available_module_streams.status = 'enabled'",
+            :content_facet_id => self.content_facet.host.id).select(:id)
+      end
+
+      # Installed packages that are locked for the host due to enabled module stream membership
+      def locked_modular_installed_packages(enabled_module_streams)
+        rpms_in_enabled_module_streams = ::Katello::Rpm.
+          joins("INNER JOIN katello_module_stream_rpms ON katello_rpms.id = katello_module_stream_rpms.rpm_id").
+          where("katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)",
+                :enabled_module_streams => enabled_module_streams).select(:nvra, :epoch)
+
+        ::Katello::InstalledPackage.where(nvra: rpms_in_enabled_module_streams.map(&:nvra),
+                                          epoch: rpms_in_enabled_module_streams.map(&:epoch)).select(:id)
       end
 
       def newest_distinct_installed_packages_query
@@ -104,20 +122,22 @@ module Katello
       end
 
       def applicable_differences
-        consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
-        content_ids = fetch_content_ids
+        ActiveRecord::Base.connection.uncached do
+          consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
+          content_ids = fetch_content_ids
 
-        to_remove = consumer_ids - content_ids
-        to_add = content_ids - consumer_ids
+          to_remove = consumer_ids - content_ids
+          to_add = content_ids - consumer_ids
 
-        [to_add, to_remove]
+          [to_add, to_remove]
+        end
       end
 
       def insert(applicable_ids)
         unless applicable_ids.empty?
           inserts = applicable_ids.map { |applicable_id| "(#{applicable_id.to_i}, #{content_facet.id.to_i})" }
           sql = "INSERT INTO #{content_facet_association_class.table_name} (#{content_unit_association_id}, content_facet_id) VALUES #{inserts.join(', ')}"
-          ActiveRecord::Base.connection.execute(sql)
+          ActiveRecord::Base.connection.exec_insert(sql)
         end
       end
 

data/app/services/katello/pulp3/api/core.rb

@@ -58,6 +58,20 @@ module Katello
           fail NotImplementedError
         end
 
+        def self.ignore_409_exception(*)
+          yield
+        rescue => e
+          raise e unless e&.code == 409
+          nil
+        end
+
+        def cancel_task(task_href)
+          data = PulpcoreClient::TaskResponse.new(state: 'canceled')
+          self.class.ignore_409_exception do
+            tasks_api.tasks_cancel(task_href, data)
+          end
+        end
+
         def exporter_api
           PulpcoreClient::ExportersPulpApi.new(core_api_client)
         end

data/app/services/katello/pulp3/erratum.rb

@@ -35,7 +35,8 @@ module Katello
         custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])
         custom_json["updated"] = convert_date_if_epoch(custom_json["updated"]) unless custom_json["updated"].blank?
 
-        if model.updated.blank? || (custom_json['updated'].to_datetime != model.updated.to_datetime)
+        if model.updated.blank? ||
+            (custom_json['updated'] && (custom_json['updated'].to_datetime != model.updated.to_datetime))
           custom_json['errata_id'] = custom_json.delete('id')
           custom_json['errata_type'] = custom_json.delete('type')
           custom_json['updated'] = custom_json['updated'].blank? ? custom_json['issued'] : custom_json['updated']

data/app/services/katello/pulp3/migration.rb

@@ -4,6 +4,8 @@ module Katello
   module Pulp3
     class Migration
       attr_accessor :smart_proxy, :reimport_all, :task_id
+      attr_reader :repository_types
+
       GET_QUERY_ID_LENGTH = 90
 
       MUTABLE_CONTENT_TYPES = [
@@ -15,6 +17,12 @@ module Katello
         Katello::Erratum
       ].freeze
 
+      CORRUPTABLE_CONTENT_TYPES = [
+        Katello::Rpm,
+        Katello::FileUnit,
+        Katello::Deb
+      ].freeze
+
       def self.repository_types_for_migration
         #we can migrate types that pulp3 supports, but are overridden to pulp2.  These are in 'migration mode'
         overridden = (SETTINGS[:katello][:use_pulp_2_for_content_type] || {}).keys.select { |key| SETTINGS[:katello][:use_pulp_2_for_content_type][key] }
@@ -66,7 +74,9 @@ module Katello
 
       def last_successful_migration_time
         task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigration.to_s, :result => 'success').order("started_at desc").first
-        if reimport_all || task.nil?
+        reset_task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigrationReset.to_s).order("started_at desc").first
+        reset_more_recent = reset_task && task && reset_task.started_at > task.started_at
+        if reimport_all || task.nil? || reset_more_recent
           0
         else
           task.started_at.to_i
@@ -104,6 +114,7 @@ module Katello
             Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
               Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type.label}) do
                 import_content_type(content_type)
+                mark_missing_content(content_type)
               end
             end
           end
@@ -114,6 +125,45 @@ module Katello
         Katello::Pulp3::MigrationPlan.new(@repository_types).generate.as_json
       end
 
+      def reset
+        if @repository_types.empty?
+          fail ::Katello::Errors::Pulp3MigrationError, 'There are no Pulp 3 content types to reset'
+        end
+
+        plugins = @repository_types.sort.map do |repository_type|
+          {
+            type: ::Katello::Pulp3::MigrationPlan.pulp2_repository_type(repository_type)
+          }
+        end
+        plan = { plugins: plugins }
+
+        pulp3_task = migration_plan_api.reset(migration_plan_api.create(plan: plan).pulp_href)
+
+        content_types_for_migration.each do |content_type|
+          if content_type.model_class == ::Katello::Erratum
+            ::Katello::RepositoryErratum.update_all(erratum_pulp3_href: nil)
+          else
+            content_type.model_class.update_all(migrated_pulp3_href: nil, missing_from_migration: false, ignore_missing_from_migration: false)
+          end
+        end
+
+        @repository_types.each do |repo_type|
+          if repo_type == "file"
+            ::Katello::Repository.file_type.update_all(remote_href: nil, publication_href: nil, version_href: nil)
+          elsif repo_type == "docker"
+            ::Katello::Repository.docker_type.update_all(remote_href: nil, publication_href: nil, version_href: nil)
+          elsif repo_type == "yum"
+            ::Katello::Repository.yum_type.update_all(remote_href: nil, publication_href: nil, version_href: nil)
+          end
+        end
+
+        ::Katello::Pulp3::RepositoryReference.destroy_all
+        ::Katello::Pulp3::DistributionReference.destroy_all
+        ::Katello::Pulp3::ContentGuard.destroy_all
+
+        pulp3_task
+      end
+
       def create_migrations
         plan = migration_plan
         Rails.logger.info("Migration Plan: #{plan}")
@@ -127,7 +177,7 @@ module Katello
       end
 
       def start_migration(plan_href)
-        migration_plan_api.run(plan_href, dry_run: false, validate: true)
+        migration_plan_api.run(plan_href, dry_run: false, validate: true, skip_corrupted: true)
       end
 
       def import_repositories(repository_type_label)
@@ -255,10 +305,17 @@ module Katello
               to_import[[errata_id, repo_id]] ||= {erratum_id: errata_id, erratum_pulp3_href: pulp3_href, repository_id: repo_id}
             end
           end
+
+          Katello::RepositoryErratum.import([:erratum_id, :erratum_pulp3_href, :repository_id], to_import.values, :validate => false,
+                                            on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
+          to_import = {}
         end
+      end
 
-        Katello::RepositoryErratum.import([:erratum_id, :erratum_pulp3_href, :repository_id], to_import.values, :validate => false,
-                                          on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
+      def mark_missing_content(content_type)
+        unless [Katello::DockerTag, Katello::DockerManifest, Katello::Erratum].include?(content_type.model_class)
+          content_type.model_class.where(:migrated_pulp3_href => nil).update_all(:missing_from_migration => true)
+        end
       end
 
       def import_content_type(content_type)
@@ -278,10 +335,9 @@ module Katello
           unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
             current_count += needing_hrefs.count
             update_import_status("Importing migrated content type #{content_type.label}: #{current_count}/#{total_count}")
-            migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
+            migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map(&:pulp_id))
             migrated_units.results.each do |migrated_unit|
-              matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }
-              matching_record&.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content)
+              content_type.model_class.where(pulp_id: migrated_unit.pulp2_id).update_all(migrated_pulp3_href: migrated_unit.pulp3_content)
             end
           end
         end

data/app/services/katello/pulp3/migration_plan.rb

@@ -21,13 +21,13 @@ module Katello
       def generate_plugins
         @repository_types.sort.map do |repository_type|
           {
-            type: pulp2_repository_type(repository_type),
+            type: self.class.pulp2_repository_type(repository_type),
             repositories: repository_migrations(repository_type)
           }
         end
       end
 
-      def pulp2_repository_type(repository_type)
+      def self.pulp2_repository_type(repository_type)
         if repository_type == 'yum'
           return 'rpm' #migration plugin uses rpm
         else

data/app/services/katello/pulp3/migration_switchover.rb

@@ -21,6 +21,20 @@ module Katello
           Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_content_types") { combine_duplicate_content_types }
           Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_docker_tags") { combine_duplicate_docker_tags } if docker_migration?
           Katello::Logging.time("CONTENT_SWITCHOVER - migrate_pulp3_hrefs") { migrate_pulp3_hrefs }
+          Katello::Logging.time("CONTENT_SWITCHOVER - remove_missing_content") { remove_missing_content }
+        end
+      end
+
+      def remove_orphaned_content
+        models = []
+        @migration.repository_types.each do |repo_type_label|
+          repo_type = ::Katello::RepositoryTypeManager.repository_types[repo_type_label]
+          indexable_types = repo_type.content_types_to_index
+          models += indexable_types&.map(&:model_class)
+          models.select! { |model| model.many_repository_associations }
+        end
+        models.each do |model|
+          model.joins("left join katello_#{model.repository_association} on #{model.table_name}.id = katello_#{model.repository_association}.#{model.unit_id_field}").where("katello_#{model.repository_association}.#{model.unit_id_field} IS NULL").destroy_all
         end
       end
 
@@ -106,12 +120,29 @@ module Katello
         end
       end
 
-      def migrated_content_type_check
+      def remove_missing_content
         content_types.each do |content_type|
-          if content_type.model_class == Katello::Erratum
-            migrated_errata_check
-          elsif content_type.model_class.where(migrated_pulp3_href: nil).any?
-            fail SwitchOverError, "ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n"
+          if Migration::CORRUPTABLE_CONTENT_TYPES.include?(content_type.model_class)
+            content_type.model_class.ignored_missing_migrated_content.destroy_all
+          elsif content_type.model_class == Katello::Erratum
+            Katello::RepositoryErratum.where(:erratum_pulp3_href => nil).delete_all
+          else
+            content_type.model_class.unmigrated_content.destroy_all
+          end
+        end
+      end
+
+      def migrated_content_type_check
+        content_classes = content_types.map(&:model_class)
+        migrated_errata_check if content_classes.include?(Katello::Erratum)
+
+        (content_classes & Migration::CORRUPTABLE_CONTENT_TYPES).each do |content_type|
+          if content_type.missing_migrated_content.any?
+            fail SwitchOverError, "ERROR: at least one #{content_type.table_name} record has been detected as corrupt or missing. Run 'foreman-rake katello:pulp3_migration_stats' for more information.\n"
+          end
+
+          if content_type.unmigrated_content.any?
+            fail SwitchOverError, "ERROR: at least one #{content_type.table_name} record was not able to be migrated\n"
           end
         end
       end

data/app/services/katello/pulp3/repository.rb

@@ -1,3 +1,4 @@
+# rubocop:disable Metrics/ClassLength
 require "pulpcore_client"
 module Katello
   module Pulp3
@@ -157,10 +158,7 @@ module Katello
       end
 
       def compute_remote_options(computed_options = remote_options)
-        [:client_cert, :client_key, :ca_cert].each do |key|
-          computed_options[key] = Digest::SHA256.hexdigest(computed_options[key].chomp) if computed_options[key]
-        end
-        computed_options.except(:name)
+        computed_options.except(:name, :client_key)
       end
 
       def create
@@ -220,11 +218,38 @@ module Katello
       end
 
       def refresh_distributions
-        dist_ref = distribution_reference
-        if dist_ref
-          update_distribution
+        if repo.docker?
+          dist = lookup_distributions(base_path: repo.container_repository_name).first
         else
+          dist = lookup_distributions(base_path: repo.relative_path).first
+        end
+
+        # First check if the distribution exists
+        if dist
+          dist_ref = distribution_reference
+          # If we have a DistributionReference, update the distribution
+          if dist_ref
+            return update_distribution
+          # If no DistributionReference, create a DistributionReference and return
+          else
+            save_distribution_references([dist.pulp_href])
+            return []
+          end
+        end
+
+        # So far, it looks like there is no distribution. Try to create one.
+        begin
           create_distribution(relative_path)
+        rescue api.class.client_module::ApiError => e
+          # Now it seems there is a distribution. Fetch it and save the reference.
+          if e.message.include?("\"base_path\":[\"This field must be unique.\"]") ||
+              e.message.include?("\"base_path\":[\"Overlaps with existing distribution\"")
+            dist = lookup_distributions(base_path: repo.relative_path).first
+            save_distribution_references([dist.pulp_href])
+            return []
+          else
+            raise e
+          end
         end
       end
 
@@ -257,8 +282,12 @@ module Katello
         create_version(:base_version => from_repository.version_href)
       end
 
+      def version_zero?
+        repo.version_href.ends_with?('/versions/0/')
+      end
+
       def delete_version
-        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
+        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) } unless version_zero?
       end
 
       def create_version(options = {})
@@ -270,7 +299,8 @@ module Katello
           pulp3_distribution_data = api.get_distribution(href)
           path, content_guard_href = pulp3_distribution_data&.base_path, pulp3_distribution_data&.content_guard
           unless distribution_reference
-            DistributionReference.create!(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href)
+            # Ensure that duplicates won't be created in the case of a race condition
+            DistributionReference.where(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href).first_or_create!
           end
         end
       end
@@ -300,7 +330,7 @@ module Katello
         }
         remote_options[:url] = root.url unless root.url.blank?
         remote_options[:download_concurrency] = root.download_concurrency unless root.download_concurrency.blank?
-        if root.upstream_username && root.upstream_password
+        if !root.upstream_username.blank? && !root.upstream_password.blank?
           remote_options.merge!(username: root.upstream_username,
                                password: root.upstream_password)
         end