katello 3.18.2.1 → 3.18.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c5ad1aaf8a0bb032cc1ffd42c4c070c77cc538044396af1cd40db8d00132e58
-  data.tar.gz: fd418dbd20507da2ca47e937df543ccf1f5a2c8d254f17623aec03f0d5d2767b
+  metadata.gz: a81840cf9dd6dd5b365f58ff15fcb56479a8a4d28222aaacd1c62f8601684a9a
+  data.tar.gz: f11313470e9660b9a88f0e992cb9f7ea85545b48f7b7148bcf29c4a1e40a7cd7
 SHA512:
-  metadata.gz: b21464bd7f2341093cb31c47a6ee1792d508b8d16f6c9f47db4e8cffb3bc737f02aa1150e50cc05352c668db37eadc5fe708cd0d2705010fbd0072f46b887877
-  data.tar.gz: b67b69add02fae0357308a8301ac689814f2e32d835b5270c421d94cf46f6068b6398cd18cbb7da55913106deac0be816668939c605e7bf20d0a671740039402
+  metadata.gz: 48801531bfad55800f818b0cd7570dc9710bfa9853d87850936aadc95f01d538bc2235681785a8a7d6744143fc6365d4750afc37754d486f8cfe92f231b66051
+  data.tar.gz: 8fa055b8365689b7781c63b9c115a28bcccd858c274e1ded11b7c226f58efe3d2a787daf7eb754483c379d9e82bfa8028b23d917e7f8ad6ac77a07ba6eb6c44d

data/app/controllers/katello/api/registry/registry_proxies_controller.rb

@@ -184,8 +184,8 @@ module Katello
                          'application/vnd.docker.distribution.manifest.v1+prettyjws'
                        end
         end
-
-        render json: manifest_response, content_type: media_type
+        response.headers['Content-Type'] = media_type
+        render json: manifest_response
       end
     end
 

data/app/lib/actions/katello/orphan_cleanup/remove_orphans.rb

@@ -8,7 +8,7 @@ module Actions
         def plan(proxy)
           sequence do
             plan_action(Actions::Pulp::Orchestration::OrphanCleanup::RemoveOrphans, proxy)
-            if proxy.pulp3_enabled?
+            if proxy.pulp3_enabled? && ::Katello::Ping.pulpcore_enabled
               plan_action(
                 Actions::Pulp3::Orchestration::OrphanCleanup::RemoveOrphans,
                 proxy)

data/app/models/katello/concerns/hostgroup_extensions.rb

@@ -126,7 +126,7 @@ module Katello
           facet_model = Facets.registered_facets[facet].hostgroup_configuration.model
           value = facet_model.where.not(attribute => nil).joins(:hostgroup).merge(
             ::Hostgroup.where(id: self.ancestor_ids).reorder(ancestry: :desc)
-          ).limit(1).pluck(attribute)
+          ).limit(1).pluck(attribute).first
         end
         value
       end

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -122,20 +122,22 @@ module Katello
       end
 
       def applicable_differences
-        consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
-        content_ids = fetch_content_ids
+        ActiveRecord::Base.connection.uncached do
+          consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
+          content_ids = fetch_content_ids
 
-        to_remove = consumer_ids - content_ids
-        to_add = content_ids - consumer_ids
+          to_remove = consumer_ids - content_ids
+          to_add = content_ids - consumer_ids
 
-        [to_add, to_remove]
+          [to_add, to_remove]
+        end
       end
 
       def insert(applicable_ids)
         unless applicable_ids.empty?
           inserts = applicable_ids.map { |applicable_id| "(#{applicable_id.to_i}, #{content_facet.id.to_i})" }
           sql = "INSERT INTO #{content_facet_association_class.table_name} (#{content_unit_association_id}, content_facet_id) VALUES #{inserts.join(', ')}"
-          ActiveRecord::Base.connection.execute(sql)
+          ActiveRecord::Base.connection.exec_insert(sql)
         end
       end
 

data/app/services/katello/pulp3/api/core.rb

@@ -58,6 +58,20 @@ module Katello
           fail NotImplementedError
         end
 
+        def self.ignore_409_exception(*)
+          yield
+        rescue => e
+          raise e unless e&.code == 409
+          nil
+        end
+
+        def cancel_task(task_href)
+          data = PulpcoreClient::TaskResponse.new(state: 'canceled')
+          self.class.ignore_409_exception do
+            tasks_api.tasks_cancel(task_href, data)
+          end
+        end
+
         def exporter_api
           PulpcoreClient::ExportersPulpApi.new(core_api_client)
         end

data/app/services/katello/pulp3/migration.rb

@@ -4,6 +4,8 @@ module Katello
   module Pulp3
     class Migration
       attr_accessor :smart_proxy, :reimport_all, :task_id
+      attr_reader :repository_types
+
       GET_QUERY_ID_LENGTH = 90
 
       MUTABLE_CONTENT_TYPES = [
@@ -16,7 +18,9 @@ module Katello
       ].freeze
 
       CORRUPTABLE_CONTENT_TYPES = [
-        Katello::Rpm, Katello::FileUnit
+        Katello::Rpm,
+        Katello::FileUnit,
+        Katello::Deb
       ].freeze
 
       def self.repository_types_for_migration
@@ -70,7 +74,9 @@ module Katello
 
       def last_successful_migration_time
         task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigration.to_s, :result => 'success').order("started_at desc").first
-        if reimport_all || task.nil?
+        reset_task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigrationReset.to_s).order("started_at desc").first
+        reset_more_recent = reset_task && task && reset_task.started_at > task.started_at
+        if reimport_all || task.nil? || reset_more_recent
           0
         else
           task.started_at.to_i
@@ -131,30 +137,31 @@ module Katello
         end
         plan = { plugins: plugins }
 
-        # TODO: Don't provide the plan as a string once this is resolved: https://pulp.plan.io/issues/8211
-        migration_plan_api.reset(migration_plan_api.create(plan: plan).pulp_href, plan.to_json)
+        pulp3_task = migration_plan_api.reset(migration_plan_api.create(plan: plan).pulp_href)
 
         content_types_for_migration.each do |content_type|
           if content_type.model_class == ::Katello::Erratum
             ::Katello::RepositoryErratum.update_all(erratum_pulp3_href: nil)
           else
-            content_type.model_class.update_all(migrated_pulp3_href: nil)
+            content_type.model_class.update_all(migrated_pulp3_href: nil, missing_from_migration: false, ignore_missing_from_migration: false)
           end
         end
 
         @repository_types.each do |repo_type|
           if repo_type == "file"
-            ::Katello::Repository.file_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+            ::Katello::Repository.file_type.update_all(remote_href: nil, publication_href: nil, version_href: nil)
           elsif repo_type == "docker"
-            ::Katello::Repository.docker_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+            ::Katello::Repository.docker_type.update_all(remote_href: nil, publication_href: nil, version_href: nil)
           elsif repo_type == "yum"
-            ::Katello::Repository.yum_type.update(remote_href: nil, publication_href: nil, version_href: nil)
+            ::Katello::Repository.yum_type.update_all(remote_href: nil, publication_href: nil, version_href: nil)
           end
         end
 
         ::Katello::Pulp3::RepositoryReference.destroy_all
         ::Katello::Pulp3::DistributionReference.destroy_all
         ::Katello::Pulp3::ContentGuard.destroy_all
+
+        pulp3_task
       end
 
       def create_migrations
@@ -298,10 +305,11 @@ module Katello
               to_import[[errata_id, repo_id]] ||= {erratum_id: errata_id, erratum_pulp3_href: pulp3_href, repository_id: repo_id}
             end
           end
-        end
 
-        Katello::RepositoryErratum.import([:erratum_id, :erratum_pulp3_href, :repository_id], to_import.values, :validate => false,
-                                          on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
+          Katello::RepositoryErratum.import([:erratum_id, :erratum_pulp3_href, :repository_id], to_import.values, :validate => false,
+                                            on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
+          to_import = {}
+        end
       end
 
       def mark_missing_content(content_type)
@@ -327,7 +335,7 @@ module Katello
           unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
             current_count += needing_hrefs.count
             update_import_status("Importing migrated content type #{content_type.label}: #{current_count}/#{total_count}")
-            migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
+            migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map(&:pulp_id))
             migrated_units.results.each do |migrated_unit|
               content_type.model_class.where(pulp_id: migrated_unit.pulp2_id).update_all(migrated_pulp3_href: migrated_unit.pulp3_content)
             end

data/app/services/katello/pulp3/migration_switchover.rb

@@ -25,6 +25,19 @@ module Katello
         end
       end
 
+      def remove_orphaned_content
+        models = []
+        @migration.repository_types.each do |repo_type_label|
+          repo_type = ::Katello::RepositoryTypeManager.repository_types[repo_type_label]
+          indexable_types = repo_type.content_types_to_index
+          models += indexable_types&.map(&:model_class)
+          models.select! { |model| model.many_repository_associations }
+        end
+        models.each do |model|
+          model.joins("left join katello_#{model.repository_association} on #{model.table_name}.id = katello_#{model.repository_association}.#{model.unit_id_field}").where("katello_#{model.repository_association}.#{model.unit_id_field} IS NULL").destroy_all
+        end
+      end
+
       def deduplicated_content_types
         #even though YumMetatadataFile is de-depulicated, we're not indexing it in pulp3
         [Katello::PackageGroup]

data/app/services/katello/pulp3/repository.rb

@@ -1,7 +1,7 @@
+# rubocop:disable Metrics/ClassLength
 require "pulpcore_client"
 module Katello
   module Pulp3
-    # rubocop:disable Metrics/ClassLength
     class Repository
       include Katello::Util::HttpProxy
       attr_accessor :repo
@@ -158,10 +158,7 @@ module Katello
       end
 
       def compute_remote_options(computed_options = remote_options)
-        [:client_cert, :client_key, :ca_cert].each do |key|
-          computed_options[key] = Digest::SHA256.hexdigest(computed_options[key].chomp) if computed_options[key]
-        end
-        computed_options.except(:name)
+        computed_options.except(:name, :client_key)
       end
 
       def create
@@ -221,11 +218,38 @@ module Katello
       end
 
       def refresh_distributions
-        dist_ref = distribution_reference
-        if dist_ref
-          update_distribution
+        if repo.docker?
+          dist = lookup_distributions(base_path: repo.container_repository_name).first
         else
+          dist = lookup_distributions(base_path: repo.relative_path).first
+        end
+
+        # First check if the distribution exists
+        if dist
+          dist_ref = distribution_reference
+          # If we have a DistributionReference, update the distribution
+          if dist_ref
+            return update_distribution
+          # If no DistributionReference, create a DistributionReference and return
+          else
+            save_distribution_references([dist.pulp_href])
+            return []
+          end
+        end
+
+        # So far, it looks like there is no distribution. Try to create one.
+        begin
           create_distribution(relative_path)
+        rescue api.class.client_module::ApiError => e
+          # Now it seems there is a distribution. Fetch it and save the reference.
+          if e.message.include?("\"base_path\":[\"This field must be unique.\"]") ||
+              e.message.include?("\"base_path\":[\"Overlaps with existing distribution\"")
+            dist = lookup_distributions(base_path: repo.relative_path).first
+            save_distribution_references([dist.pulp_href])
+            return []
+          else
+            raise e
+          end
         end
       end
 
@@ -275,7 +299,8 @@ module Katello
           pulp3_distribution_data = api.get_distribution(href)
           path, content_guard_href = pulp3_distribution_data&.base_path, pulp3_distribution_data&.content_guard
           unless distribution_reference
-            DistributionReference.create!(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href)
+            # Ensure that duplicates won't be created in the case of a race condition
+            DistributionReference.where(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href).first_or_create!
           end
         end
       end

data/app/services/katello/pulp3/repository/apt.rb

@@ -27,8 +27,7 @@ module Katello
           popts = super(repository_version)
           popts.merge!(
             {
-              # structured is not necessary for subscription-manager
-              #structured: true, # publish real suites (e.g. 'stable')
+              structured: true, # publish real suites (e.g. 'stable')
               simple: true # publish all into 'default'-suite
             }
           )

data/app/services/katello/pulp3/task.rb

@@ -56,8 +56,10 @@ module Katello
         @pulp_data ||= tasks_api.read(@href).as_json.with_indifferent_access
       end
 
-      def tasks_api
-        ::Katello::Pulp3::Api::Core.new(@smart_proxy).tasks_api
+      delegate :tasks_api, to: :core_api
+
+      def core_api
+        ::Katello::Pulp3::Api::Core.new(@smart_proxy)
       end
 
       def task_group_href
@@ -94,10 +96,11 @@ module Katello
       end
 
       def cancel
-        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
-        tasks_api.tasks_cancel(task_data['pulp_href'], data)
+        core_api.cancel_task(task_data['pulp_href'])
         #the main task may have completed, so cancel spawned tasks too
-        task_data['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
+        task_data['spawned_tasks']&.each do |spawned|
+          core_api.cancel_task(spawned['pulp_href'])
+        end
       end
     end
   end

data/app/services/katello/pulp3/task_group.rb

@@ -71,17 +71,25 @@ module Katello
       end
 
       def error
+        return if task_group_data[WAITING] > 0 || task_group_data[RUNNING] > 0
         if task_group_data[FAILED] > 0
           "#{task_group_data[FAILED]} subtask(s) failed for task group #{@href}."
+        elsif task_group_data[CANCELLED] > 0
+          "#{task_group_data[CANCELLED]} subtask(s) cancelled for task group #{@href}."
         end
       end
 
+      def core_api
+        ::Katello::Pulp3::Api::Core.new(@smart_proxy)
+      end
+
       def cancel
-        tasks_api = ::Katello::Pulp3::Api::Core.new(@smart_proxy).tasks_api
-        tasks_response = tasks_api.list(task_group: task_group_data['pulp_href'])
-        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
-        tasks_response.results.collect do |result|
-          tasks_api.tasks_cancel(result.pulp_href, data)
+        tasks_api = core_api.tasks_api
+        tasks_response = core_api.fetch_from_list do |page_opts|
+          tasks_api.list(page_opts.merge(task_group: task_group_data['pulp_href'], state__in: 'running,waiting'))
+        end
+        tasks_response.collect do |result|
+          ::Katello::Pulp3::Api::Core.new(@smart_proxy).cancel_task(result.pulp_href)
         end
       end
     end

data/db/migrate/20210420140050_add_pulp3_hrefs_to_content_types_deb.rb

@@ -0,0 +1,5 @@
+class AddPulp3HrefsToContentTypesDeb < ActiveRecord::Migration[6.0]
+  def change
+    add_column Katello::Deb.table_name, :migrated_pulp3_href, :string, :default => nil, :null => true
+  end
+end

data/lib/katello/engine.rb

@@ -75,7 +75,7 @@ module Katello
 
     # make sure the Katello plugin is initialized before `after_initialize`
     # hook so that the resumed Dynflow tasks can rely on everything ready.
-    initializer 'katello.register_plugin', :before => :finisher_hook do
+    initializer 'katello.register_plugin', :before => :finisher_hook, :after => 'foreman_remote_execution.register_plugin' do
       ::Foreman::AccessControl::Permission.prepend ::Katello::Concerns::PermissionExtensions
       require 'katello/plugin'
 

data/lib/katello/tasks/check_config.rake

@@ -0,0 +1,18 @@
+namespace :katello do
+  task :check_config => ['environment'] do
+    desc "Task that can be run before a content migration to check that the configuration valid"
+    fail_msg = _("The system appears to already be using pulp3 with all content migrated.")
+
+    puts "Checking for valid Katello configuraton."
+    if SETTINGS[:katello][:use_pulp_2_for_content_type].nil?
+      fail fail_msg
+    end
+
+    if !SETTINGS[:katello][:use_pulp_2_for_content_type][:docker] &&
+      !SETTINGS[:katello][:use_pulp_2_for_content_type][:file] &&
+      !SETTINGS[:katello][:use_pulp_2_for_content_type][:yum] &&
+      !SETTINGS[:katello][:use_pulp_2_for_content_type][:deb]
+      fail fail_msg
+    end
+  end
+end

data/lib/katello/tasks/pulp3_content_switchover.rake

@@ -3,13 +3,15 @@ require "#{Katello::Engine.root}/app/services/katello/pulp3/migration_switchover
 
 namespace :katello do
   desc "Runs a Pulp 3 migration of pulp3 hrefs to pulp ids for supported content types."
-  task :pulp3_content_switchover => ["dynflow:client"] do
+  task :pulp3_content_switchover => ["dynflow:client", "check_config"] do
     dryrun = ENV['DRYRUN']
+
     begin
       User.current = User.anonymous_admin
 
+      switchover_service = Katello::Pulp3::MigrationSwitchover.new(SmartProxy.pulp_primary)
+      switchover_service.remove_orphaned_content #run out of transaction for easier re-run
       ActiveRecord::Base.transaction do
-        switchover_service = Katello::Pulp3::MigrationSwitchover.new(SmartProxy.pulp_primary)
         switchover_service.run
         fail "Dryrun completed without error, aborting and rolling back" if dryrun
       end
@@ -18,7 +20,7 @@ namespace :katello do
       exit 1
     end
 
-    task = ForemanTasks.sync_task(Actions::Pulp3::ContentGuard::RefreshAllDistributions, SmartProxy.pulp_primary)
+    task = ForemanTasks.async_task(Actions::Pulp3::ContentGuard::RefreshAllDistributions, SmartProxy.pulp_primary)
     until !task.pending? || task.paused?
       sleep(10)
       task = ForemanTasks::Task.find(task.id)

data/lib/katello/tasks/pulp3_migration.rake

@@ -1,6 +1,6 @@
 namespace :katello do
   desc "Runs a Pulp 2 to 3 Content Migration for supported types.  May be run multiple times.  Use wait=false to immediately return with a task url."
-  task :pulp3_migration => ["dynflow:client"] do
+  task :pulp3_migration => ["dynflow:client", 'check_config'] do
     services = [:candlepin, :foreman_tasks, :pulp3, :pulp, :pulp_auth]
     Katello::Ping.ping!(services: services)
 

data/lib/katello/tasks/pulp3_migration_stats.rake

@@ -10,7 +10,8 @@ namespace :katello do
     migratable_repo_count = ::Katello::Repository.count - ::Katello::Repository.puppet_type.count -
       ::Katello::Repository.ostree_type.count - ::Katello::Repository.deb_type.count
 
-    on_demand_rpm_count = Katello::RepositoryRpm.where(:repository_id => Katello::Repository.yum_type.on_demand).distinct.count
+    on_demand_rpm_count = Katello::RepositoryRpm.where(:repository_id => Katello::Repository.yum_type.on_demand).
+      select(:rpm_id).distinct.count
     on_demand_unmigrated_rpm_count = on_demand_rpm_count - migrated_rpm_count
     immediate_unmigrated_rpm_count = ::Katello::Rpm.count - migrated_rpm_count - on_demand_unmigrated_rpm_count
 

data/lib/katello/tasks/reimport.rake

@@ -2,7 +2,7 @@ require File.expand_path("../engine", File.dirname(__FILE__))
 
 namespace :katello do
   desc "Runs a katello ping and prints out the statuses of each service"
-  task :check_ping => :environment do
+  task :check_ping => [:environment, "dynflow:client"] do
     ::User.current = ::User.anonymous_admin
     ping_results = Katello::Ping.ping
     if ping_results[:status] != "ok"

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "3.18.2.1".freeze
+  VERSION = "3.18.3".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: katello
 version: !ruby/object:Gem::Version
-  version: 3.18.2.1
+  version: 3.18.3
 platform: ruby
 authors:
 - N/A
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-16 00:00:00.000000000 Z
+date: 2021-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -366,40 +366,40 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.10.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.10.0
+        version: 3.11.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.10.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.10.0
+        version: 3.11.0
 - !ruby/object:Gem::Dependency
   name: pulp_2to3_migration_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: pulp_certguard_client
   requirement: !ruby/object:Gem::Requirement
@@ -2142,6 +2142,7 @@ files:
 - db/migrate/20201021150008_add_import_only_to_katello_content_view.rb
 - db/migrate/20201119211133_pulp3_migration_progress.rb
 - db/migrate/20210201165835_add_migration_missing_content.rb
+- db/migrate/20210420140050_add_pulp3_hrefs_to_content_types_deb.rb
 - db/seeds.d/101-locations.rb
 - db/seeds.d/102-organizations.rb
 - db/seeds.d/104-proxy.rb
@@ -4484,6 +4485,7 @@ files:
 - lib/katello/repository_types/puppet.rb
 - lib/katello/repository_types/yum.rb
 - lib/katello/scheduled_jobs.rb
+- lib/katello/tasks/check_config.rake
 - lib/katello/tasks/clean_backend_objects.rake
 - lib/katello/tasks/clean_old_file_repos.rake
 - lib/katello/tasks/clean_published_repo_directories.rake