katello 3.15.0.rc2 → 3.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85ee4f1e17bf63eec8c349a58dbe28aea05c1736937fa253c921c997e50247f8
-  data.tar.gz: 696c9f65f17d7a2405f4b2c807d6db42a269b728bb30cb10484fe0c6b75a88b5
+  metadata.gz: d7c699693c15591342b13df91353c83817bebe73159d71cb44af695b650decaf
+  data.tar.gz: a577aa95a181684ca801159c932943ebd6f91bacec215dc151e3e2de30d87cb3
 SHA512:
-  metadata.gz: 882b95014f435cf75227bf1969ce47294f45988d29f757c61d6c5c3a02d43af75d4b4238b8c654609cc03fa1269bed5afbc43bc2590ff49d59fab908c508a812
-  data.tar.gz: 9edd9187a9e65364ccd41687adf5a81414a8651eb9a02d20524e7d22fb56ddcd41d817b64d81112ba851ce7b13327eadb9640796e7f8db7e247936f819de9d90
+  metadata.gz: 074d5eb93cbeff706338e63c0e41113efe1e8b1ddc1537ac9d4190dc138c067ca8300579d4494aeed2a8eb5b76762728f902ec34d97ee5ad754ed31e4c586e60
+  data.tar.gz: 2e1a26e50825ef423edf40001003a8e7f0da557ae70df33920cafaeffc90c76bef6556715279096dabc815f2ba6a42bcd214a7b2e5a449724091ab4fc8d3496f

data/app/lib/katello/util/package.rb

@@ -22,7 +22,6 @@ module Katello
       def self.parse_nvrea(name)
         name, suffix = extract_suffix(name)
         name, arch = extract_arch(name)
-        return unless arch
 
         if (nvre = parse_nvre(name))
           nvre.merge(:suffix => suffix, :arch => arch).delete_if { |_k, v| v.nil? }

data/app/models/katello/docker_meta_tag.rb

@@ -153,17 +153,14 @@ module Katello
         end
 
         unless params_to_query_for_delete.empty?
-          DockerMetaTag.where(:id => repository_association_class.
-                              where(:repository_id => repo.id).
-                              select(:docker_meta_tag_id)).
-                              where(params_to_query_for_delete.join(" OR ")).delete_all
+          RepositoryDockerMetaTag.where(:docker_meta_tag => DockerMetaTag.where(params_to_query_for_delete.join(" OR "))).delete_all
         end
 
         metatags = []
         (tag_table_values - meta_tag_table_values).each do |schema1, schema2, name|
           metatags << DockerMetaTag.where(:schema1_id => schema1,
                               :schema2_id => schema2,
-                              :name => name).create!
+                              :name => name).first_or_create
         end
         repo.docker_meta_tags += metatags
       end

data/app/services/katello/pulp3/migration_switchover.rb

@@ -0,0 +1,94 @@
+require 'pulp_2to3_migration_client'
+
+module Katello
+  module Pulp3
+    class SwitchoverError < StandardError; end
+
+    class MigrationSwitchover
+      def initialize(*argv)
+        @migration = Katello::Pulp3::Migration.new(*argv)
+      end
+
+      def content_types
+        @migration.content_types_for_migration
+      end
+
+      def run
+        check_already_migrated_content
+        cleanup_v1_docker_tags if docker_migration?
+        migrated_content_type_check
+        combine_duplicate_docker_tags if docker_migration?
+        migrate_pulp3_hrefs
+      end
+
+      def docker_migration?
+        content_types.any? { |content_type| content_type.model_class::CONTENT_TYPE == "docker_tag" }
+      end
+
+      def migrate_pulp3_hrefs
+        content_types.each do |content_type|
+          content_type.model_class
+              .where.not("pulp_id=migrated_pulp3_href")
+              .update_all("pulp_id = migrated_pulp3_href")
+        end
+      end
+
+      def check_already_migrated_content
+        content_types.each do |content_type|
+          if content_type.model_class.where("pulp_id=migrated_pulp3_href").any?
+            Rails.logger.error("Content Switchover: #{content_type.label} seems to have already migrated content, switchover may fail.  Did you already perform the switchover?")
+          end
+        end
+      end
+
+      def cleanup_v1_docker_tags
+        unmigrated_docker_tags = Katello::DockerTag.includes(:schema1_meta_tag, :schema2_meta_tag).where(migrated_pulp3_href: nil)
+        unmigrated_docker_tags.find_in_batches(batch_size: 50_000) do |batch|
+          to_delete = []
+
+          batch.each do |unmigrated_tag|
+            if unmigrated_tag.schema1_meta_tag && unmigrated_tag.schema1_meta_tag.schema2.try(:migrated_pulp3_href)
+              Rails.logger.warn("Content Switchover: Deleting Docker tag #{unmigrated_tag.name} with pulp id: #{unmigrated_tag.pulp_id}")
+              to_delete << unmigrated_tag.id
+            end
+          end
+          Katello::DockerMetaTag.where(:schema1_id => to_delete).update_all(:schema1_id => nil)
+          Katello::RepositoryDockerTag.where(:docker_tag_id => to_delete).delete_all
+          Katello::DockerTag.where(:id => to_delete).delete_all
+        end
+
+        Katello::DockerMetaTag.cleanup_tags
+      end
+
+      def combine_duplicate_docker_tags
+        to_delete = []
+        Katello::DockerTag.having("count(migrated_pulp3_href) > 1").group(:migrated_pulp3_href).pluck(:migrated_pulp3_href).each do |duplicate_href|
+          tags = Katello::DockerTag.where(:migrated_pulp3_href => duplicate_href).includes(:schema1_meta_tag, :schema2_meta_tag).to_a
+          main_tag = tags.pop
+          main_meta_v1 = main_tag.schema1_meta_tag
+          main_meta_v2 = main_tag.schema2_meta_tag
+
+          Katello::RepositoryDockerTag.where(:docker_tag_id => tags.map(&:id)).update_all(:docker_tag_id => main_tag.id)
+          Katello::RepositoryDockerMetaTag.joins(:docker_meta_tag).where("#{Katello::DockerMetaTag.table_name}.schema1_id" => tags).update_all(:docker_meta_tag_id => main_meta_v1.id) if main_meta_v1
+          Katello::RepositoryDockerMetaTag.joins(:docker_meta_tag).where("#{Katello::DockerMetaTag.table_name}.schema2_id" => tags).update_all(:docker_meta_tag_id => main_meta_v2.id) if main_meta_v2
+
+          to_delete += tags.map(&:id)
+        end
+
+        to_delete.each_slice(10_000) do |group|
+          Katello::RepositoryDockerTag.where(:docker_tag_id => group).delete_all
+          Katello::DockerMetaTag.where(:schema1_id => group).or(Katello::DockerMetaTag.where(:schema2_id => group)).delete_all
+          Katello::DockerTag.where(:id => group).delete_all
+        end
+      end
+
+      def migrated_content_type_check
+        content_types.each do |content_type|
+          if content_type.model_class.where(migrated_pulp3_href: nil).any?
+            fail SwitchoverError, "ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n"
+          end
+        end
+      end
+    end
+  end
+end

data/db/migrate/20200402130013_add_repsoitory_docker_meta_tag_f_key.rb

@@ -0,0 +1,40 @@
+class AddRepsoitoryDockerMetaTagFKey < ActiveRecord::Migration[5.2]
+  def up
+    Katello::RepositoryDockerMetaTag.where.not(:repository_id => Katello::Repository.select(:id)).delete_all
+    Katello::RepositoryDockerMetaTag.where.not(:docker_meta_tag_id => Katello::DockerMetaTag.select(:id)).delete_all
+
+    add_foreign_key :katello_repository_docker_meta_tags, :katello_repositories, :column => :repository_id
+    add_foreign_key :katello_repository_docker_meta_tags, :katello_docker_meta_tags, :column => :docker_meta_tag_id
+
+    query = "DELETE FROM katello_repository_docker_meta_tags T1
+              USING   katello_repository_docker_meta_tags T2
+              WHERE   T1.ctid < T2.ctid
+              AND T1.repository_id  = T2.repository_id
+              AND T1.docker_meta_tag_id = T2.docker_meta_tag_id;"
+    ActiveRecord::Base.connection.execute(query)
+    add_index :katello_repository_docker_meta_tags, [:repository_id, :docker_meta_tag_id], :unique => true, :name => 'repository_docker_meta_tags_rid_dmtid'
+
+    Katello::RepositoryDockerTag.where.not(:repository_id => Katello::Repository.select(:id)).delete_all
+    Katello::RepositoryDockerTag.where.not(:docker_tag_id => Katello::DockerTag.select(:id)).delete_all
+
+    add_foreign_key :katello_repository_docker_tags, :katello_repositories, :column => :repository_id
+    add_foreign_key :katello_repository_docker_tags, :katello_docker_tags, :column => :docker_tag_id
+
+    query = "DELETE FROM katello_repository_docker_tags T1
+              USING   katello_repository_docker_tags T2
+              WHERE   T1.ctid < T2.ctid
+              AND T1.repository_id = T2.repository_id
+              AND T1.docker_tag_id = T2.docker_tag_id;"
+    ActiveRecord::Base.connection.execute(query)
+    add_index :katello_repository_docker_tags, [:repository_id, :docker_tag_id], :unique => true, :name => 'repository_docker_tags_rid_dtid'
+  end
+
+  def down
+    remove_foreign_key :katello_repository_docker_meta_tags, :katello_repositories
+    remove_foreign_key :katello_repository_docker_meta_tags, :katello_docker_meta_tags
+    remove_index :katello_repository_docker_meta_tags,  [:repository_id, :docker_meta_tag_id]
+    remove_foreign_key :katello_repository_docker_tags, :katello_repositories
+    remove_foreign_key :katello_repository_docker_tags, :katello_docker_tags
+    remove_index :katello_repository_docker_tags, [:repository_id, :docker_tag_id]
+  end
+end

data/lib/katello/tasks/pulp3_content_switchover.rake

@@ -3,20 +3,14 @@ require File.expand_path("../engine", File.dirname(__FILE__))
 namespace :katello do
   desc "Runs a Pulp 3 migration of pulp3 hrefs to pulp ids for supported content types."
   task :pulp3_content_switchover => :environment do
-    migration_service = Katello::Pulp3::Migration.new(SmartProxy.pulp_master)
-    content_types = migration_service.content_types_for_migration
-
-    content_types.each do |content_type|
-      if content_type.model_class.where(migrated_pulp3_href: nil).any?
-        $stderr.print("ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n")
-        exit 1
+    begin
+      ActiveRecord::Base.transaction do
+        switchover_service = Katello::Pulp3::MigrationSwitchover.new(SmartProxy.pulp_master)
+        switchover_service.run
       end
-    end
-
-    content_types.each do |content_type|
-      content_type.model_class
-        .where.not("pulp_id=migrated_pulp3_href")
-        .update_all("pulp_id = migrated_pulp3_href")
+    rescue Katello::Pulp3::SwitchoverError => e
+      $stderr.print(e.message)
+      exit 1
     end
   end
 end

data/lib/katello/tasks/pulp3_post_migration_check.rake

@@ -6,25 +6,41 @@ namespace :katello do
     repository_types = Katello::Pulp3::Migration::REPOSITORY_TYPES
 
     repository_types.each do |type|
-      filter = 'version_href is NULL OR remote_href is NULL'
-
-      unless type == Katello::Repository::DOCKER_TYPE
-        filter += ' OR publication_href is NULL'
+      # check version
+      nil_version_ids = Katello::Repository.with_type(type).where(version_href: nil).pluck(:id)
+      unless nil_version_ids.empty?
+        $stderr.print("ERROR: #{type} repositories with ID [#{nil_version_ids.join(',')}] have a NULL value for version_href\n")
+        exit 1
       end
-      repositories = Katello::Repository.with_type(type).where(filter)
 
-      if repositories.any?
-        $stderr.print("ERROR: #{type} repository #{repositories.first.id} has a NULL value for remote_href, version_href, publication_href\n")
+      # check remote
+      nil_remote_ids = Katello::Repository.with_type(type).where(remote_href: nil).collect do |repo|
+        repo.id if repo.in_default_view? && !repo.root.url.nil?
+      end
+      nil_remote_ids.compact!
+      unless nil_remote_ids.empty?
+        $stderr.print("ERROR: #{type} repositories with ID [#{nil_remote_ids.join(',')}] have a NULL value for remote_href\n")
         exit 1
       end
 
+      # check publication
+      unless type == Katello::Repository::DOCKER_TYPE
+        nil_publication_ids = Katello::Repository.with_type(type).where(publication_href: nil).pluck(:id)
+        unless nil_publication_ids.empty?
+          $stderr.print("ERROR: #{type} repositories with ID [#{nil_publication_ids.join(',')}] have a NULL value for publication_href\n")
+          exit 1
+        end
+      end
+
+      # check distribution
       non_archived_repositories = Katello::Repository.with_type(type).non_archived
-      with_no_distribution_references = non_archived_repositories
-        .left_outer_joins(:distribution_references)
-        .where(katello_distribution_references: { id: nil })
+      with_no_distribution_references = non_archived_repositories.
+        left_outer_joins(:distribution_references).
+        where(katello_distribution_references: { id: nil }).
+        pluck(:id)
 
       if with_no_distribution_references.any?
-        $stderr.print("ERROR: A non-archive #{type} repository #{with_no_distribution_references.first.id} did not have a distribution reference\n")
+        $stderr.print("ERROR: Non-archived #{type} repositories with ID [#{with_no_distribution_references.join(',')}] have no distribution reference\n")
         exit 1
       end
     end

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "3.15.0.rc2".freeze
+  VERSION = "3.15.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: katello
 version: !ruby/object:Gem::Version
-  version: 3.15.0.rc2
+  version: 3.15.0
 platform: ruby
 authors:
 - N/A
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-30 00:00:00.000000000 Z
+date: 2020-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -1317,6 +1317,7 @@ files:
 - app/services/katello/pulp3/file_unit.rb
 - app/services/katello/pulp3/migration.rb
 - app/services/katello/pulp3/migration_plan.rb
+- app/services/katello/pulp3/migration_switchover.rb
 - app/services/katello/pulp3/module_stream.rb
 - app/services/katello/pulp3/package_group.rb
 - app/services/katello/pulp3/pulp_content_unit.rb
@@ -1892,6 +1893,7 @@ files:
 - db/migrate/20200109162354_drop_host_update_lock_setting.rb
 - db/migrate/20200121213430_katello_repository_rpms_id_big_int.rb
 - db/migrate/20200129172534_add_epoch_version_release_arch_to_katello_installed_packages.rb
+- db/migrate/20200402130013_add_repsoitory_docker_meta_tag_f_key.rb
 - db/seeds.d/101-locations.rb
 - db/seeds.d/102-organizations.rb
 - db/seeds.d/104-proxy.rb
@@ -4643,9 +4645,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3
 signing_key: