katello 3.15.0.rc2 → 3.15.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 85ee4f1e17bf63eec8c349a58dbe28aea05c1736937fa253c921c997e50247f8
-  data.tar.gz: 696c9f65f17d7a2405f4b2c807d6db42a269b728bb30cb10484fe0c6b75a88b5
+SHA1:
+  metadata.gz: 4ad9b04f40e657a51875c8695653860da9fb8ae7
+  data.tar.gz: 812424e72605d87d742fa45d14b09a9eaebeaa30
 SHA512:
-  metadata.gz: 882b95014f435cf75227bf1969ce47294f45988d29f757c61d6c5c3a02d43af75d4b4238b8c654609cc03fa1269bed5afbc43bc2590ff49d59fab908c508a812
-  data.tar.gz: 9edd9187a9e65364ccd41687adf5a81414a8651eb9a02d20524e7d22fb56ddcd41d817b64d81112ba851ce7b13327eadb9640796e7f8db7e247936f819de9d90
+  metadata.gz: 5e2c8979164f4403578cd093a6b34cb88bc091aa28179bb2c420730cc2bb10e2bf0cdf7ba2e5ac445b78cc0ba3fa46a21c4ce85c3447b88a8535b0ef5898f7c4
+  data.tar.gz: a8634db2c5e979b92723e368dbf92e42766d5f20d0f8757b4f88f3ac6bf56eb7c7e4f86e14510b6bb6101d517713b915aff01f6d5145ea2ac2c85ac05e0fb87c

data/app/controllers/katello/api/v2/docker_tags_controller.rb

@@ -25,6 +25,23 @@ module Katello
       end
     end
 
+    api :GET, "/docker_tags/:id/repositories", N_("List of repositories for a docker meta tag")
+    def repositories
+      tag = DockerMetaTag.find(params[:id])
+
+      if tag.repositories.size > 1 #pulp3
+        repos = tag.repositories.non_archived
+      else
+        repos = []
+        tag.related_tags.each do |related|
+          repos << related.repositories.non_archived
+        end
+        repos.flatten!
+      end
+
+      respond_with_template_collection('index', 'repositories', collection: full_result_response(repos))
+    end
+
     private
 
     def find_repositories

data/app/lib/actions/katello/capsule_content/sync_capsule.rb

@@ -31,6 +31,10 @@ module Actions
           end
         end
 
+        def resource_locks
+          :link
+        end
+
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end

data/app/lib/actions/katello/host/upload_profiles.rb

@@ -33,16 +33,51 @@ module Actions
           Dynflow::Action::Rescue::Skip
         end
 
+        def self.upload_modules_to_pulp(available_streams, host)
+          query_name_streams = available_streams.map do |profile|
+            ::Katello::ModuleStream.where(profile.slice(:name, :stream))
+          end
+
+          updated_profiles = []
+          unless query_name_streams.empty?
+            query_name_streams = query_name_streams.inject(&:or)
+
+            bound_library_instances = host.content_facet.bound_repositories.map(&:library_instance_or_self)
+            query = ::Katello::ModuleStream.in_repositories(bound_library_instances).
+                                            select(:name, :stream, :version, :context, :arch).
+                                              merge(query_name_streams)
+
+            updated_profiles = query.map do |module_stream|
+              module_stream.slice(:name, :stream, :version, :context, :arch)
+            end
+          end
+
+          # We also need to pass module streams that are not found in the ModuleStream table
+          # but are present on the content host
+          unassociated_profiles = available_streams.select do |profile|
+            updated_profiles.none? { |p| p[:name] == profile[:name] && p[:stream] == profile[:stream] }
+          end
+
+          module_stream_profile = updated_profiles + unassociated_profiles
+
+          unless module_stream_profile.empty?
+            begin
+              ::Katello::Pulp::Consumer.new(host.content_facet.uuid).
+                upload_module_stream_profile(module_stream_profile)
+            rescue RestClient::ResourceNotFound
+              Rails.logger.warn("Host with ID %s was not known to Pulp, continuing" % host.id)
+            end
+          end
+        end
+
         def import_module_streams(payload, host)
           enabled_payload = payload.map do |profile|
-            profile.slice("name", "stream", "version", "context", "arch") if profile["status"] == "enabled"
+            profile.slice("name", "stream", "version", "context", "arch").with_indifferent_access if profile["status"] == "enabled"
           end
           enabled_payload.compact!
 
-          ::Katello::Pulp::Consumer.new(host.content_facet.uuid).upload_module_stream_profile(enabled_payload)
+          UploadProfiles.upload_modules_to_pulp(enabled_payload, host)
           host.import_module_streams(payload)
-        rescue RestClient::ResourceNotFound
-          Rails.logger.warn("Host with ID %s was not known to Pulp, continuing" % input[:host_id])
         end
 
         def import_deb_package_profile(host, profile)
@@ -69,6 +104,7 @@ module Actions
             payload = profiles.dig("deb_package_profile", "deb_packages") || []
             import_deb_package_profile(host, payload)
           else
+            module_streams = []
             profiles.each do |profile|
               payload = profile["profile"]
               case profile["content_type"]
@@ -79,9 +115,13 @@ module Actions
               when "enabled_repos"
                 host.import_enabled_repositories(payload)
               else
-                import_module_streams(payload, host)
+                module_streams << payload
               end
             end
+
+            module_streams.each do |module_stream_payload|
+              import_module_streams(module_stream_payload, host)
+            end
           end
         end
       end

data/app/lib/actions/katello/orphan_cleanup/remove_orphans.rb

@@ -13,6 +13,19 @@ module Actions
                 Actions::Pulp3::Orchestration::OrphanCleanup::RemoveOrphans,
                 proxy)
             end
+            plan_self
+          end
+        end
+
+        def run
+          models = []
+          ::Katello::RepositoryTypeManager.repository_types.each_value do |repo_type|
+            indexable_types = repo_type.content_types_to_index
+            models += indexable_types&.map(&:model_class)
+            models.select! { |model| model.many_repository_associations }
+          end
+          models.each do |model|
+            model.joins("left join katello_#{model.repository_association} on #{model.table_name}.id = katello_#{model.repository_association}.#{model.unit_id_field}").where("katello_#{model.repository_association}.#{model.unit_id_field} IS NULL").destroy_all
           end
         end
       end

data/app/lib/actions/katello/pulp_selector.rb

@@ -3,6 +3,7 @@ module Actions
     module PulpSelector
       def plan_pulp_action(backend_actions, repository, smart_proxy, *args)
         fail "nil smart_proxy passed to PulpSelector" if smart_proxy.nil?
+        smart_proxy.fix_pulp3_capabilities(repository.content_type)
         planned = plan_optional_pulp_action(backend_actions, repository, smart_proxy, *args)
         fail "Could not locate an action for type #{smart_proxy.backend_service_type(repository)}" unless planned
         planned

data/app/lib/katello/errors.rb

@@ -45,6 +45,12 @@ module Katello
       end
     end
 
+    class PulpcoreMissingCapabilities < StandardError
+      def message
+        _("A smart proxy seems to have been refreshed without pulpcore being running.  You may want to ")
+      end
+    end
+
     class ConnectionRefusedException < StandardError; end
 
     class MaxHostsReachedException < StandardError; end

data/app/lib/katello/util/package.rb

@@ -22,7 +22,6 @@ module Katello
       def self.parse_nvrea(name)
         name, suffix = extract_suffix(name)
         name, arch = extract_arch(name)
-        return unless arch
 
         if (nvre = parse_nvre(name))
           nvre.merge(:suffix => suffix, :arch => arch).delete_if { |_k, v| v.nil? }

data/app/models/katello/concerns/host_managed_extensions.rb

@@ -106,7 +106,7 @@ module Katello
 
       def import_package_profile(simple_packages)
         found = import_package_profile_in_bulk(simple_packages)
-        sync_package_associations(found.map(&:id))
+        sync_package_associations(found.map(&:id).uniq)
       end
 
       def import_package_profile_in_bulk(simple_packages)
@@ -127,9 +127,14 @@ module Katello
                                           :arch => simple_package.arch)
         end
         InstalledPackage.import(installed_packages, validate: false, on_duplicate_key_ignore: true)
+        #re-lookup all imported to pickup any duplicates/conflicts
+        imported = InstalledPackage.where(:nvrea => installed_packages.map(&:nvrea)).select(:id).to_a
 
-        found << installed_packages
-        found.flatten
+        if imported.count != installed_packages.count
+          Rails.logger.warn("Mismatch found in installed package insertion, expected #{installed_packages.count} but only could find #{imported.count}.  This is most likley a bug.")
+        end
+
+        (found + imported).flatten
       end
 
       def import_enabled_repositories(repos)

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -124,6 +124,8 @@ module Katello
           config.ssl_client_key = ::Cert::Certs.ssl_client_key
           config.debugging = true
           config.logger = ::Foreman::Logging.logger('katello/pulp_rest')
+          config.username = self.setting(PULP3_FEATURE, 'username')
+          config.password = self.setting(PULP3_FEATURE, 'password')
         end
       end
 
@@ -152,6 +154,19 @@ module Katello
         end
       end
 
+      def missing_pulp3_capabilities?
+        pulp3_enabled? && self.capabilities(PULP3_FEATURE).empty?
+      end
+
+      def fix_pulp3_capabilities(type)
+        if missing_pulp3_capabilities? && !pulp2_preferred_for_type?(type)
+          self.refresh
+          if self.capabilities(::SmartProxy::PULP3_FEATURE).empty?
+            fail Katello::Errors::PulpcoreMissingCapabilities
+          end
+        end
+      end
+
       def pulp3_repository_type_support?(repository_type, check_pulp2_preferred = true)
         repository_type_obj = repository_type.is_a?(String) ? Katello::RepositoryTypeManager.repository_types[repository_type] : repository_type
         fail "Cannot find repository type #{repository_type}, is it enabled?" unless repository_type_obj
@@ -182,7 +197,7 @@ module Katello
         pulp3_uri!.host
       end
 
-      def pulp3_url(path = '/pulp/api/v3/')
+      def pulp3_url(path = '/pulp/api/v3')
         pulp_url = self.setting(PULP3_FEATURE, 'pulp_url')
         path.blank? ? pulp_url : "#{pulp_url.sub(%r|/$|, '')}/#{path.sub(%r|^/|, '')}"
       end

data/app/models/katello/docker_meta_tag.rb

@@ -107,7 +107,7 @@ module Katello
       if grouped
         grouped_fields = "#{table_name}.name, katello_repositories.root_id"
         ids = distinct.select("ON (#{grouped_fields}) #{table_name}.id").joins(:repositories)
-        where(:id => ids).where("#{self.table_name}.schema1_id in (#{sql}) or #{self.table_name}.schema2_id in (#{sql})")
+        self.where(:id => ids)
       else
         self.where("#{self.table_name}.schema1_id in (#{sql}) or #{self.table_name}.schema2_id in (#{sql})")
       end
@@ -153,17 +153,14 @@ module Katello
         end
 
         unless params_to_query_for_delete.empty?
-          DockerMetaTag.where(:id => repository_association_class.
-                              where(:repository_id => repo.id).
-                              select(:docker_meta_tag_id)).
-                              where(params_to_query_for_delete.join(" OR ")).delete_all
+          RepositoryDockerMetaTag.where(:docker_meta_tag => DockerMetaTag.where(params_to_query_for_delete.join(" OR "))).delete_all
         end
 
         metatags = []
         (tag_table_values - meta_tag_table_values).each do |schema1, schema2, name|
           metatags << DockerMetaTag.where(:schema1_id => schema1,
                               :schema2_id => schema2,
-                              :name => name).create!
+                              :name => name).first_or_create
         end
         repo.docker_meta_tags += metatags
       end

data/app/models/katello/installed_package.rb

@@ -4,8 +4,8 @@ module Katello
       allow :nvra, :nvrea, :name
     end
 
-    has_many :hosts, :through => :host_installed_packages, :class_name => "::Host"
     has_many :host_installed_packages, :class_name => "Katello::HostInstalledPackage", :dependent => :destroy, :inverse_of => :installed_package
+    has_many :hosts, :through => :host_installed_packages, :class_name => "::Host"
 
     scoped_search :on => :name, :complete_value => true
     scoped_search :on => :nvrea

data/app/models/katello/repository.rb

@@ -422,8 +422,12 @@ module Katello
       path
     end
 
+    def library_instance_or_self
+      self.library_instance || self
+    end
+
     def generate_repo_path(content_path = nil)
-      _org, _content, content_path = (self.library_instance || self).relative_path.split("/", 3) if content_path.blank?
+      _org, _content, content_path = library_instance_or_self.relative_path.split("/", 3) if content_path.blank?
       content_path = content_path.sub(%r|^/|, '')
       if self.environment
         cve = ContentViewEnvironment.where(:environment_id => self.environment,
@@ -496,7 +500,7 @@ module Katello
         end
       end
       clone = Repository.new(:environment => to_env,
-                     :library_instance => self.library_instance || self,
+                     :library_instance => library_instance_or_self,
                      :root => self.root,
                      :content_view_version => to_version,
                      :saved_checksum_type => checksum_type)

data/app/services/katello/pulp/simple_package.rb

@@ -13,8 +13,8 @@ module Katello
       end
 
       def nvrea
-        if epoch == "0"
-          @nvra
+        if epoch.nil? || epoch.to_s == "0"
+          nvra
         else
           "#{@name}-#{@epoch}:#{@version}-#{@release}.#{@arch}"
         end

data/app/services/katello/pulp3/migration_switchover.rb

@@ -0,0 +1,94 @@
+require 'pulp_2to3_migration_client'
+
+module Katello
+  module Pulp3
+    class SwitchoverError < StandardError; end
+
+    class MigrationSwitchover
+      def initialize(*argv)
+        @migration = Katello::Pulp3::Migration.new(*argv)
+      end
+
+      def content_types
+        @migration.content_types_for_migration
+      end
+
+      def run
+        check_already_migrated_content
+        cleanup_v1_docker_tags if docker_migration?
+        migrated_content_type_check
+        combine_duplicate_docker_tags if docker_migration?
+        migrate_pulp3_hrefs
+      end
+
+      def docker_migration?
+        content_types.any? { |content_type| content_type.model_class::CONTENT_TYPE == "docker_tag" }
+      end
+
+      def migrate_pulp3_hrefs
+        content_types.each do |content_type|
+          content_type.model_class
+              .where.not("pulp_id=migrated_pulp3_href")
+              .update_all("pulp_id = migrated_pulp3_href")
+        end
+      end
+
+      def check_already_migrated_content
+        content_types.each do |content_type|
+          if content_type.model_class.where("pulp_id=migrated_pulp3_href").any?
+            Rails.logger.error("Content Switchover: #{content_type.label} seems to have already migrated content, switchover may fail.  Did you already perform the switchover?")
+          end
+        end
+      end
+
+      def cleanup_v1_docker_tags
+        unmigrated_docker_tags = Katello::DockerTag.includes(:schema1_meta_tag, :schema2_meta_tag).where(migrated_pulp3_href: nil)
+        unmigrated_docker_tags.find_in_batches(batch_size: 50_000) do |batch|
+          to_delete = []
+
+          batch.each do |unmigrated_tag|
+            if unmigrated_tag.schema1_meta_tag && unmigrated_tag.schema1_meta_tag.schema2.try(:migrated_pulp3_href)
+              Rails.logger.warn("Content Switchover: Deleting Docker tag #{unmigrated_tag.name} with pulp id: #{unmigrated_tag.pulp_id}")
+              to_delete << unmigrated_tag.id
+            end
+          end
+          Katello::DockerMetaTag.where(:schema1_id => to_delete).update_all(:schema1_id => nil)
+          Katello::RepositoryDockerTag.where(:docker_tag_id => to_delete).delete_all
+          Katello::DockerTag.where(:id => to_delete).delete_all
+        end
+
+        Katello::DockerMetaTag.cleanup_tags
+      end
+
+      def combine_duplicate_docker_tags
+        to_delete = []
+        Katello::DockerTag.having("count(migrated_pulp3_href) > 1").group(:migrated_pulp3_href).pluck(:migrated_pulp3_href).each do |duplicate_href|
+          tags = Katello::DockerTag.where(:migrated_pulp3_href => duplicate_href).includes(:schema1_meta_tag, :schema2_meta_tag).to_a
+          main_tag = tags.pop
+          main_meta_v1 = main_tag.schema1_meta_tag
+          main_meta_v2 = main_tag.schema2_meta_tag
+
+          Katello::RepositoryDockerTag.where(:docker_tag_id => tags.map(&:id)).update_all(:docker_tag_id => main_tag.id)
+          Katello::RepositoryDockerMetaTag.joins(:docker_meta_tag).where("#{Katello::DockerMetaTag.table_name}.schema1_id" => tags).update_all(:docker_meta_tag_id => main_meta_v1.id) if main_meta_v1
+          Katello::RepositoryDockerMetaTag.joins(:docker_meta_tag).where("#{Katello::DockerMetaTag.table_name}.schema2_id" => tags).update_all(:docker_meta_tag_id => main_meta_v2.id) if main_meta_v2
+
+          to_delete += tags.map(&:id)
+        end
+
+        to_delete.each_slice(10_000) do |group|
+          Katello::RepositoryDockerTag.where(:docker_tag_id => group).delete_all
+          Katello::DockerMetaTag.where(:schema1_id => group).or(Katello::DockerMetaTag.where(:schema2_id => group)).delete_all
+          Katello::DockerTag.where(:id => group).delete_all
+        end
+      end
+
+      def migrated_content_type_check
+        content_types.each do |content_type|
+          if content_type.model_class.where(migrated_pulp3_href: nil).any?
+            fail SwitchoverError, "ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n"
+          end
+        end
+      end
+    end
+  end
+end

data/app/views/foreman/job_templates/install_errata_-_katello_ansible_default.erb

@@ -15,4 +15,4 @@ kind: job_template
 %>
 
 <% advisories = input(:errata).split(',').map { |e| "--advisory=#{e}" }.join(' ') %>
-<%= render_template('Run Command - Ansible Default', :command => "yum update-minimal #{advisories}") %>
+<%= render_template('Run Command - Ansible Default', :command => "yum -y update-minimal #{advisories}") %>

data/app/views/foreman/job_templates/install_group_-_katello_ansible_default.erb

@@ -13,4 +13,4 @@ provider_type: Ansible
 kind: job_template
 %>
 
-<%= render_template('Run Command - Ansible Default', :command => "yum group install #{input('package')}") %>
+<%= render_template('Run Command - Ansible Default', :command => "yum -y group install #{input('package')}") %>

data/app/views/foreman/job_templates/remove_group_-_katello_ansible_default.erb

@@ -13,4 +13,4 @@ provider_type: Ansible
 kind: job_template
 %>
 
-<%= render_template('Run Command - Ansible Default', :command => "yum group remove #{input('package')}") %>
+<%= render_template('Run Command - Ansible Default', :command => "yum -y group remove #{input('package')}") %>

data/app/views/foreman/job_templates/update_group_-_katello_ansible_default.erb

@@ -13,4 +13,4 @@ provider_type: Ansible
 kind: job_template
 %>
 
-<%= render_template('Run Command - Ansible Default', :command => "yum group update #{input('package')}") %>
+<%= render_template('Run Command - Ansible Default', :command => "yum -y group update #{input('package')}") %>

data/app/views/foreman/job_templates/update_package_-_katello_ansible_default.erb

@@ -13,4 +13,4 @@ provider_type: Ansible
 kind: job_template
 %>
 
-<%= render_template('Run Command - Ansible Default', :command => "yum update #{input('package')}") %>
+<%= render_template('Run Command - Ansible Default', :command => "yum -y update #{input('package')}") %>

data/app/views/katello/api/v2/docker_tags/_base.json.rabl

@@ -18,7 +18,7 @@ child :docker_manifest => :manifest do
   attributes :schema_version, :digest, :manifest_type
 end
 
-child :repository => :repository do
+child :repositories => :repositories do
   attributes :id, :name, :full_path
 end
 

data/app/views/katello/api/v2/repositories/base.json.rabl

@@ -57,3 +57,11 @@ end
 child :content_view => :content_view do |_repo|
   attribute :id, :name
 end
+
+child :content_view_version do
+  attributes :id, :name, :content_view_id
+end
+
+child :environment do
+  attributes :id, :name
+end

data/config/routes/api/v2.rb

@@ -163,6 +163,9 @@ Katello::Engine.routes.draw do
             get :auto_complete_name
             get :compare
           end
+          member do
+            get :repositories
+          end
         end
 
         api_resources :environments, :only => [:index, :show, :create, :update, :destroy] do

data/db/migrate/20200402130013_add_repsoitory_docker_meta_tag_f_key.rb

@@ -0,0 +1,44 @@
+class AddRepsoitoryDockerMetaTagFKey < ActiveRecord::Migration[5.2]
+  def up
+    Katello::RepositoryDockerMetaTag.where.not(:repository_id => Katello::Repository.select(:id)).delete_all
+    Katello::RepositoryDockerMetaTag.where.not(:docker_meta_tag_id => Katello::DockerMetaTag.select(:id)).delete_all
+
+    add_foreign_key :katello_repository_docker_meta_tags, :katello_repositories, :column => :repository_id
+    add_foreign_key :katello_repository_docker_meta_tags, :katello_docker_meta_tags, :column => :docker_meta_tag_id
+
+    if Katello::DockerMetaTag.any?
+      query = "DELETE FROM katello_repository_docker_meta_tags T1
+                USING   katello_repository_docker_meta_tags T2
+                WHERE   T1.ctid < T2.ctid
+                AND T1.repository_id  = T2.repository_id
+                AND T1.docker_meta_tag_id = T2.docker_meta_tag_id;"
+      ActiveRecord::Base.connection.execute(query)
+    end
+    add_index :katello_repository_docker_meta_tags, [:repository_id, :docker_meta_tag_id], :unique => true, :name => 'repository_docker_meta_tags_rid_dmtid'
+
+    Katello::RepositoryDockerTag.where.not(:repository_id => Katello::Repository.select(:id)).delete_all
+    Katello::RepositoryDockerTag.where.not(:docker_tag_id => Katello::DockerTag.select(:id)).delete_all
+
+    add_foreign_key :katello_repository_docker_tags, :katello_repositories, :column => :repository_id
+    add_foreign_key :katello_repository_docker_tags, :katello_docker_tags, :column => :docker_tag_id
+
+    if Katello::DockerTag.any?
+      query = "DELETE FROM katello_repository_docker_tags T1
+                USING   katello_repository_docker_tags T2
+                WHERE   T1.ctid < T2.ctid
+                AND T1.repository_id = T2.repository_id
+                AND T1.docker_tag_id = T2.docker_tag_id;"
+      ActiveRecord::Base.connection.execute(query)
+    end
+    add_index :katello_repository_docker_tags, [:repository_id, :docker_tag_id], :unique => true, :name => 'repository_docker_tags_rid_dtid'
+  end
+
+  def down
+    remove_foreign_key :katello_repository_docker_meta_tags, :katello_repositories
+    remove_foreign_key :katello_repository_docker_meta_tags, :katello_docker_meta_tags
+    remove_index :katello_repository_docker_meta_tags,  [:repository_id, :docker_meta_tag_id]
+    remove_foreign_key :katello_repository_docker_tags, :katello_repositories
+    remove_foreign_key :katello_repository_docker_tags, :katello_docker_tags
+    remove_index :katello_repository_docker_tags, [:repository_id, :docker_tag_id]
+  end
+end

data/db/migrate/20200429153103_installed_package_bad_nvrea.rb

@@ -0,0 +1,14 @@
+class InstalledPackageBadNvrea < ActiveRecord::Migration[5.2]
+  def up
+    Katello::InstalledPackage.where(:epoch => "0").find_each do |pkg|
+      simple = Katello::Pulp::SimplePackage.new(pkg.attributes)
+      if pkg.nvrea != simple.nvrea
+        pkg.update_column(:nvrea, simple.nvrea)
+      end
+    end
+  end
+
+  def down
+    #noop
+  end
+end

data/db/migrate/20200501155054_installed_package_unique_nvrea.rb

@@ -0,0 +1,64 @@
+class InstalledPackageUniqueNvrea < ActiveRecord::Migration[5.2]
+  def fix_missing_attributes
+    #bug in dynflow may have resulted in old code running and not properly populating fields
+    # This block is basically copied from 20200129172534_add_epoch_version_release_arch_to_katello_installed_packages.rb
+    epoch_non_0 = ::Katello::Rpm.where.not(epoch: [0, nil]).pluck(:nvra, :epoch).to_h
+    installed_packages = []
+    ::Katello::InstalledPackage.where(:nvrea => nil).each do |pkg|
+      epoch = epoch_non_0[pkg.nvra] || "0"
+
+      attributes_hash = ::Katello::Util::Package.parse_nvrea(pkg.nvra)
+      attributes_hash[:epoch] = epoch
+      attributes_hash[:nvra] = pkg.nvra
+      if epoch == "0"
+        attributes_hash[:nvrea] = pkg.nvra
+      else
+        attributes_hash[:nvrea] = "#{pkg.name}-#{epoch}:#{attributes_hash[:version]}-"\
+                                  "#{attributes_hash[:release]}.#{attributes_hash[:arch]}"
+      end
+
+      installed_packages << ::Katello::InstalledPackage.new(attributes_hash)
+    end
+    ::Katello::InstalledPackage.import(installed_packages, validate: false, batch_size: 50_000,
+                                       on_duplicate_key_update: {conflict_target: [:nvra],
+                                                                 columns: [:nvrea, :epoch, :version, :release, :arch]})
+  end
+
+  def consolidate_duplicate_nvreas
+    host_installed_packages = []
+    deletable_installed_package_ids = []
+    Katello::InstalledPackage.having('count(nvrea) > 1').group(:nvrea).pluck(:nvrea).each do |nvrea|
+      found = Katello::InstalledPackage.includes(:host_installed_packages).where(:nvrea => nvrea).to_a
+      to_keep = found.pop
+      found.each do |duplicate|
+        duplicate.host_ids.each do |host_id|
+          host_installed_packages << {:installed_package_id => to_keep.id, :host_id => host_id}
+        end
+        deletable_installed_package_ids << duplicate.id
+      end
+    end
+    if host_installed_packages.any?
+      Katello::HostInstalledPackage.import(host_installed_packages, validate: false, on_duplicate_key_ignore: true)
+    end
+    if deletable_installed_package_ids.any?
+      Katello::HostInstalledPackage.where(installed_package_id: deletable_installed_package_ids).delete_all
+      Katello::InstalledPackage.where(id: deletable_installed_package_ids).delete_all
+    end
+  end
+
+  def up
+    fix_missing_attributes
+    #now there should be no NULL nvreas
+    change_column :katello_installed_packages, :nvrea, :string, :null => false
+
+    consolidate_duplicate_nvreas
+    add_index "katello_installed_packages", [:nvrea], :unique => true
+    remove_index "katello_installed_packages", [:nvra]
+  end
+
+  def down
+    remove_index "katello_installed_packages", [:nvrea]
+    add_index "katello_installed_packages", [:nvra], :unique => true
+    change_column :katello_installed_packages, :nvrea, :string, :null => true
+  end
+end

data/db/seeds.d/111-upgrade_tasks.rb

@@ -12,6 +12,7 @@ UpgradeTask.define_tasks(:katello) do
     {:name => 'katello:upgrades:3.12:remove_pulp2_notifier'},
     {:name => 'katello:upgrades:3.13:republish_deb_metadata'},
     {:name => 'katello:upgrades:3.15:set_sub_facet_dmi_uuid'},
-    {:name => 'katello:upgrades:3.15:reindex_rpm_modular'}
+    {:name => 'katello:upgrades:3.15:reindex_rpm_modular'},
+    {:name => 'katello:upgrades:3.16:update_applicable_el8_hosts'}
   ]
 end

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/docker-tags/details/docker-tag-environments.controller.js

@@ -20,36 +20,24 @@ angular.module('Bastion.docker-tags').controller('DockerTagEnvironmentsControlle
             'sort_order': 'ASC',
             'paged': false
         };
+
         var nutupane = new Nutupane(DockerTag, params, null, {disableAutoLoad: true});
 
         var renderTable = function () {
-            var ids = _.map($scope.tag.related_tags, 'id');
-            var newParams = {
-                'organization_id': CurrentOrganization,
-                'search': $location.search().search || "",
-                'sort_by': 'name',
-                'sort_order': 'ASC',
-                'paged': false,
-                'ids[]': ids
-            };
-            $scope.table = nutupane.table;
-            nutupane.setParams(newParams);
-            $scope.panel.loading = false;
-            if (!_.isEmpty(ids)) {
-                nutupane.refresh();
-            }
+            params.action = 'repositories';
+            params.id = $scope.tag.id;
+
+            nutupane.setParams(params);
+            nutupane.refresh();
         };
 
+        $scope.table = nutupane.table;
+
         $scope.controllerName = 'katello_docker_tags';
 
-        if ($scope.tag) {
+        $scope.tag.$promise.then(function() {
             $scope.panel.loading = false;
-        }
-
-        if ($scope.tag && $scope.tag.related_tags) {
             renderTable();
-        } else {
-            $scope.tag.$promise.then(renderTable);
-        }
+        });
     }
 ]);

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/docker-tags/details/views/docker-tag-environments.html

@@ -27,19 +27,19 @@
 
       <tbody>
         <tr bst-table-row
-            ng-repeat="tag in table.rows | filter:dockerTagFilter">
+            ng-repeat="repo in table.rows | filter:dockerTagFilter">
           <td bst-table-cell>
-            <a ui-sref="environment.details({environmentId: tag.environment.id})">
-              {{ tag.environment.name }}
+            <a ui-sref="environment.details({environmentId: repo.kt_environment.id})">
+              {{ repo.kt_environment.name }}
             </a>
           </td>
           <td bst-table-cell>
-            <a ui-sref="content-view.version.docker({contentViewId: tag.content_view_version.content_view_id, versionId: tag.content_view_version.id})">
-              {{ tag.content_view_version.name }}
+            <a ui-sref="content-view.version.docker({contentViewId: repo.content_view_version.content_view_id, versionId: repo.content_view_version.id})">
+              {{ repo.content_view_version.name }}
             </a>
           </td>
           <td bst-table-cell>
-            {{ tag.repository.full_path }}:{{ tag.name }}
+            {{ repo.full_path }}:{{ tag.name }}
           </td>
         </tr>
       </tbody>

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/docker-tags/docker-tags.controller.js

@@ -44,5 +44,13 @@ angular.module('Bastion.docker-tags').controller('DockerTagsController',
             }
             return versions.join(", ");
         };
+
+        $scope.getRepositoryNames = function (tag) {
+            return tag.repositories.map(function(repo) {
+                return repo.name;
+            }).filter(function(item, index, names) {
+                return names.indexOf(item) >= index;
+            }).sort().join(', ');
+        };
     }]
 );

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/docker-tags/docker-tags.factory.js

@@ -10,7 +10,7 @@
 angular.module('Bastion.docker-tags').factory('DockerTag',
     ['BastionResource', function (BastionResource) {
 
-        return BastionResource('katello/api/v2/docker_tags/:id/',
+        return BastionResource('katello/api/v2/docker_tags/:id/:action/',
             {id: '@id'},
             {
                 autocomplete: {method: 'GET', isArray: true, params: {id: 'auto_complete_search'}},

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/docker-tags/views/docker-tags.html

@@ -39,7 +39,7 @@
             {{ tag.product.name }}
           </td>
           <td bst-table-cell>
-            {{ tag.repository.name }}
+            {{ getRepositoryNames(tag) }}
           </td>
         </tr>
       </tbody>

data/lib/katello/permission_creator.rb

@@ -258,7 +258,12 @@ module Katello
                            'katello/api/v2/package_groups' => [:index, :show, :auto_complete_search, :compare],
                            'katello/api/v2/docker_manifests' => [:index, :show, :auto_complete_search, :compare],
                            'katello/api/v2/docker_manifest_lists' => [:index, :show, :auto_complete_search, :compare],
-                           'katello/api/v2/docker_tags' => [:index, :show, :auto_complete_search, :auto_complete_name, :compare],
+                           'katello/api/v2/docker_tags' => [:index,
+                                                            :show,
+                                                            :auto_complete_search,
+                                                            :auto_complete_name,
+                                                            :compare,
+                                                            :repositories],
                            'katello/api/v2/file_units' => [:index, :show, :auto_complete_search, :compare],
                            'katello/api/v2/ostree_branches' => [:index, :show, :auto_complete_search, :compare],
                            'katello/api/v2/errata' => [:index, :show, :auto_complete_search, :compare],

data/lib/katello/tasks/pulp3_content_switchover.rake

@@ -3,20 +3,14 @@ require File.expand_path("../engine", File.dirname(__FILE__))
 namespace :katello do
   desc "Runs a Pulp 3 migration of pulp3 hrefs to pulp ids for supported content types."
   task :pulp3_content_switchover => :environment do
-    migration_service = Katello::Pulp3::Migration.new(SmartProxy.pulp_master)
-    content_types = migration_service.content_types_for_migration
-
-    content_types.each do |content_type|
-      if content_type.model_class.where(migrated_pulp3_href: nil).any?
-        $stderr.print("ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n")
-        exit 1
+    begin
+      ActiveRecord::Base.transaction do
+        switchover_service = Katello::Pulp3::MigrationSwitchover.new(SmartProxy.pulp_master)
+        switchover_service.run
       end
-    end
-
-    content_types.each do |content_type|
-      content_type.model_class
-        .where.not("pulp_id=migrated_pulp3_href")
-        .update_all("pulp_id = migrated_pulp3_href")
+    rescue Katello::Pulp3::SwitchoverError => e
+      $stderr.print(e.message)
+      exit 1
     end
   end
 end

data/lib/katello/tasks/pulp3_post_migration_check.rake

@@ -6,25 +6,41 @@ namespace :katello do
     repository_types = Katello::Pulp3::Migration::REPOSITORY_TYPES
 
     repository_types.each do |type|
-      filter = 'version_href is NULL OR remote_href is NULL'
-
-      unless type == Katello::Repository::DOCKER_TYPE
-        filter += ' OR publication_href is NULL'
+      # check version
+      nil_version_ids = Katello::Repository.with_type(type).where(version_href: nil).pluck(:id)
+      unless nil_version_ids.empty?
+        $stderr.print("ERROR: #{type} repositories with ID [#{nil_version_ids.join(',')}] have a NULL value for version_href\n")
+        exit 1
       end
-      repositories = Katello::Repository.with_type(type).where(filter)
 
-      if repositories.any?
-        $stderr.print("ERROR: #{type} repository #{repositories.first.id} has a NULL value for remote_href, version_href, publication_href\n")
+      # check remote
+      nil_remote_ids = Katello::Repository.with_type(type).where(remote_href: nil).collect do |repo|
+        repo.id if repo.in_default_view? && !repo.root.url.nil?
+      end
+      nil_remote_ids.compact!
+      unless nil_remote_ids.empty?
+        $stderr.print("ERROR: #{type} repositories with ID [#{nil_remote_ids.join(',')}] have a NULL value for remote_href\n")
         exit 1
       end
 
+      # check publication
+      unless type == Katello::Repository::DOCKER_TYPE
+        nil_publication_ids = Katello::Repository.with_type(type).where(publication_href: nil).pluck(:id)
+        unless nil_publication_ids.empty?
+          $stderr.print("ERROR: #{type} repositories with ID [#{nil_publication_ids.join(',')}] have a NULL value for publication_href\n")
+          exit 1
+        end
+      end
+
+      # check distribution
       non_archived_repositories = Katello::Repository.with_type(type).non_archived
-      with_no_distribution_references = non_archived_repositories
-        .left_outer_joins(:distribution_references)
-        .where(katello_distribution_references: { id: nil })
+      with_no_distribution_references = non_archived_repositories.
+        left_outer_joins(:distribution_references).
+        where(katello_distribution_references: { id: nil }).
+        pluck(:id)
 
       if with_no_distribution_references.any?
-        $stderr.print("ERROR: A non-archive #{type} repository #{with_no_distribution_references.first.id} did not have a distribution reference\n")
+        $stderr.print("ERROR: Non-archived #{type} repositories with ID [#{with_no_distribution_references.join(',')}] have no distribution reference\n")
         exit 1
       end
     end

data/lib/katello/tasks/upgrades/3.16/update_applicable_el8_hosts.rake

@@ -0,0 +1,29 @@
+namespace :katello do
+  namespace :upgrades do
+    namespace '3.16' do
+      desc <<-DESCRIPTION
+      Update the applicability calculations for Rhel8 hosts.
+      This migration is to be run to address -> https://bugzilla.redhat.com/show_bug.cgi?id=1814095
+      DESCRIPTION
+      task :update_applicable_el8_hosts, [:input_file] => ["environment"] do
+        User.current = User.anonymous_api_admin
+
+        # Find me only those hosts that follow ALL the conditions below
+        # 1) Have a module stream enabled.
+        # 2) Bound to Non Library repositories. (i.e must belong to a CV thats not the default)
+        # 3) Bound repositories must have module streams in them
+        hosts = Host.joins(:content_facet => :content_facet_repositories).
+                 where("#{Host.table_name}.id" => ::Katello::HostAvailableModuleStream.enabled.select(:host_id)).
+                 where("#{Katello::ContentFacetRepository.table_name}.repository_id" =>
+                        ::Katello::Repository.joins(:repository_module_streams).
+                                          in_non_default_view.
+                                          non_archived)
+        hosts.each do |host|
+          available_streams = ::Katello::HostAvailableModuleStream.joins(:available_module_stream).
+                                                enabled.where(:host_id => host).select(:name, :stream)
+          ::Actions::Katello::Host::UploadProfiles.upload_modules_to_pulp(available_streams, host)
+        end
+      end
+    end
+  end
+end

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "3.15.0.rc2".freeze
+  VERSION = "3.15.2".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: katello
 version: !ruby/object:Gem::Version
-  version: 3.15.0.rc2
+  version: 3.15.2
 platform: ruby
 authors:
 - N/A
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-30 00:00:00.000000000 Z
+date: 2020-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -84,16 +84,16 @@ dependencies:
   name: foreman-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.14.1
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.14.1
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: dynflow
   requirement: !ruby/object:Gem::Requirement
@@ -1317,6 +1317,7 @@ files:
 - app/services/katello/pulp3/file_unit.rb
 - app/services/katello/pulp3/migration.rb
 - app/services/katello/pulp3/migration_plan.rb
+- app/services/katello/pulp3/migration_switchover.rb
 - app/services/katello/pulp3/module_stream.rb
 - app/services/katello/pulp3/package_group.rb
 - app/services/katello/pulp3/pulp_content_unit.rb
@@ -1892,6 +1893,9 @@ files:
 - db/migrate/20200109162354_drop_host_update_lock_setting.rb
 - db/migrate/20200121213430_katello_repository_rpms_id_big_int.rb
 - db/migrate/20200129172534_add_epoch_version_release_arch_to_katello_installed_packages.rb
+- db/migrate/20200402130013_add_repsoitory_docker_meta_tag_f_key.rb
+- db/migrate/20200429153103_installed_package_bad_nvrea.rb
+- db/migrate/20200501155054_installed_package_unique_nvrea.rb
 - db/seeds.d/101-locations.rb
 - db/seeds.d/102-organizations.rb
 - db/seeds.d/104-proxy.rb
@@ -4147,6 +4151,7 @@ files:
 - lib/katello/tasks/upgrades/3.13/republish_deb_metadata.rake
 - lib/katello/tasks/upgrades/3.15/reindex_rpm_modular.rake
 - lib/katello/tasks/upgrades/3.15/set_sub_facet_dmi_uuid.rake
+- lib/katello/tasks/upgrades/3.16/update_applicable_el8_hosts.rake
 - lib/katello/tasks/upgrades/3.8/clear_checksum_type.rake
 - lib/katello/tasks/virt_who_report.rake
 - lib/katello/url_constrained_cookie_store.rb
@@ -4643,11 +4648,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.5.2.1
 signing_key: 
 specification_version: 4
 summary: Content and Subscription Management plugin for Foreman