katello 3.15.1 → 3.16.0.rc2.1

data/app/models/katello/concerns/organization_extensions.rb

@@ -37,6 +37,7 @@ module Katello
 
         attr_accessor :statistics
 
+        scope :with_upstream_pools, -> { joins(:pools).merge(Katello::Pool.upstream).distinct }
         scope :having_name_or_label, ->(name_or_label) { where("name = :id or label = :id", :id => name_or_label) }
         scoped_search :on => :label, :complete_value => :true
 
@@ -69,6 +70,10 @@ module Katello
           self.providers.redhat.first
         end
 
+        def redhat_account_number
+          pools.upstream.pluck(:account_number).first
+        end
+
         def active_pools_count
           self.pools.where.not(:unmapped_guest => true).count
         end
@@ -118,6 +123,10 @@ module Katello
           end
         end
 
+        def clear_manifest_expired_notifications
+          NotificationBlueprint.find_by(name: 'manifest_expired_warning').notifications.where(:subject => self).destroy_all
+        end
+
         def redhat_repository_url
           redhat_provider.repository_url
         end

data/app/models/katello/concerns/pulp_database_unit.rb

@@ -43,6 +43,9 @@ module Katello
         self::CONTENT_TYPE
       end
 
+      def backend_identifier_field
+      end
+
       def content_unit_class
         "::Katello::Pulp::#{self.name.demodulize}".constantize
       end
@@ -101,15 +104,17 @@ module Katello
       end
 
       def import_for_repository(repository)
-        pulp_ids = []
+        pulp_id_href_map = {}
         service_class = SmartProxy.pulp_master!.content_service(content_type)
         fetch_only_ids = !repository.content_view.default? &&
-                         !repository.repository_type.unique_content_per_repo
+                         !repository.repository_type.unique_content_per_repo &&
+                         service_class.supports_id_fetch?
 
         service_class.pulp_units_batch_for_repo(repository, fetch_identifiers: fetch_only_ids).each do |units|
           units.each do |unit|
             unit = unit.with_indifferent_access
             pulp_id = unit[service_class.unit_identifier]
+            backend_identifier = unit.dig(service_class.backend_unit_identifier)
             unless fetch_only_ids
               model = Katello::Util::Support.active_record_retry do
                 self.where(:pulp_id => pulp_id).first_or_create
@@ -119,41 +124,34 @@ module Katello
               model.repository_id = repository.id unless many_repository_associations
               service.update_model(model)
             end
-            pulp_ids << pulp_id
+            pulp_id_href_map[pulp_id] = backend_identifier
           end
         end
-        sync_repository_associations(repository, :pulp_ids => pulp_ids) if self.many_repository_associations
+        sync_repository_associations(repository, :pulp_id_href_map => pulp_id_href_map) if self.many_repository_associations
       end
 
       def sync_repository_associations(repository, options = {})
         additive = options.fetch(:additive, false)
-        pulp_ids = options.fetch(:pulp_ids, nil)
-        associated_ids = with_pulp_id(pulp_ids).pluck(:id) if pulp_ids
-
-        table_name = self.repository_association_class.table_name
-        attribute_name = unit_id_field
+        pulp_id_href_map = options.dig(:pulp_id_href_map) || {}
+        pulp_ids = options.dig(:pulp_ids) || pulp_id_href_map.try(:keys)
+        ids_for_repository = with_pulp_id(pulp_ids).pluck(:id, :pulp_id)
+        associated_ids = ids_for_repository.map(&:first)
+        id_href_map_for_repository = {}
+        ids_for_repository.each { |id_href| id_href_map_for_repository[id_href[0]] = id_href[1] }
+        id_href_map_for_repository.each_pair { |k, v| id_href_map_for_repository[k] = pulp_id_href_map[v] }
 
         existing_ids = self.repository_association_class.uncached do
-          self.repository_association_class.where(:repository_id => repository).pluck(attribute_name)
-        end
-        new_ids = associated_ids - existing_ids
-        delete_ids = existing_ids - associated_ids
-
-        queries = []
-
-        if delete_ids.any? && !additive
-          queries << "DELETE FROM #{table_name} WHERE repository_id=#{repository.id} AND #{attribute_name} IN (#{delete_ids.join(', ')})"
-        end
-
-        unless new_ids.empty?
-          inserts = new_ids.map { |unit_id| "(#{unit_id.to_i}, #{repository.id.to_i}, '#{Time.now.utc.to_s(:db)}', '#{Time.now.utc.to_s(:db)}')" }
-          queries << "INSERT INTO #{table_name} (#{attribute_name}, repository_id, created_at, updated_at) VALUES #{inserts.join(', ')}"
+          self.repository_association_class.where(:repository_id => repository).pluck(unit_id_field)
         end
 
         ActiveRecord::Base.transaction do
-          queries.each do |query|
+          if !additive && (delete_ids = existing_ids - associated_ids).any?
+            query = "DELETE FROM #{self.repository_association_class.table_name} WHERE repository_id=#{repository.id} AND #{unit_id_field} IN (#{delete_ids.join(', ')})"
             ActiveRecord::Base.connection.execute(query)
           end
+          unless (new_ids = associated_ids - existing_ids).empty?
+            self.repository_association_class.import(db_columns_sync, db_values(new_ids, id_href_map_for_repository, repository), validate: false)
+          end
         end
       end
 
@@ -161,29 +159,58 @@ module Katello
         if many_repository_associations
           delete_query = "delete from #{repository_association_class.table_name} where repository_id = #{dest_repo.id} and
                          #{unit_id_field} not in (select #{unit_id_field} from #{repository_association_class.table_name} where repository_id = #{source_repo.id})"
-
-          insert_query = "insert into #{repository_association_class.table_name} (repository_id, #{unit_id_field})
-                          select #{dest_repo.id} as repository_id, #{unit_id_field} from #{repository_association_class.table_name}
-                          where repository_id = #{source_repo.id} and #{unit_id_field} not in (select #{unit_id_field}
-                          from #{repository_association_class.table_name} where repository_id = #{dest_repo.id})"
+          ActiveRecord::Base.transaction do
+            ActiveRecord::Base.connection.execute(delete_query)
+            self.repository_association_class.import(db_columns_copy, db_values_copy(source_repo, dest_repo), validate: false)
+          end
         else
           columns = column_names - ["id", "pulp_id", "created_at", "updated_at", "repository_id"]
-
-          delete_query = "delete from #{self.table_name} where repository_id = #{dest_repo.id} and
+          queries = []
+          queries << "delete from #{self.table_name} where repository_id = #{dest_repo.id} and
                           pulp_id not in (select pulp_id from #{self.table_name} where repository_id = #{source_repo.id})"
-          insert_query = "insert into #{self.table_name} (repository_id, pulp_id, #{columns.join(',')})
+          queries << "insert into #{self.table_name} (repository_id, pulp_id, #{columns.join(',')})
                     select #{dest_repo.id} as repository_id, pulp_id, #{columns.join(',')} from #{self.table_name}
                     where repository_id = #{source_repo.id} and pulp_id not in (select pulp_id
                     from #{self.table_name} where repository_id = #{dest_repo.id})"
-
+          ActiveRecord::Base.transaction do
+            queries.each do |query|
+              ActiveRecord::Base.connection.execute(query)
+            end
+          end
         end
-        ActiveRecord::Base.connection.execute(delete_query)
-        ActiveRecord::Base.connection.execute(insert_query)
       end
 
       def with_pulp_id(unit_pulp_ids)
         where('pulp_id in (?)', unit_pulp_ids)
       end
+
+      def db_columns_sync
+        [unit_id_field, backend_identifier_field, :repository_id, :created_at, :updated_at].compact
+      end
+
+      def db_columns_copy
+        [unit_id_field, backend_identifier_field, :repository_id].compact
+      end
+
+      def db_values_copy(source_repo, dest_repo)
+        db_values = []
+        existing_unit_ids = self.repository_association_class.where(repository: dest_repo).pluck(unit_id_field)
+        if existing_unit_ids.empty?
+          new_units = self.repository_association_class.where(repository: source_repo)
+        else
+          new_units = self.repository_association_class.where(repository: source_repo).where.not("#{unit_id_field} in (?) ", existing_unit_ids)
+        end
+        unit_backend_identifier_field = backend_identifier_field
+        unit_identifier_filed = unit_id_field
+        new_units.each do |unit|
+          db_values << [unit[unit_identifier_filed], unit[unit_backend_identifier_field], dest_repo.id].compact
+        end
+        db_values
+      end
+
+      def db_values(new_ids, pulp_id_href_map, repository)
+        new_ids.map { |unit_id| [unit_id.to_i, pulp_id_href_map.dig(unit_id), repository.id.to_i, Time.now.utc.to_s(:db), Time.now.utc.to_s(:db)].compact }
+      end
     end
   end
 end

data/app/models/katello/concerns/redhat_extensions.rb

@@ -58,7 +58,8 @@ module Katello
               distribution_variant: variant,
               "#{Katello::Product.table_name}.id": product_id
             ).first
-          distro.to_hash(host.content_source) if distro
+
+          distro&.to_hash(host.content_source)
         end
       end
 

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -124,6 +124,8 @@ module Katello
           config.ssl_client_key = ::Cert::Certs.ssl_client_key
           config.debugging = true
           config.logger = ::Foreman::Logging.logger('katello/pulp_rest')
+          config.username = self.setting(PULP3_FEATURE, 'username')
+          config.password = self.setting(PULP3_FEATURE, 'password')
         end
       end
 
@@ -195,7 +197,7 @@ module Katello
         pulp3_uri!.host
       end
 
-      def pulp3_url(path = '/pulp/api/v3/')
+      def pulp3_url(path = '/pulp/api/v3')
         pulp_url = self.setting(PULP3_FEATURE, 'pulp_url')
         path.blank? ? pulp_url : "#{pulp_url.sub(%r|/$|, '')}/#{path.sub(%r|^/|, '')}"
       end
@@ -294,7 +296,8 @@ module Katello
 
       def last_sync_time
         task = sync_tasks.where.not(:ended_at => nil).where(:result => 'success').order(:ended_at).last
-        task.ended_at unless task.nil?
+
+        task&.ended_at
       end
 
       def environment_syncable?(env)

data/app/models/katello/concerns/subscription_facet_host_extensions.rb

@@ -63,7 +63,7 @@ module Katello
       end
 
       def update_candlepin_associations(consumer_params = nil)
-        content_facet.save! if content_facet
+        content_facet&.save!
 
         auto_attach_enabled_via_checkin = consumer_params.try(:[], 'autoheal')
 

data/app/models/katello/content.rb

@@ -15,7 +15,7 @@ module Katello
 
     def update_repository_names
       root_repositories.each do |root|
-        root.update_attributes!(:name => root.calculate_updated_name)
+        root.update!(:name => root.calculate_updated_name)
       end
     end
 
@@ -28,7 +28,7 @@ module Katello
     end
 
     def redhat?
-      self.products.first.try(:redhat?)
+      products.redhat.any?
     end
 
     def propagate_name_change?

data/app/models/katello/content_view.rb

@@ -462,7 +462,8 @@ module Katello
 
     def update_cp_content(env)
       view_env = content_view_environment(env)
-      view_env.update_cp_content if view_env
+
+      view_env&.update_cp_content
     end
 
     # Associate an environment with this content view.  This can occur whenever
@@ -508,7 +509,7 @@ module Katello
                                            :components => components
                                           )
 
-      update_attributes(:next_version => major.to_i + 1) unless major.to_i < next_version
+      update(:next_version => major.to_i + 1) unless major.to_i < next_version
       version
     end
 

data/app/models/katello/content_view_erratum_filter.rb

@@ -26,8 +26,37 @@ module Katello
       end
     end
 
+    def content_unit_pulp_ids(repo)
+      if filter_by_id?
+        errata_ids = erratum_rules.map(&:errata_id)
+        errata_pulp_ids = errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
+      else
+        clauses = []
+        clauses << errata_from
+        clauses << errata_to
+        clauses << types_clause
+        package_filenames = Erratum.list_filenames_by_clauses(repo, clauses.compact)
+        errata_pulp_ids = errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
+      end
+      errata_pulp_ids
+    end
+
     private
 
+    def rpms_by_filename(repo, package_filenames)
+      query_params = package_filenames.map { |filename| "%#{filename}" }
+      repo.rpms.where("filename ILIKE ANY ( array[?] )", query_params)
+    end
+
+    def errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
+      rpms_by_filename(repo, package_filenames).pluck(:pulp_id)
+    end
+
+    def errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
+      package_filenames = Katello::ErratumPackage.joins(:erratum).where("#{Erratum.table_name}.errata_id" => errata_ids).pluck(:filename)
+      rpms_by_filename(repo, package_filenames).pluck(:pulp_id)
+    end
+
     def erratum_arel
       ::Katello::Erratum.arel_table
     end

data/app/models/katello/content_view_package_filter.rb

@@ -7,7 +7,11 @@ module Katello
     validates_lengths_from_database
 
     # Returns a set of Pulp/MongoDB conditions to filter out packages in the
-    # repo repository that match parameters
+    # repo repository that match parameters if the repository content type
+    # is managed by pulp2.
+    #
+    # Returns a set of pulpcore hrefs (pulp_ids) if the repository content
+    # type is managed by pulpcore.
     #
     # @param repo [Repository] a repository containing packages to filter
     # @return [Array] an array of hashes with MongoDB conditions
@@ -20,6 +24,7 @@ module Katello
       if self.original_packages
         package_filenames.concat(repo.packages_without_errata.map(&:filename))
       end
+
       ContentViewPackageFilter.generate_rpm_clauses(package_filenames)
     end
 
@@ -27,6 +32,16 @@ module Katello
       self[:original_packages] = value
     end
 
+    def content_unit_pulp_ids(repo)
+      package_filenames = []
+      self.package_rules.each do |rule|
+        package_filenames.concat(query_rpms(repo, rule))
+      end
+
+      rpms = Rpm.in_repositories(repo)
+      rpms.where(filename: package_filenames).where(:modular => false).pluck(:pulp_id).flatten.uniq
+    end
+
     def self.generate_rpm_clauses(package_filenames = [])
       { 'filename' => { "$in" => package_filenames } } unless package_filenames.empty?
     end

data/app/models/katello/content_view_package_group_filter.rb

@@ -10,5 +10,12 @@ module Katello
       package_group_ids = package_group_rules.reject { |rule| rule.uuid.blank? }.flat_map.map(&:uuid)
       { "_id" => { "$in" => package_group_ids } } unless package_group_ids.empty?
     end
+
+    def content_unit_pulp_ids(repo)
+      package_group_hrefs = package_group_rules.reject { |rule| rule.uuid.blank? }.flat_map.map(&:uuid)
+      package_group_names = repo.package_groups.
+        where(:pulp_id => package_group_hrefs).collect { |package_group| package_group.package_names }.flatten.uniq
+      repo.rpms.where(:name => package_group_names).pluck(:pulp_id).compact
+    end
   end
 end

data/app/models/katello/content_view_version.rb

@@ -314,6 +314,7 @@ module Katello
           content_counts[content_type::CONTENT_TYPE] = content_type.in_repositories(self.repositories.archived).count
         end
       end
+      self.content_counts[PuppetModule::CONTENT_TYPE] = self.puppet_modules.count
       save!
     end
 

data/app/models/katello/erratum.rb

@@ -8,6 +8,7 @@ module Katello
 
     TYPES = [SECURITY, BUGZILLA, ENHANCEMENT].flatten.freeze
     CONTENT_TYPE = "erratum".freeze
+    BACKEND_IDENTIFIER_FIELD = "erratum_pulp3_href".freeze
 
     has_many :content_facet_errata, :class_name => "Katello::ContentFacetErratum", :dependent => :destroy, :inverse_of => :content_facet
     has_many :content_facets, :through => :content_facet_errata, :class_name => "Katello::Host::ContentFacet", :source => :content_facet
@@ -47,13 +48,17 @@ module Katello
     scope :security, -> { of_type(Erratum::SECURITY) }
     scope :bugfix, -> { of_type(Erratum::BUGZILLA) }
     scope :enhancement, -> { of_type(Erratum::ENHANCEMENT) }
-    scope :modular, -> { where(:id => Erratum.joins(:packages => :module_stream_errata_packages)) }
+    scope :modular, -> { where(:id => joins(:packages => :module_stream_errata_packages)) }
     scope :non_modular, -> { where.not(:id => modular) }
 
     def self.content_facet_association_class
       ContentFacetErratum
     end
 
+    def self.backend_identifier_field
+      SmartProxy.pulp_master!.content_service(CONTENT_TYPE).backend_unit_identifier ? BACKEND_IDENTIFIER_FIELD.to_sym : nil
+    end
+
     def self.applicable_to_hosts(hosts)
       # Note: ContentFacetErrata actually holds the "Applicable Errata" to that host
       # It is not the errata "belonging" to the host. Its rather the errata that is "applicable"

data/app/models/katello/erratum_cve.rb

@@ -1,5 +1,9 @@
 module Katello
   class ErratumCve < Katello::Model
     belongs_to :erratum, :inverse_of => :cves, :class_name => 'Katello::Erratum'
+
+    class Jail < ::Safemode::Jail
+      allow :erratum, :cve_id, :href
+    end
   end
 end

data/app/models/katello/events/generate_host_applicability.rb

@@ -0,0 +1,28 @@
+module Katello
+  module Events
+    class GenerateHostApplicability
+      EVENT_TYPE = 'generate_host_applicability'.freeze
+
+      def self.retry_seconds
+        180
+      end
+
+      def initialize(object_id)
+      end
+
+      def run
+        return if ::Katello::ApplicableHostQueue.queue_depth == 0
+
+        begin
+          while ::Katello::ApplicableHostQueue.queue_depth != 0
+            hosts = ::Katello::ApplicableHostQueue.pop_hosts
+            ForemanTasks.async_task(::Actions::Katello::Applicability::Hosts::BulkGenerate, host_ids: hosts.map(&:host_id))
+          end
+        rescue => e
+          self.retry = true if e.is_a?(ForemanTasks::Lock::LockConflict)
+          raise e
+        end
+      end
+    end
+  end
+end

data/app/models/katello/glue/candlepin/activation_key.rb

@@ -30,7 +30,7 @@ module Katello
         key_pools = Resources::Candlepin::ActivationKey.get(self.cp_id)[0][:pools]
         key_pools.map do |key_pool|
           {
-            :amount => (key_pool[:quantity] ? key_pool[:quantity] : 0),
+            :amount => (key_pool[:quantity] || 0),
             :id => key_pool[:poolId]
           }.with_indifferent_access
         end

data/app/models/katello/glue/candlepin/pool.rb

@@ -69,10 +69,8 @@ module Katello
           end
         end
 
-        if json["calculatedAttributes"]
-          json["calculatedAttributes"].each do |key|
-            json["suggested_quantity"] = json["calculatedAttributes"]["suggested_quantity"].to_i if key == 'suggested_quantity'
-          end
+        json["calculatedAttributes"]&.each do |key|
+          json["suggested_quantity"] = json["calculatedAttributes"]["suggested_quantity"].to_i if key == 'suggested_quantity'
         end
 
         json["product_id"] = json["productId"] if json["productId"]
@@ -91,7 +89,7 @@ module Katello
         providers.any?
       end
 
-      # rubocop:disable MethodLength,Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
       def import_data(index_hosts_and_activation_keys = false)
         pool_attributes = {}.with_indifferent_access
         pool_json = self.backend_data
@@ -140,13 +138,13 @@ module Katello
 
         pool_attributes['stack_id'] = pool_json['stackId']
         exceptions = pool_attributes.keys.map(&:to_sym) - self.attribute_names.map(&:to_sym)
-        self.update_attributes(pool_attributes.except!(*exceptions))
+        self.update(pool_attributes.except!(*exceptions))
         self.save!
         self.create_activation_key_associations if index_hosts_and_activation_keys
         self.create_product_associations
         self.import_hosts if index_hosts_and_activation_keys
       end
-      # rubocop:enable MethodLength,Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength,Metrics/AbcSize
 
       def create_product_associations
         products = self.backend_data["providedProducts"] + self.backend_data["derivedProvidedProducts"]
@@ -166,7 +164,7 @@ module Katello
       def import_hosts
         uuids = Resources::Candlepin::Pool.consumer_uuids(self.cp_id)
 
-        sub_facet_ids_from_cp, host_ids_from_cp = Katello::Host::SubscriptionFacet.where(:uuid => uuids).pluck([:id, :host_id]).transpose
+        sub_facet_ids_from_cp, host_ids_from_cp = Katello::Host::SubscriptionFacet.where('uuid in (?)', uuids).pluck([:id, :host_id]).transpose
         sub_facet_ids_from_cp ||= []
         host_ids_from_cp ||= []