katello 3.16.0.rc1.1 → 3.16.0.rc4

data/app/lib/katello/logging.rb

@@ -0,0 +1,14 @@
+module Katello
+  module Logging
+    def self.time(message, data: {}, logger: Rails.logger, level: :info)
+      start = Time.now
+
+      yield
+
+      data[:duration] = ((Time.now - start) * 1000).truncate(2)
+      data_string = data.map { |k, v| "#{k}=#{v}" }.join(' ')
+
+      logger.send(level, "#{message} #{data_string}")
+    end
+  end
+end

data/app/lib/katello/resources/candlepin.rb

@@ -142,7 +142,7 @@ module Katello
           def upstream_consumer
             fail _("Current organization not set.") unless Organization.current
             upstream_consumer = Organization.current.owner_details['upstreamConsumer']
-            fail _("Current organization has no manifest imported.") unless upstream_consumer
+            fail Katello::Errors::NoManifestImported unless upstream_consumer
 
             upstream_consumer
           end

data/app/lib/katello/resources/candlepin/upstream_consumer.rb

@@ -9,6 +9,12 @@ module Katello
             super(id)
           end
 
+          def ping
+            resource.head
+          rescue RestClient::Unauthorized, RestClient::Gone
+            raise ::Katello::Errors::UpstreamConsumerGone
+          end
+
           # Overrides the HttpResource get method to check if the upstream
           # consumer exists.
           def get(params)

data/app/lib/katello/util/errata.rb

@@ -45,6 +45,19 @@ module Katello
           return errata_list
         end
       end
+
+      def filter_errata_by_pulp_href(errata, package_pulp_hrefs)
+        return [] if package_pulp_hrefs.empty?
+        rpms = Katello::Rpm.where(:pulp_id => package_pulp_hrefs)
+        rpm_filenames = rpms.map { |rpm| File.basename(rpm.filename) }
+        matching_errata = []
+        errata.each do |erratum|
+          if erratum.packages.any? && (erratum.packages.pluck(:filename) - rpm_filenames).empty?
+            matching_errata << erratum
+          end
+        end
+        matching_errata
+      end
     end
   end
 end

data/app/lib/katello/util/pulpcore_content_filters.rb

@@ -0,0 +1,32 @@
+module Katello
+  module Util
+    module PulpcoreContentFilters
+      def filter_distribution_trees_by_pulp_hrefs(distributiontree_results, _content_pulp_hrefs)
+        distributiontree_results.collect { |result| result.pulp_href }.flatten.uniq
+      end
+
+      def filter_package_groups_by_pulp_href(package_groups, package_pulp_hrefs)
+        rpms = Katello::Rpm.where(:pulp_id => package_pulp_hrefs)
+        package_groups.reject do |package_group|
+          (package_group.package_names & rpms.pluck(:name)).empty?
+        end
+      end
+
+      def filter_package_environments_by_pulp_hrefs(packageenvironment_results, package_pulp_hrefs)
+        matching_package_env_groups = []
+
+        packageenvironment_results.each do |result|
+          if (result.packagegroups & package_pulp_hrefs).any?
+            matching_package_env_groups << result.pulp_href
+          end
+        end
+
+        matching_package_env_groups.flatten.uniq
+      end
+
+      def filter_metadatafiles_by_pulp_hrefs(metadatafiles_results, _package_pulp_hrefs)
+        metadatafiles_results.collect { |result| result.pulp_href }.flatten.uniq
+      end
+    end
+  end
+end

data/app/models/katello/concerns/host_managed_extensions.rb

@@ -50,6 +50,7 @@ module Katello
 
         before_save :correct_puppet_environment
         before_validation :correct_kickstart_repository
+        before_update :check_host_registration, :if => proc { organization_id_changed? }
 
         scope :with_pools_expiring_in_days, ->(days) { joins(:pools).merge(Katello::Pool.expiring_in_days(days)).distinct }
 
@@ -76,6 +77,12 @@ module Katello
         end
       end
 
+      def check_host_registration
+        if subscription_facet
+          fail ::Katello::Errors::HostRegisteredException
+        end
+      end
+
       def correct_kickstart_repository
         return unless content_facet
 

data/app/models/katello/concerns/http_proxy_extensions.rb

@@ -8,6 +8,7 @@ module Katello
           :inverse_of => :http_proxy, :dependent => :nullify
         after_update :update_default_proxy_setting
         after_commit :update_repository_proxy_details
+        before_destroy :remove_references_to_proxy
 
         def self.default_global_content_proxy
           if Setting[:content_default_http_proxy]
@@ -16,6 +17,34 @@ module Katello
         end
       end
 
+      def repositories_with_proxy(proxy_id)
+        root_repos = RootRepository.with_selected_proxy(proxy_id)
+
+        if self == HttpProxy.default_global_content_proxy
+          root_repos += RootRepository.with_global_proxy
+        end
+
+        root_repos
+      end
+
+      def remove_references_to_proxy
+        root_repos = repositories_with_proxy(nil).uniq.sort
+
+        setting = Setting.find_by(name: 'content_default_http_proxy')
+        if setting&.value && setting.value == self.name
+          setting.update_attribute(:value, '')
+        end
+
+        unless root_repos.empty?
+          ForemanTasks.async_task(
+            ::Actions::BulkAction,
+            ::Actions::Katello::Repository::Update,
+            root_repos,
+            http_proxy_policy: Katello::RootRepository::GLOBAL_DEFAULT_HTTP_PROXY,
+            http_proxy_id: nil)
+        end
+      end
+
       def update_default_proxy_setting
         changes = self.previous_changes
         if changes.key?(:name)
@@ -32,13 +61,7 @@ module Katello
         changes = self.previous_changes
         if changes.key?(:url) || changes.key?(:username) || changes.key?(:password)
 
-          root_repos = RootRepository.with_selected_proxy(id)
-
-          if self == HttpProxy.default_global_content_proxy
-            root_repos += RootRepository.with_global_proxy
-          end
-
-          repos = root_repos.uniq.collect(&:library_instance)
+          repos = repositories_with_proxy(id).uniq.collect(&:library_instance)
 
           unless repos.empty?
             ForemanTasks.async_task(

data/app/models/katello/concerns/organization_extensions.rb

@@ -82,6 +82,20 @@ module Katello
           self.providers.anonymous.first
         end
 
+        def manifest_imported?
+          owner_details['upstreamConsumer'].present?
+        end
+
+        def manifest_expired?
+          manifest_expiry = owner_details.dig(:upstreamConsumer, :idCert, :serial, :expiration)
+
+          if manifest_expiry
+            DateTime.parse(manifest_expiry) < DateTime.now
+          else
+            false
+          end
+        end
+
         def manifest_history
           imports.map { |i| OpenStruct.new(i) }
         end

data/app/models/katello/concerns/pulp_database_unit.rb

@@ -44,7 +44,6 @@ module Katello
       end
 
       def backend_identifier_field
-        nil
       end
 
       def content_unit_class
@@ -108,7 +107,8 @@ module Katello
         pulp_id_href_map = {}
         service_class = SmartProxy.pulp_master!.content_service(content_type)
         fetch_only_ids = !repository.content_view.default? &&
-                         !repository.repository_type.unique_content_per_repo
+                         !repository.repository_type.unique_content_per_repo &&
+                         service_class.supports_id_fetch?
 
         service_class.pulp_units_batch_for_repo(repository, fetch_identifiers: fetch_only_ids).each do |units|
           units.each do |unit|

data/app/models/katello/content_view_erratum_filter.rb

@@ -26,8 +26,37 @@ module Katello
       end
     end
 
+    def content_unit_pulp_ids(repo)
+      if filter_by_id?
+        errata_ids = erratum_rules.map(&:errata_id)
+        errata_pulp_ids = errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
+      else
+        clauses = []
+        clauses << errata_from
+        clauses << errata_to
+        clauses << types_clause
+        package_filenames = Erratum.list_filenames_by_clauses(repo, clauses.compact)
+        errata_pulp_ids = errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
+      end
+      errata_pulp_ids
+    end
+
     private
 
+    def rpms_by_filename(repo, package_filenames)
+      query_params = package_filenames.map { |filename| "%#{filename}" }
+      repo.rpms.where("filename ILIKE ANY ( array[?] )", query_params)
+    end
+
+    def errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
+      rpms_by_filename(repo, package_filenames).pluck(:pulp_id)
+    end
+
+    def errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
+      package_filenames = Katello::ErratumPackage.joins(:erratum).where("#{Erratum.table_name}.errata_id" => errata_ids).pluck(:filename)
+      rpms_by_filename(repo, package_filenames).pluck(:pulp_id)
+    end
+
     def erratum_arel
       ::Katello::Erratum.arel_table
     end

data/app/models/katello/content_view_package_filter.rb

@@ -7,7 +7,11 @@ module Katello
     validates_lengths_from_database
 
     # Returns a set of Pulp/MongoDB conditions to filter out packages in the
-    # repo repository that match parameters
+    # repo repository that match parameters if the repository content type
+    # is managed by pulp2.
+    #
+    # Returns a set of pulpcore hrefs (pulp_ids) if the repository content
+    # type is managed by pulpcore.
     #
     # @param repo [Repository] a repository containing packages to filter
     # @return [Array] an array of hashes with MongoDB conditions
@@ -20,6 +24,7 @@ module Katello
       if self.original_packages
         package_filenames.concat(repo.packages_without_errata.map(&:filename))
       end
+
       ContentViewPackageFilter.generate_rpm_clauses(package_filenames)
     end
 
@@ -27,6 +32,16 @@ module Katello
       self[:original_packages] = value
     end
 
+    def content_unit_pulp_ids(repo)
+      package_filenames = []
+      self.package_rules.each do |rule|
+        package_filenames.concat(query_rpms(repo, rule))
+      end
+
+      rpms = Rpm.in_repositories(repo)
+      rpms.where(filename: package_filenames).where(:modular => false).pluck(:pulp_id).flatten.uniq
+    end
+
     def self.generate_rpm_clauses(package_filenames = [])
       { 'filename' => { "$in" => package_filenames } } unless package_filenames.empty?
     end

data/app/models/katello/content_view_package_group_filter.rb

@@ -10,5 +10,12 @@ module Katello
       package_group_ids = package_group_rules.reject { |rule| rule.uuid.blank? }.flat_map.map(&:uuid)
       { "_id" => { "$in" => package_group_ids } } unless package_group_ids.empty?
     end
+
+    def content_unit_pulp_ids(repo)
+      package_group_hrefs = package_group_rules.reject { |rule| rule.uuid.blank? }.flat_map.map(&:uuid)
+      package_group_names = repo.package_groups.
+        where(:pulp_id => package_group_hrefs).collect { |package_group| package_group.package_names }.flatten.uniq
+      repo.rpms.where(:name => package_group_names).pluck(:pulp_id).compact
+    end
   end
 end

data/app/models/katello/events/generate_host_applicability.rb

@@ -7,16 +7,17 @@ module Katello
         180
       end
 
-      def initialize(host_id)
-        @host = ::Host.find_by_id(host_id)
-        Rails.logger.warn "Host not found for ID #{object_id}" if @host.nil?
+      def initialize(object_id)
       end
 
       def run
-        return unless @host
+        return if ::Katello::ApplicableHostQueue.queue_depth == 0
 
         begin
-          ForemanTasks.async_task(::Actions::Katello::Applicability::Host::Generate, host_id: @host.id)
+          while ::Katello::ApplicableHostQueue.queue_depth != 0
+            hosts = ::Katello::ApplicableHostQueue.pop_hosts
+            ForemanTasks.async_task(::Actions::Katello::Applicability::Hosts::BulkGenerate, host_ids: hosts.map(&:host_id))
+          end
         rescue => e
           self.retry = true if e.is_a?(ForemanTasks::Lock::LockConflict)
           raise e

data/app/models/katello/glue/candlepin/candlepin_object.rb

@@ -36,11 +36,15 @@ module Katello
 
           objects = self.in_organization(org)
           objects.each do |item|
-            if candlepin_ids.include?(item.cp_id)
-              item.import_data
-              item.import_managed_associations if import_managed_associations && item.respond_to?(:import_managed_associations)
-            else
-              item.destroy
+            exists_in_candlepin = candlepin_ids.include?(item.cp_id)
+
+            Katello::Logging.time("Imported #{self}", data: { cp_id: item.cp_id, destroyed: !exists_in_candlepin }) do
+              if exists_in_candlepin
+                item.import_data
+                item.import_managed_associations if import_managed_associations && item.respond_to?(:import_managed_associations)
+              else
+                item.destroy
+              end
             end
           end
         end

data/app/models/katello/glue/candlepin/pool.rb

@@ -164,7 +164,7 @@ module Katello
       def import_hosts
         uuids = Resources::Candlepin::Pool.consumer_uuids(self.cp_id)
 
-        sub_facet_ids_from_cp, host_ids_from_cp = Katello::Host::SubscriptionFacet.where(:uuid => uuids).pluck([:id, :host_id]).transpose
+        sub_facet_ids_from_cp, host_ids_from_cp = Katello::Host::SubscriptionFacet.where('uuid in (?)', uuids).pluck([:id, :host_id]).transpose
         sub_facet_ids_from_cp ||= []
         host_ids_from_cp ||= []
 
@@ -187,8 +187,13 @@ module Katello
       end
 
       def import_managed_associations
-        import_hosts
-        create_activation_key_associations
+        Katello::Logging.time("Imported host associations") do
+          import_hosts
+        end
+
+        Katello::Logging.time("Imported activation key associations") do
+          create_activation_key_associations
+        end
       end
 
       def hosts

data/app/models/katello/glue/provider.rb

@@ -132,11 +132,18 @@ module Katello
         cp_products = cp_products.select { |prod| Glue::Candlepin::Product.engineering_product_id?(prod['id']) }
 
         prod_content_importer = Katello::ProductContentImporter.new
-        cp_products.each do |product_json|
-          product = import_product(product_json)
-          prod_content_importer.add_product_content(product, product_json['productContent']) if product.redhat?
+
+        Katello::Logging.time("Imported #{cp_products.size} products") do
+          cp_products.each do |product_json|
+            product = import_product(product_json)
+            prod_content_importer.add_product_content(product, product_json['productContent']) if product.redhat?
+          end
+        end
+
+        Katello::Logging.time("Imported product content") do
+          prod_content_importer.import
         end
-        prod_content_importer.import
+
         self.index_subscriptions(self.organization)
         prod_content_importer
       end

data/app/models/katello/host/content_facet.rb

@@ -82,10 +82,10 @@ module Katello
       def update_bound_repositories(repos)
         self.bound_repositories = repos
         self.save!
-        self.propagate_yum_repos
         if SETTINGS[:katello][:katello_applicability]
           ::Katello::EventQueue.push_event(::Katello::Events::GenerateHostApplicability::EVENT_TYPE, self.host.id)
         else
+          self.propagate_yum_repos
           ForemanTasks.async_task(Actions::Katello::Host::GenerateApplicability, [self.host])
         end
       end

data/app/models/katello/host_queue_element.rb

@@ -0,0 +1,4 @@
+module Katello
+  class HostQueueElement < Katello::Model
+  end
+end

data/app/models/katello/package_group.rb

@@ -25,7 +25,8 @@ module Katello
     end
 
     def package_names
-      group = Pulp::PackageGroup.new(self.pulp_id)
+      service_class = SmartProxy.pulp_master!.content_service(CONTENT_TYPE)
+      group = service_class.new(self.pulp_id)
       group.default_package_names + group.conditional_package_names + group.optional_package_names + group.mandatory_package_names
     end
   end

data/app/models/katello/pool.rb

@@ -9,7 +9,7 @@ module Katello
     has_many :pool_activation_keys, :class_name => "Katello::PoolActivationKey", :dependent => :destroy, :inverse_of => :pool
     has_many :activation_keys, :through => :pool_activation_keys, :class_name => "Katello::ActivationKey"
 
-    has_many :subscription_facet_pools, :class_name => "Katello::SubscriptionFacetPool", :dependent => :destroy
+    has_many :subscription_facet_pools, :class_name => "Katello::SubscriptionFacetPool", :dependent => :delete_all
     has_many :subscription_facets, :through => :subscription_facet_pools
 
     belongs_to :organization, :class_name => 'Organization', :inverse_of => :pools

data/app/models/katello/pulp3/content_guard.rb

@@ -0,0 +1,14 @@
+module Katello
+  module Pulp3
+    class ContentGuard < Katello::Model
+      def self.import(smart_proxy = SmartProxy.pulp_master!, force = false)
+        return unless (count == 0 || force)
+        content_guard_api = Katello::Pulp3::Api::ContentGuard.new(smart_proxy)
+        content_guard = content_guard_api.list&.results&.first
+        return unless content_guard
+        katello_content_guard = self.new(name: content_guard.name, pulp_href: content_guard.pulp_href)
+        katello_content_guard.save!
+      end
+    end
+  end
+end