katello 3.12.3 → 3.13.0.rc1

data/app/models/katello/host_installed_deb.rb

@@ -0,0 +1,7 @@
+module Katello
+  class HostInstalledDeb < Katello::Model
+    # Do not use active record callbacks in this join model.  Direct INSERTs and DELETEs are done
+    belongs_to :host, :inverse_of => :host_installed_debs, :class_name => '::Host::Managed'
+    belongs_to :installed_deb, :inverse_of => :host_installed_debs, :class_name => 'Katello::InstalledDeb'
+  end
+end

data/app/models/katello/installed_deb.rb

@@ -0,0 +1,10 @@
+module Katello
+  class InstalledDeb < Katello::Model
+    has_many :hosts, :through => :host_installed_debs, :class_name => "::Host"
+    has_many :host_installed_debs, :class_name => "Katello::HostInstalledDeb", :dependent => :destroy, :inverse_of => :installed_deb
+
+    scoped_search :on => :name, :complete_value => true
+    scoped_search :on => :version
+    scoped_search :on => :architecture
+  end
+end

data/app/models/katello/ping.rb

@@ -3,17 +3,22 @@ module Katello
     OK_RETURN_CODE = 'ok'.freeze
     FAIL_RETURN_CODE = 'FAIL'.freeze
     PACKAGES = %w(katello candlepin pulp qpid foreman tfm hammer).freeze
-
-    SERVICES = [:pulp, :pulp_auth, :candlepin, :candlepin_auth, :foreman_tasks].freeze
-
     class << self
+      def services(capsule_id = nil)
+        services = [:pulp, :pulp_auth, :candlepin, :candlepin_auth, :foreman_tasks]
+        services += [:pulp3] if fetch_proxy(capsule_id)&.pulp3_enabled?
+        services
+      end
+
       #
       # Calls "status" services in all backend engines.
       #
-      def ping(services: SERVICES, capsule_id: nil)
+      def ping(services: nil, capsule_id: nil)
+        services ||= self.services(capsule_id)
         result = {}
         services.each { |service| result[service] = {} }
 
+        ping_pulp3_without_auth(result[:pulp3], capsule_id) if result.include?(:pulp3)
         ping_pulp_without_auth(result[:pulp], capsule_id) if result.include?(:pulp)
         ping_candlepin_without_auth(result[:candlepin]) if result.include?(:candlepin)
 
@@ -29,6 +34,12 @@ module Katello
         result
       end
 
+      def ping_pulp3_without_auth(service_result, capsule_id)
+        exception_watch(service_result) do
+          Ping.pulp3_without_auth(fetch_proxy(capsule_id).pulp3_url("api/v3"))
+        end
+      end
+
       def ping_pulp_without_auth(service_result, capsule_id)
         exception_watch(service_result) do
           Ping.pulp_without_auth(pulp_url(capsule_id))
@@ -105,12 +116,9 @@ module Katello
       end
 
       def pulp_url(capsule_id)
-        if capsule_id
-          uri = URI.parse(SmartProxy.find(capsule_id).pulp_url)
-          "#{uri.scheme}://#{uri.host.downcase}/pulp/api/v2"
-        else
-          SETTINGS[:katello][:pulp][:url]
-        end
+        proxy = fetch_proxy(capsule_id)
+        uri = URI.parse(proxy.pulp_url)
+        "#{uri.scheme}://#{uri.host.downcase}/pulp/api/v2"
       end
 
       # this checks Pulp is running and responding without need
@@ -137,6 +145,29 @@ module Katello
         json
       end
 
+      def pulp3_without_auth(url)
+        json = backend_status(url, :pulp)
+        fail _("Pulp does not appear to be running at %s.") % url if json.empty?
+
+        if json['database_connection'] && json['database_connection']['connected'] != true
+          fail _("Pulp database connection issue at %s.") % url
+        end
+
+        if json['redis_connection'] && json['redis_connection']['connected'] != true
+          fail _("Pulp redis connection issue at %s.") % url
+        end
+
+        workers = json["online_workers"] || []
+        resource_manager_exists = workers.any? { |worker| worker["name"].include?("resource-manager@") }
+        reservered_resource_worker_exists = workers.any? { |worker| worker["name"] =~ /reserved-resource-worker-./ }
+
+        unless resource_manager_exists && reservered_resource_worker_exists
+          fail _("Not all necessary pulp workers running at %s.") % url
+        end
+
+        json
+      end
+
       def all_pulp_workers_present?(json)
         worker_ids = json["known_workers"].collect { |worker| worker["_id"] }
         return false unless worker_ids.any?
@@ -148,6 +179,10 @@ module Katello
 
       private
 
+      def fetch_proxy(capsule_id)
+        capsule_id ? SmartProxy.find(capsule_id) : SmartProxy.pulp_master
+      end
+
       def backend_status(url, backend)
         ca_file = SETTINGS[:katello][backend][:ca_cert_file]
         options = {}

data/app/models/katello/product_content.rb

@@ -36,7 +36,7 @@ module Katello
 
     # used by Katello::Api::V2::RepositorySetsController#index
     def repositories
-      Katello::Repository.where(:root_id => product.root_repositories.has_url.where(:content_id => content.cp_content_id))
+      Katello::Repository.in_default_view.where(:root_id => product.root_repositories.has_url.where(:content_id => content.cp_content_id))
     end
   end
 end

data/app/models/katello/purpose_addon.rb

@@ -0,0 +1,11 @@
+module Katello
+  class PurposeAddon < Katello::Model
+    self.table_name = 'katello_purpose_addons'
+
+    has_many :subscription_facet_purpose_addons, :class_name => "Katello::SubscriptionFacetPurposeAddon", :dependent => :destroy, :inverse_of => :purpose_addon
+    has_many :subscription_facets, :through => :subscription_facet_purpose_addons, :class_name => "Katello::Host::SubscriptionFacet"
+
+    has_many :activation_key_purpose_addons, :class_name => "Katello::ActivationKeyPurposeAddon", :dependent => :destroy, :inverse_of => :purpose_addon
+    has_many :activation_keys, :through => :activation_key_purpose_addons, :class_name => "Katello::ActivationKey"
+  end
+end

data/app/models/katello/repository.rb

@@ -25,6 +25,7 @@ module Katello
     PUPPET_TYPE = 'puppet'.freeze
     DOCKER_TYPE = 'docker'.freeze
     OSTREE_TYPE = 'ostree'.freeze
+    ANSIBLE_COLLECTION_TYPE = 'ansible_collection'.freeze
 
     define_model_callbacks :sync, :only => :after
 
@@ -90,11 +91,11 @@ module Katello
     has_many :repository_module_streams, class_name: "Katello::RepositoryModuleStream", dependent: :delete_all
     has_many :module_streams, through: :repository_module_streams
 
-    # rubocop:disable HasAndBelongsToMany
-    # TODO: change this into has_many :through association
-    has_and_belongs_to_many :filters, :class_name => "Katello::ContentViewFilter",
-                                      :join_table => :katello_content_view_filters_repositories,
-                                      :foreign_key => :content_view_filter_id
+    has_many :repository_ansible_collections, :class_name => "Katello::RepositoryAnsibleCollection", :dependent => :delete_all
+    has_many :ansible_collections, :through => :repository_ansible_collections
+    has_many :repository_content_view_filters, :class_name => "Katello::RepositoryContentViewFilter", :dependent => :delete_all
+    has_many :filters, :through => :repository_content_view_filters
+
     belongs_to :content_view_version, :inverse_of => :repositories, :class_name => "Katello::ContentViewVersion"
 
     validates_with Validators::ContainerImageNameValidator, :attributes => :container_repository_name, :allow_blank => false, :if => :docker?
@@ -116,6 +117,7 @@ module Katello
     scope :puppet_type, -> { with_type(PUPPET_TYPE) }
     scope :docker_type, -> { with_type(DOCKER_TYPE) }
     scope :ostree_type, -> { with_type(OSTREE_TYPE) }
+    scope :ansible_collection_type, -> { with_type(ANSIBLE_COLLECTION_TYPE) }
     scope :non_puppet, -> { with_type(RepositoryTypeManager.repository_types.keys - [PUPPET_TYPE]) }
     scope :non_archived, -> { where('environment_id is not NULL') }
     scope :archived, -> { where('environment_id is NULL') }
@@ -145,7 +147,7 @@ module Katello
     scoped_search :on => :content_label, :ext_method => :search_by_content_label
 
     delegate :product, :redhat?, :custom?, :to => :root
-    delegate :yum?, :docker?, :puppet?, :deb?, :file?, :ostree?, :to => :root
+    delegate :yum?, :docker?, :puppet?, :deb?, :file?, :ostree?, :ansible_collection?, :to => :root
     delegate :name, :label, :docker_upstream_name, :url, :to => :root
 
     delegate :name, :created_at, :updated_at, :major, :minor, :gpg_key_id, :gpg_key, :content_id, :arch, :label, :url, :unprotected,
@@ -153,7 +155,7 @@ module Katello
              :download_policy, :verify_ssl_on_sync, :"verify_ssl_on_sync?", :upstream_username, :upstream_password,
              :ostree_upstream_sync_policy, :ostree_upstream_sync_depth, :deb_releases, :deb_components, :deb_architectures,
              :ignore_global_proxy, :ssl_ca_cert_id, :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_id, :ssl_client_key_id,
-             :ssl_client_key, :ignorable_content, :description, :docker_tags_whitelist, :to => :root
+             :ssl_client_key, :ignorable_content, :description, :docker_tags_whitelist, :ansible_collection_whitelist, :http_proxy_policy, :http_proxy_id, :to => :root
 
     def self.with_type(content_type)
       joins(:root).where("#{RootRepository.table_name}.content_type" => content_type)
@@ -265,30 +267,40 @@ module Katello
       redhat? ? "redhat" : "custom"
     end
 
-    def empty_errata
+    def self.errata_with_package_counts(repo)
       repository_rpm = Katello::RepositoryRpm.table_name
       repository_errata = Katello::RepositoryErratum.table_name
       rpm = Katello::Rpm.table_name
       errata = Katello::Erratum.table_name
       erratum_package = Katello::ErratumPackage.table_name
-
-      errata_with_packages = Erratum.joins(
+      ::Katello::Erratum.joins(
         "INNER JOIN #{erratum_package} on #{erratum_package}.erratum_id = #{errata}.id",
         "INNER JOIN #{repository_errata} on #{repository_errata}.erratum_id = #{errata}.id",
         "INNER JOIN #{rpm} on #{rpm}.filename = #{erratum_package}.filename",
         "INNER JOIN #{repository_rpm} on #{repository_rpm}.rpm_id = #{rpm}.id").
-        where("#{repository_rpm}.repository_id" => self.id).
-        where("#{repository_errata}.repository_id" => self.id)
+        where("#{repository_rpm}.repository_id" => repo.id).
+        where("#{repository_errata}.repository_id" => repo.id).
+        group("#{errata}.id").count
+    end
 
-      if errata_with_packages.any?
-        self.errata.where("#{Katello::Erratum.table_name}.id NOT IN (?)", errata_with_packages.pluck("#{errata}.id"))
-      else
-        self.errata
+    def partial_errata
+      return [] if library_instance?
+
+      errata_with_package_counts = ::Katello::Repository.errata_with_package_counts(self)
+      partial_errata = self.errata
+      if errata_with_package_counts.any?
+        errata_with_packages_in_library = ::Katello::Repository.errata_with_package_counts(library_instance)
+        errata_with_package_counts.keep_if { |id| errata_with_package_counts[id] == errata_with_packages_in_library[id] }
+        unless errata_with_package_counts.empty?
+          partial_errata = self.errata.where("#{Katello::Erratum.table_name}.id NOT IN (?)", errata_with_package_counts.keys)
+        end
       end
+
+      partial_errata
     end
 
-    def empty_errata!
-      found = empty_errata.to_a
+    def remove_partial_errata!
+      found = partial_errata.to_a
       yield(found) if block_given?
       self.repository_errata.where(:erratum_id => found.map(&:id)).delete_all
       found
@@ -521,14 +533,15 @@ module Katello
       self.ostree_branches.map(&:name)
     end
 
-    def units_for_removal(ids)
+    def units_for_removal(ids, type_class = nil)
+      removable_unit_association = unit_type_for_removal(type_class)
       table_name = removable_unit_association.table_name
       is_integer = Integer(ids.first) rescue false #assume all ids are either integers or not
 
       if is_integer
-        self.removable_unit_association.where("#{table_name}.id in (?)", ids)
+        removable_unit_association.where("#{table_name}.id in (?)", ids)
       else
-        self.removable_unit_association.where("#{table_name}.pulp_id in (?)", ids)
+        removable_unit_association.where("#{table_name}.pulp_id in (?)", ids)
       end
     end
 
@@ -739,23 +752,31 @@ module Katello
       RepositoryTypeManager.find(self.content_type)
     end
 
+    def copy_indexed_data(source_repository)
+      repository_type.content_types_to_index.each do |type|
+        type.model_class.copy_repository_associations(source_repository, self)
+        repository_type.index_additional_data_proc&.call(self, source_repository)
+      end
+    end
+
     def index_linked_repo
       if (base_repo = self.target_repository)
-        repository_type.content_types_to_index.each do |type|
-          type.model_class.copy_repository_associations(base_repo, self)
-          repository_type.index_additional_data_proc&.call(self, target_repository)
-        end
+        copy_indexed_data(base_repo)
       else
         Rails.logger.error("Cannot index #{self.id}, no target repository found.")
       end
     end
 
-    def index_content(full_index = false)
+    def index_content(options = {})
+      source_repository = options.fetch(:source_repository, nil)
+
       if self.yum? && !self.master?
         index_linked_repo
+      elsif source_repository
+        copy_indexed_data(source_repository)
       else
         repository_type.content_types_to_index.each do |type|
-          type.model_class.import_for_repository(self, full_index)
+          type.model_class.import_for_repository(self)
         end
         repository_type.index_additional_data_proc&.call(self)
       end
@@ -764,21 +785,11 @@ module Katello
 
     protected
 
-    def removable_unit_association
-      if yum?
-        self.rpms
-      elsif docker?
-        self.docker_manifests
-      elsif puppet?
-        self.puppet_modules
-      elsif ostree?
-        self.ostree_branches
-      elsif file?
-        self.files
-      elsif deb?
-        self.debs
+    def unit_type_for_removal(type_class = nil)
+      if type_class
+        Katello::RepositoryTypeManager.find_content_type(type_class).model_class
       else
-        fail "Content type not supported for removal"
+        Katello::RepositoryTypeManager.find(self.content_type).default_managed_content_type.model_class
       end
     end
 

data/app/models/katello/repository_ansible_collection.rb

@@ -0,0 +1,6 @@
+module Katello
+  class RepositoryAnsibleCollection < Katello::Model
+    belongs_to :repository, inverse_of: :repository_ansible_collections, class_name: 'Katello::Repository'
+    belongs_to :ansible_collection, inverse_of: :repository_ansible_collections, class_name: 'Katello::AnsibleCollection'
+  end
+end

data/app/models/katello/repository_content_view_filter.rb

@@ -0,0 +1,7 @@
+module Katello
+  class RepositoryContentViewFilter < Katello::Model
+    # Do not use active record callbacks in this join model.  Direct INSERTs and DELETEs are done
+    belongs_to :repository, :inverse_of => :repository_content_view_filters, :class_name => 'Katello::Repository'
+    belongs_to :filter, :inverse_of => :repository_content_view_filters, :class_name => 'Katello::ContentViewFilter', :foreign_key => :content_view_filter_id
+  end
+end

data/app/models/katello/rhsm_fact_parser.rb

@@ -54,11 +54,17 @@ module Katello
       os_name = ::Katello::Candlepin::Consumer.distribution_to_puppet_os(name)
       major, minor = version.split('.')
       if os_name && !invalid_centos_os?(os_name, minor)
-        os_attributes = {:major => major, :minor => minor || '', :name => os_name}
+        os_attributes = {:major => major, :minor => minor || '', :name => os_name, :release_name => os_release_name(os_name)}
         ::Operatingsystem.find_by(os_attributes) || ::Operatingsystem.create!(os_attributes)
       end
     end
 
+    def os_release_name(os_name)
+      if os_name.match(::Operatingsystem::FAMILIES['Debian'])
+        facts['distribution.id']
+      end
+    end
+
     def invalid_centos_os?(name, minor_version)
       name == 'CentOS' && minor_version.blank?
     end
@@ -73,6 +79,10 @@ module Katello
     def ipmi_interface
     end
 
+    def boot_timestamp
+      facts['proc_stat.btime']&.to_i
+    end
+
     private
 
     def get_rhsm_ip(interface)

data/app/models/katello/root_repository.rb

@@ -23,11 +23,20 @@ module Katello
 
     }.freeze
 
+    NO_DEFAULT_HTTP_PROXY = 'none'.freeze
+    GLOBAL_DEFAULT_HTTP_PROXY = 'global_default_http_proxy'.freeze
+    USE_SELECTED_HTTP_PROXY = 'use_selected_http_proxy'.freeze
+    HTTP_PROXY_POLICIES = [
+      GLOBAL_DEFAULT_HTTP_PROXY,
+      NO_DEFAULT_HTTP_PROXY,
+      USE_SELECTED_HTTP_PROXY].freeze
+
     belongs_to :product, :inverse_of => :root_repositories, :class_name => "Katello::Product"
     belongs_to :gpg_key, :inverse_of => :root_repositories, :class_name => "Katello::GpgKey"
     belongs_to :ssl_ca_cert, :class_name => "Katello::GpgKey", :inverse_of => :ssl_ca_root_repos
     belongs_to :ssl_client_cert, :class_name => "Katello::GpgKey", :inverse_of => :ssl_client_root_repos
     belongs_to :ssl_client_key, :class_name => "Katello::GpgKey", :inverse_of => :ssl_key_root_repos
+    belongs_to :http_proxy, :inverse_of => :root_repositories
     has_many :repositories, :class_name => "Katello::Repository", :foreign_key => :root_id,
                           :inverse_of => :root, :dependent => :destroy
 
@@ -48,6 +57,7 @@ module Katello
     validates_with Validators::ContainerImageNameValidator, :attributes => :docker_upstream_name, :allow_blank => true, :if => :docker?
 
     validate :ensure_valid_docker_attributes, :if => :docker?
+    validate :ensure_valid_ansible_collection_attributes, :if => :ansible_collection?
     validate :ensure_docker_repo_unprotected, :if => :docker?
     validate :ensure_ostree_repo_protected, :if => :ostree?
     validate :ensure_compatible_download_policy, :if => :yum?
@@ -71,7 +81,10 @@ module Katello
       :in => ::Runcible::Models::YumImporter::DOWNLOAD_POLICIES,
       :message => _("must be one of the following: %s") % ::Runcible::Models::YumImporter::DOWNLOAD_POLICIES.join(', ')
     }, if: :yum?
-
+    validates :http_proxy_policy, inclusion: {
+      :in => HTTP_PROXY_POLICIES,
+      :message => _("must be one of the following: %s") % HTTP_PROXY_POLICIES.join(', ')
+    }
     scope :subscribable, -> { where(content_type: RootRepository::SUBSCRIBABLE_TYPES) }
     scope :has_url, -> { where.not(:url => nil) }
     scope :with_repository_attribute, ->(attr, value) { joins(:repositories).where("#{Katello::Repository.table_name}.#{attr}" => value) }
@@ -82,6 +95,13 @@ module Katello
     scope :puppet_type, -> { where(:content_type => Repository::PUPPET_TYPE) }
     scope :docker_type, -> { where(:content_type => Repository::DOCKER_TYPE) }
     scope :ostree_type, -> { where(:content_type => Repository::OSTREE_TYPE) }
+    scope :ansible_collection_type, -> { where(:content_type => Repository::ANSIBLE_COLLECTION_TYPE) }
+    scope :with_global_proxy, -> { where(:http_proxy_policy => RootRepository::GLOBAL_DEFAULT_HTTP_PROXY) }
+    scope :with_no_proxy, -> { where(:http_proxy_policy => RootRepository::NO_DEFAULT_HTTP_PROXY) }
+    scope :with_selected_proxy, ->(http_proxy_id) {
+      where(:http_proxy_policy => RootRepository::USE_SELECTED_HTTP_PROXY).
+      where("http_proxy_id = ?", http_proxy_id)
+    }
     delegate :redhat?, :provider, :organization, to: :product
 
     def library_instance
@@ -193,6 +213,10 @@ module Katello
       end
     end
 
+    def ensure_valid_ansible_collection_attributes
+      errors.add(:base, N_("Whitelist cannot be blank.")) if ansible_collection_whitelist.blank?
+    end
+
     def ensure_valid_upstream_authorization
       return if (self.upstream_username.blank? && self.upstream_password.blank?)
       if redhat?
@@ -247,6 +271,10 @@ module Katello
       self.content_type == Repository::DEB_TYPE
     end
 
+    def ansible_collection?
+      self.content_type == Repository::ANSIBLE_COLLECTION_TYPE
+    end
+
     def metadata_generate_needed?
       (%w(unprotected checksum_type container_repsoitory_name) & previous_changes.keys).any?
     end
@@ -258,9 +286,10 @@ module Katello
     def pulp_update_needed?
       changeable_attributes = %w(url unprotected checksum_type docker_upstream_name download_policy mirror_on_sync verify_ssl_on_sync
                                  upstream_username upstream_password ostree_upstream_sync_policy ostree_upstream_sync_depth ignore_global_proxy ignorable_content
-                                 ssl_ca_cert_id ssl_client_cert_id ssl_client_key_id)
+                                 ssl_ca_cert_id ssl_client_cert_id ssl_client_key_id http_proxy_policy http_proxy_id)
       changeable_attributes += %w(name container_repository_name docker_tags_whitelist) if docker?
       changeable_attributes += %w(deb_releases deb_components deb_architectures gpg_key_id) if deb?
+      changeable_attributes += %w(ansible_collection_whitelist) if ansible_collection?
       changeable_attributes.any? { |key| previous_changes.key?(key) }
     end
 
@@ -268,16 +297,29 @@ module Katello
       self.content.content_url
     end
 
+    def repo_mapper
+      Katello::Candlepin::RepositoryMapper.new(self.product, self.content, self.substitutions)
+    end
+
     def calculate_updated_name
       fail _("Cannot calculate name for custom repos") if custom?
-      Katello::Candlepin::RepositoryMapper.new(self.product, self.content, self.substitutions).name
+      repo_mapper.name
     end
 
     def substitutions
       {
         :releasever => self.minor,
         :basearch => self.arch
-      }
+      }.compact
+    end
+
+    def http_proxy
+      if http_proxy_policy == NO_DEFAULT_HTTP_PROXY
+        return nil
+      elsif http_proxy_policy == GLOBAL_DEFAULT_HTTP_PROXY
+        return HttpProxy.default_global_content_proxy
+      end
+      super
     end
 
     class Jail < ::Safemode::Jail