katello 3.12.3 → 3.13.0.rc1

data/app/lib/katello/resources/cdn.rb

@@ -18,7 +18,6 @@ module Katello
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
         attr_reader :url, :product, :options
-        attr_accessor :proxy_host, :proxy_port, :proxy_user, :proxy_password
 
         def substitutor(logger = nil)
           @logger = logger
@@ -39,7 +38,6 @@ module Katello
             options.reverse_merge!(:cert_store => store)
           end
 
-          load_proxy_settings
           @product = options[:product]
 
           @url = url
@@ -137,25 +135,14 @@ module Katello
         end
 
         def net_http_class
-          if proxy_host
-            Net::HTTP::Proxy(proxy_host, proxy_port, proxy_user, proxy_password)
+          if (proxy = ::HttpProxy.default_global_content_proxy)
+            uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
+            Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else
             Net::HTTP
           end
         end
 
-        def load_proxy_settings
-          if SETTINGS[:katello][:cdn_proxy] && SETTINGS[:katello][:cdn_proxy][:host]
-            self.proxy_host = parse_host(SETTINGS[:katello][:cdn_proxy][:host])
-            self.proxy_port = SETTINGS[:katello][:cdn_proxy][:port]
-            self.proxy_user = SETTINGS[:katello][:cdn_proxy][:user]
-            self.proxy_password = SETTINGS[:katello][:cdn_proxy][:password]
-          end
-        rescue URI::Error => e
-          Rails.logger.error "Could not parse cdn_proxy:"
-          Rails.logger.error e.to_s
-        end
-
         def parse_host(host_or_url)
           uri = URI.parse(host_or_url)
           return uri.host || uri.path

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -68,9 +68,6 @@ module Katello
         futures.each do |future|
           begin
             resolved << future.value
-          rescue StandardError => e
-            Rails.logger.error("Error Recieved: #{e.to_s}")
-            Rails.logger.error("Error Recieved: #{e.backtrace.join("\n")}")
           end
         end
 

data/app/lib/katello/util/http_proxy.rb

@@ -6,34 +6,34 @@ module Katello
       def proxy_uri
         #Reset the scheme to proxy(s) based on http or https to handle cgi unescaping in rest-client
         # this relies on katello/util/proxy_uri
-        if proxy_host
+        if proxy
           scheme = 'proxy' if proxy_scheme == 'http'
           scheme = 'proxys' if proxy_scheme == 'https'
 
           uri = URI("#{scheme}://#{proxy_host}:#{proxy_port}")
-          if proxy_config && proxy_config[:user]
-            uri.user = CGI.escape(proxy_config[:user])
-            uri.password = CGI.escape(proxy_config[:password])
+          if proxy && proxy.username.present?
+            uri.user = CGI.escape(proxy.username)
+            uri.password = CGI.escape(proxy.password)
           end
 
           uri.to_s
         end
       end
 
-      def proxy_config
-        SETTINGS[:katello][:cdn_proxy]
+      def proxy
+        ::HttpProxy.default_global_content_proxy
       end
 
       def proxy_host
-        proxy_config && URI.parse(proxy_config[:host]).host
+        URI(proxy.url).host
       end
 
       def proxy_scheme
-        proxy_config && URI.parse(proxy_config[:host]).scheme
+        URI(proxy.url).scheme
       end
 
       def proxy_port
-        proxy_config && proxy_config[:port]
+        URI(proxy.url).port
       end
     end # HttpProxy
   end # Util

data/app/lib/katello/validators/hostgroup_kickstart_repository_validator.rb

@@ -11,7 +11,7 @@ module Katello
         return unless hostgroup.kickstart_repository_id
 
         msg = if hostgroup.content_source.blank?
-                hostgroup.errors.add(:base, _("Please select a content source before assigning a kickstart repository"))
+                _("Please select a content source before assigning a kickstart repository")
               elsif hostgroup.operatingsystem.blank?
                 _("Please select an operating system before assigning a kickstart repository")
               elsif !hostgroup.operatingsystem.is_a?(Redhat)

data/app/models/katello/activation_key.rb

@@ -22,10 +22,15 @@ module Katello
     has_many :subscription_facet_activation_keys, :class_name => "Katello::SubscriptionFacetActivationKey", :dependent => :destroy
     has_many :subscription_facets, :through => :subscription_facet_activation_keys
 
+    has_many :activation_key_purpose_addons, :class_name => "Katello::ActivationKeyPurposeAddon", :dependent => :destroy, :inverse_of => :activation_key
+    has_many :purpose_addons, :class_name => "Katello::PurposeAddon", :through => :activation_key_purpose_addons
+
     alias_method :lifecycle_environment, :environment
 
     before_validation :set_default_content_view, :unless => :persisted?
 
+    accepts_nested_attributes_for :purpose_addons
+
     validates_lengths_from_database
     validates_with Validators::KatelloNameFormatValidator, :attributes => :name
     validates :name, :presence => true
@@ -61,6 +66,9 @@ module Katello
     scoped_search :on => :name, :relation => :subscriptions, :rename => :subscription_name, :complete_value => true, :ext_method => :find_by_subscription_name
     scoped_search :on => :id, :relation => :subscriptions, :rename => :subscription_id, :complete_value => true,
                   :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER, :ext_method => :find_by_subscription_id
+    scoped_search :on => :purpose_usage, :rename => :usage, :complete_value => true
+    scoped_search :on => :purpose_role, :rename => :role, :complete_value => true
+    scoped_search :on => :name, :rename => :addon, :relation => :purpose_addon, :complete_value => true, :ext_method => :find_by_purpose_addons
 
     def environment_exists
       if environment_id && environment.nil?
@@ -162,6 +170,12 @@ module Katello
       end
     end
 
+    def self.find_by_purpose_addons(_key, operator, value)
+      conditions = sanitize_sql_for_conditions(["#{Katello::PurposeAddon.table_name}.name #{operator} ?", value_to_sql(operator, value)])
+      activation_keys = ::Katello::ActivationKey.joins(:purpose_addons).where(conditions)
+      return_activation_keys_by_id(activation_keys.pluck(:id))
+    end
+
     private
 
     def set_default_content_view

data/app/models/katello/activation_key_purpose_addon.rb

@@ -0,0 +1,6 @@
+module Katello
+  class ActivationKeyPurposeAddon < Katello::Model
+    belongs_to :activation_key, inverse_of: :activation_key_purpose_addons, class_name: 'Katello::ActivationKey'
+    belongs_to :purpose_addon, inverse_of: :activation_key_purpose_addons, class_name: 'Katello::PurposeAddon'
+  end
+end

data/app/models/katello/ansible_collection.rb

@@ -0,0 +1,32 @@
+module Katello
+  class AnsibleCollection < ApplicationRecord
+    include Concerns::PulpDatabaseUnit
+
+    self.table_name = 'katello_ansible_collections'
+
+    CONTENT_TYPE = 'ansible collection'.freeze
+
+    has_many :repository_ansible_collections, :class_name => "Katello::RepositoryAnsibleCollection", :dependent => :destroy, :inverse_of => :ansible_collection, :foreign_key => :ansible_collection_id
+    has_many :repositories, :through => :repository_ansible_collections, :class_name => "Katello::Repository"
+
+    scoped_search :on => :name, :complete_value => true
+    scoped_search :on => :namespace, :complete_value => true
+    scoped_search :on => :version, :complete_value => true
+
+    def self.default_sort
+      order(:name)
+    end
+
+    def self.repository_association_class
+      RepositoryAnsibleCollection
+    end
+
+    def self.unit_id_field
+      'ansible_collection_id'
+    end
+
+    def self.total_for_repositories(repos)
+      self.in_repositories(repos).count
+    end
+  end
+end

data/app/models/katello/candlepin/repository_mapper.rb

@@ -123,7 +123,7 @@ module Katello
 
       def download_policy
         if katello_content_type == Repository::YUM_TYPE
-          Setting[:default_download_policy]
+          Setting[:default_redhat_download_policy]
         else
           ""
         end

data/app/models/katello/concerns/host_managed_extensions.rb

@@ -39,6 +39,8 @@ module Katello
         has_many :host_available_module_streams, :class_name => "::Katello::HostAvailableModuleStream", :foreign_key => :host_id, :dependent => :delete_all
         has_many :available_module_streams, :class_name => "::Katello::AvailableModuleStream", :through => :host_available_module_streams
 
+        has_many :host_installed_debs, :class_name => "::Katello::HostInstalledDeb", :foreign_key => :host_id, :dependent => :delete_all
+        has_many :installed_debs, :class_name => "::Katello::InstalledDeb", :through => :host_installed_debs
         has_many :host_traces, :class_name => "::Katello::HostTracer", :foreign_key => :host_id, :dependent => :destroy
 
         has_many :host_collection_hosts, :class_name => "::Katello::HostCollectionHosts", :foreign_key => :host_id, :dependent => :destroy
@@ -55,6 +57,7 @@ module Katello
         scoped_search :relation => :installed_packages, :on => :name, :complete_value => true, :rename => :installed_package_name, :only_explicit => true
         scoped_search :relation => :available_module_streams, :on => :name, :complete_value => true, :rename => :available_module_stream_name, :only_explicit => true
         scoped_search :relation => :available_module_streams, :on => :stream, :complete_value => true, :rename => :available_module_stream_stream, :only_explicit => true
+        scoped_search :relation => :installed_debs, :on => :name, :complete_value => true, :rename => :installed_package_name, :only_explicit => true
         scoped_search :relation => :host_traces, :on => :application, :complete_value => true, :rename => :trace_app, :only_explicit => true
         scoped_search :relation => :host_traces, :on => :app_type, :complete_value => true, :rename => :trace_app_type, :only_explicit => true
         scoped_search :relation => :host_traces, :on => :helper, :complete_value => true, :rename => :trace_helper, :only_explicit => true

data/app/models/katello/concerns/http_proxy_extensions.rb

@@ -0,0 +1,60 @@
+module Katello
+  module Concerns
+    module HttpProxyExtensions
+      extend ActiveSupport::Concern
+
+      included do
+        has_many :root_repositories, :class_name => "::Katello::RootRepository", :foreign_key => :http_proxy_id,
+          :inverse_of => :http_proxy, :dependent => :nullify
+        after_update :update_default_proxy_setting
+        after_commit :update_repository_proxy_details
+
+        def self.default_global_content_proxy
+          if Setting[:content_default_http_proxy]
+            HttpProxy.unscoped.find_by(name: Setting[:content_default_http_proxy])
+          end
+        end
+      end
+
+      def update_default_proxy_setting
+        changes = self.previous_changes
+        if changes.key?(:name)
+          previous_name = changes[:name].first
+          setting = Setting.find_by(name: 'content_default_http_proxy')
+
+          if setting && setting.value == previous_name && !previous_name.blank?
+            setting.update_attribute(:value, self.name)
+          end
+        end
+      end
+
+      def update_repository_proxy_details
+        changes = self.previous_changes
+        if changes.key?(:url) || changes.key?(:username) || changes.key?(:password)
+
+          root_repos = RootRepository.with_selected_proxy(id)
+
+          if self == HttpProxy.default_global_content_proxy
+            root_repos += RootRepository.with_global_proxy
+          end
+
+          repos = root_repos.uniq.collect(&:library_instance)
+
+          unless repos.empty?
+            ForemanTasks.async_task(
+              ::Actions::BulkAction,
+              ::Actions::Katello::Repository::UpdateHttpProxyDetails,
+              repos.sort_by(&:pulp_id))
+          end
+        end
+      end
+
+      def name_and_url
+        uri = URI(url)
+        uri.password = nil
+        uri.user = nil
+        "#{name} (#{uri})"
+      end
+    end
+  end
+end

data/app/models/katello/concerns/location_extensions.rb

@@ -7,6 +7,7 @@ module Katello
         after_initialize :set_default_overrides, :if => :new_record?
         after_save :reset_settings
         before_destroy :assert_deletable
+        after_create :associate_default_http_proxy
       end
 
       def set_default_overrides
@@ -58,6 +59,13 @@ module Katello
         end
       end
 
+      def associate_default_http_proxy
+        if (default_proxy = ::HttpProxy.default_global_content_proxy)
+          default_proxy.locations << self
+          default_proxy.save
+        end
+      end
+
       module ClassMethods
         def default_puppet_content_location
           ::Location.unscoped.find_by_title(::Setting[:default_location_puppet_content]) if ::Setting[:default_location_puppet_content].present?

data/app/models/katello/concerns/organization_extensions.rb

@@ -41,6 +41,8 @@ module Katello
         scoped_search :on => :label, :complete_value => :true
 
         after_create :associate_default_capsule
+        after_create :associate_default_http_proxy
+
         validates_with Validators::KatelloLabelFormatValidator, :attributes => :label
         validates :label, :uniqueness => true
 
@@ -96,6 +98,13 @@ module Katello
           smart_proxy.organizations << self if smart_proxy
         end
 
+        def associate_default_http_proxy
+          if (default_proxy = ::HttpProxy.default_global_content_proxy)
+            default_proxy.organizations << self
+            default_proxy.save
+          end
+        end
+
         def create_anonymous_provider
           self.providers << Katello::Provider.new(:name => Katello::Provider::ANONYMOUS, :provider_type => Katello::Provider::ANONYMOUS)
         end

data/app/models/katello/concerns/pulp_database_unit.rb

@@ -24,7 +24,7 @@ module Katello
         "::Katello::Pulp::#{self.name.demodulize}".constantize
       end
 
-      def manage_repository_association
+      def many_repository_associations
         true
       end
 
@@ -44,7 +44,7 @@ module Katello
       end
 
       def in_repositories(repos)
-        if manage_repository_association
+        if many_repository_associations
           where(:id => repository_association_class.where(:repository_id => repos).select(unit_id_field))
         else
           where(:repository_id => repos)
@@ -55,85 +55,55 @@ module Katello
         content_unit_class.new(pulp_id)
       end
 
-      # Import all units of a single type and refresh their repository associations
-      def import_all(pulp_ids = nil, options = {})
-        index_repository_association = options.fetch(:index_repository_association, true) && self.manage_repository_association
-        service_class = SmartProxy.pulp_master!.content_service(content_type)
+      def unit_id_field
+        "#{self.name.demodulize.underscore}_id"
+      end
 
-        process_block = lambda do |units|
+      def import_all(pulp_ids = nil, repository = nil)
+        ids_to_associate = []
+        service_class = SmartProxy.pulp_master!.content_service(content_type)
+        service_class.pulp_units_batch_all(pulp_ids).each do |units|
           units.each do |unit|
             unit = unit.with_indifferent_access
             model = Katello::Util::Support.active_record_retry do
-              self.where(:pulp_id => unit['_id']).first_or_create
+              self.where(:pulp_id => unit[service_class.unit_identifier]).first_or_create
             end
-
             service = service_class.new(model.pulp_id)
             service.backend_data = unit
             service.update_model(model)
+            ids_to_associate << model.pulp_id
           end
-          if index_repository_association
-            units.map { |unit| unit.slice('_id', 'repository_memberships') }
-          else
-            units.count
-          end
-        end
-
-        if pulp_ids
-          results = content_unit_class.fetch_by_uuids(pulp_ids, &process_block).flatten
-          update_repository_associations(results, true) if index_repository_association
-        else
-          results = content_unit_class.fetch_all(&process_block).flatten
-          update_repository_associations(results) if index_repository_association
-        end
-      end
-
-      def import_for_repository(repository, force = false)
-        ids = content_unit_class.ids_for_repository(repository.pulp_id)
-        # Rpms cannot change in Pulp so we do not index them if they are already present
-        # in our database. Errata and Package Groups can change in Pulp, so we index
-        # all of them in the repository on each sync.
-        if immutable_unit_types.include?(self) && !force
-          ids_to_import = ids - repository.rpms.map(&:pulp_id)
-        else
-          ids_to_import = ids
         end
-        self.import_all(ids_to_import, :index_repository_association => false) if repository.content_view.default? || force
-        self.sync_repository_associations(repository, :pulp_ids => ids) if self.manage_repository_association
-      end
-
-      def unit_id_field
-        "#{self.name.demodulize.underscore}_id"
+        sync_repository_associations(repository, :pulp_ids => ids_to_associate, :additive => true) if self.many_repository_associations && repository && ids_to_associate.present?
       end
 
-      def copy_repository_associations(source_repo, dest_repo)
-        if manage_repository_association
-          delete_query = "delete from #{repository_association_class.table_name} where repository_id = #{dest_repo.id} and
-                         #{unit_id_field} not in (select #{unit_id_field} from #{repository_association_class.table_name} where repository_id = #{source_repo.id})"
-
-          insert_query = "insert into #{repository_association_class.table_name} (repository_id, #{unit_id_field})
-                          select #{dest_repo.id} as repository_id, #{unit_id_field} from #{repository_association_class.table_name}
-                          where repository_id = #{source_repo.id} and #{unit_id_field} not in (select #{unit_id_field}
-                          from #{repository_association_class.table_name} where repository_id = #{dest_repo.id})"
-        else
-          columns = column_names - ["id", "pulp_id", "created_at", "updated_at", "repository_id"]
-
-          delete_query = "delete from #{self.table_name} where repository_id = #{dest_repo.id} and
-                          pulp_id not in (select pulp_id from #{self.table_name} where repository_id = #{source_repo.id})"
-          insert_query = "insert into #{self.table_name} (repository_id, pulp_id, #{columns.join(',')})
-                    select #{dest_repo.id} as repository_id, pulp_id, #{columns.join(',')} from #{self.table_name}
-                    where repository_id = #{source_repo.id} and pulp_id not in (select pulp_id
-                    from #{self.table_name} where repository_id = #{dest_repo.id})"
+      def import_for_repository(repository)
+        pulp_ids = []
+        service_class = SmartProxy.pulp_master!.content_service(content_type)
+        fetch_only_ids = !repository.content_view.default?
 
+        service_class.pulp_units_batch_for_repo(repository, fetch_identifiers: fetch_only_ids).each do |units|
+          units.each do |unit|
+            unit = unit.with_indifferent_access
+            pulp_id = unit[service_class.unit_identifier]
+            unless fetch_only_ids
+              model = Katello::Util::Support.active_record_retry do
+                self.where(:pulp_id => pulp_id).first_or_create
+              end
+              service = service_class.new(model.pulp_id)
+              service.backend_data = unit
+              model.repository_id = repository.id unless many_repository_associations
+              service.update_model(model)
+            end
+            pulp_ids << pulp_id
+          end
         end
-        ActiveRecord::Base.connection.execute(delete_query)
-        ActiveRecord::Base.connection.execute(insert_query)
+        sync_repository_associations(repository, :pulp_ids => pulp_ids) if self.many_repository_associations
       end
 
       def sync_repository_associations(repository, options = {})
         additive = options.fetch(:additive, false)
-        associated_ids = options.fetch(:ids, nil)
-        pulp_ids = options.fetch(:pulp_ids) if associated_ids.nil?
-
+        pulp_ids = options.fetch(:pulp_ids)
         associated_ids = with_pulp_id(pulp_ids).pluck(:id) if pulp_ids
 
         table_name = self.repository_association_class.table_name
@@ -163,27 +133,32 @@ module Katello
         end
       end
 
-      def with_pulp_id(unit_pulp_ids)
-        where(:pulp_id => unit_pulp_ids)
-      end
+      def copy_repository_associations(source_repo, dest_repo)
+        if many_repository_associations
+          delete_query = "delete from #{repository_association_class.table_name} where repository_id = #{dest_repo.id} and
+                         #{unit_id_field} not in (select #{unit_id_field} from #{repository_association_class.table_name} where repository_id = #{source_repo.id})"
 
-      def update_repository_associations(units_json, additive = false)
-        ActiveRecord::Base.transaction do
-          repo_unit_id = {}
-          repo_cache = {}
-          units_json.each do |unit_json|
-            unit_json['repository_memberships'].each do |repo_pulp_id|
-              if (repo_cache[repo_pulp_id] ||= Repository.exists?(:pulp_id => repo_pulp_id))
-                repo_unit_id[repo_pulp_id] ||= []
-                repo_unit_id[repo_pulp_id] << unit_json['_id']
-              end
-            end
-          end
+          insert_query = "insert into #{repository_association_class.table_name} (repository_id, #{unit_id_field})
+                          select #{dest_repo.id} as repository_id, #{unit_id_field} from #{repository_association_class.table_name}
+                          where repository_id = #{source_repo.id} and #{unit_id_field} not in (select #{unit_id_field}
+                          from #{repository_association_class.table_name} where repository_id = #{dest_repo.id})"
+        else
+          columns = column_names - ["id", "pulp_id", "created_at", "updated_at", "repository_id"]
+
+          delete_query = "delete from #{self.table_name} where repository_id = #{dest_repo.id} and
+                          pulp_id not in (select pulp_id from #{self.table_name} where repository_id = #{source_repo.id})"
+          insert_query = "insert into #{self.table_name} (repository_id, pulp_id, #{columns.join(',')})
+                    select #{dest_repo.id} as repository_id, pulp_id, #{columns.join(',')} from #{self.table_name}
+                    where repository_id = #{source_repo.id} and pulp_id not in (select pulp_id
+                    from #{self.table_name} where repository_id = #{dest_repo.id})"
 
-          repo_unit_id.each do |repo_pulp_id, unit_pulp_ids|
-            sync_repository_associations(Repository.find_by(:pulp_id => repo_pulp_id), :pulp_ids => unit_pulp_ids, :additive => additive)
-          end
         end
+        ActiveRecord::Base.connection.execute(delete_query)
+        ActiveRecord::Base.connection.execute(insert_query)
+      end
+
+      def with_pulp_id(unit_pulp_ids)
+        where('pulp_id in (?)', unit_pulp_ids)
       end
     end
   end