katello 4.3.0 → 4.4.0.rc1

data/app/lib/actions/pulp3/orchestration/content_view_version/import.rb

@@ -15,17 +15,19 @@ module Actions
                 metadata: metadata
               ).output
 
-              import_output = plan_action(
+              plan_action(
                 ::Actions::Pulp3::ContentViewVersion::Import,
                 content_view_version_id: content_view_version.id,
                 smart_proxy_id: smart_proxy.id,
                 importer_data: importer_output[:importer_data],
                 path: path,
                 metadata: metadata
-              ).output
-
-              plan_action(Actions::Pulp3::Repository::SaveVersions, content_view_version.importable_repositories.pluck(:id),
-                          tasks: import_output[:pulp_tasks])
+              )
+              concurrence do
+                content_view_version.importable_repositories.each do |repo|
+                  plan_action(Actions::Pulp3::Repository::SaveVersion, repo)
+                end
+              end
               plan_action(
                 ::Actions::Pulp3::ContentViewVersion::CreateImportHistory,
                 content_view_version_id: content_view_version.id,

data/app/lib/actions/pulp3/repository/copy_content.rb

@@ -13,7 +13,7 @@ module Actions
           target = ::Katello::Repository.find(input[:target_repository_id] || input[:target_repository])
           service = target.backend_service(smart_proxy)
           output[:pulp_tasks] = if input[:copy_all]
-                                  service.copy_all(source, mirror: input[:mirror] || false)
+                                  service.copy_all(source, input)
                                 else
                                   service.copy_content_for_source(source, input)
                                 end

data/app/lib/actions/pulp3/repository/save_artifact.rb

@@ -27,6 +27,7 @@ module Actions
               []
             else
               output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
+                                                                                repository_id: repository.id,
                                                                                 artifact: artifact_href,
                                                                                 content_type: content_type)]
             end

data/app/lib/katello/concerns/base_template_scope_extensions.rb

@@ -288,6 +288,17 @@ module Katello
         Katello.with_katello_agent?
       end
 
+      include Katello::ContentSourceHelper
+
+      apipie :method, "Generate script to change a host's content source" do
+        returns String
+      end
+      def change_content_source(host, ca_cert)
+        return missing_content_source(host) unless host.content_source
+
+        prepare_ssl_cert(ca_cert) + configure_subman(host.content_source)
+      end
+
       private
 
       def host_subscription_facet(host)

data/app/lib/katello/errors.rb

@@ -145,10 +145,10 @@ module Katello
       end
     end
 
-    class DisconnectedMode < StandardError
+    class SubscriptionConnectionNotEnabled < StandardError
       def message
-        _("You are currently operating in disconnected mode where access to Red Hat Subcription Management " \
-        "is prohibited. If you would like to change this, please update the content setting 'Disconnected mode'.")
+        _("Access to Red Hat Subscription Management " \
+        "is prohibited. If you would like to change this, please update the content setting 'Subscription connection enabled'.")
       end
     end
 

data/app/lib/katello/logging.rb

@@ -19,9 +19,9 @@ module Katello
     class Timer
       def initialize(key = "default")
         @key = key
-        @start_time = Time.now
         Thread.current[:timers] ||= {}
         Thread.current[:timers][key] = self
+        self.start
       end
 
       def start
@@ -38,6 +38,11 @@ module Katello
         Rails.logger.info "Timer #{@key} stopping at #{Time.now}: #{duration} sec"
       end
 
+      def log(msg = nil)
+        duration = (Time.now - @start_time).truncate(2)
+        Rails.logger.info ["Timer #{@key} running at #{Time.now}", msg, "#{duration} sec"].compact.join(': ')
+      end
+
       def self.find_by_key(key)
         if Thread.current&.[](:timers)&.[](key)
           Thread.current[:timers][key]

data/app/lib/katello/repo_discovery.rb

@@ -1,10 +1,12 @@
 module Katello
   class RepoDiscovery
+    include Katello::Util::HttpProxy
+
     attr_reader :found, :crawled, :to_follow
 
     # rubocop:disable Metrics/ParameterLists
     def initialize(url, content_type = 'yum', upstream_username = nil,
-      upstream_password = nil, search = '*', proxy = {}, crawled = [],
+      upstream_password = nil, search = '*', crawled = [],
       found = [], to_follow = [])
       @uri = uri(url)
       @content_type = content_type
@@ -14,7 +16,6 @@ module Katello
       @found = found
       @crawled = crawled
       @to_follow = to_follow
-      @proxy = proxy
     end
     # rubocop:enable Metrics/ParameterLists
 
@@ -42,42 +43,49 @@ module Katello
     private
 
     def docker_search
-      params = {
-        :accept => :json
+      request_params = {
+        method: :get,
+        headers: { accept: :json },
+        url: "#{@uri}v1/search?q=#{@search}"
       }
-      params[:user] = @upstream_username unless @upstream_username.empty?
-      params[:password] = @upstream_password unless @upstream_password.empty?
-
-      RestClient.proxy = nil
 
-      if @proxy && @proxy[:proxy_host]
-        proxy_uri = URI(@proxy[:proxy_host])
-        proxy_uri.user = @proxy[:proxy_user] if @proxy[:proxy_user]
-        proxy_uri.password = @proxy[:proxy_password] if @proxy[:proxy_password]
-        proxy_uri.port = @proxy[:proxy_port] if @proxy[:proxy_port]
-
-        RestClient.proxy = proxy_uri.to_s
-      end
+      request_params[:headers][:user] = @upstream_username unless @upstream_username.empty?
+      request_params[:headers][:password] = @upstream_password unless @upstream_password.empty?
+      request_params[:proxy] = proxy_uri if proxy
 
       begin
-        results = RestClient.get(@uri.to_s + "v1/search?q=#{@search}", params)
+        results = RestClient::Request.execute(request_params)
         JSON.parse(results)['results'].each do |result|
           @found << result['name']
         end
       rescue
         # Note: v2 endpoint does not support search
-        results = RestClient.get(@uri.to_s + "v2/_catalog", params)
+        request_params[:url] = "#{@uri}v2/_catalog"
+        results = RestClient::Request.execute(request_params)
         @found = JSON.parse(results)['repositories']
       end
       @found.sort!
     end
 
+    def anemone_proxy_details
+      details = {}
+
+      if proxy
+        details[:proxy_host] = proxy_host
+        details[:proxy_port] = proxy_port
+        details[:proxy_user] = proxy.username
+        details[:proxy_password] = proxy.password
+      end
+
+      details
+    end
+
     def http_crawl(resume_point)
       resume_point_uri = URI(resume_point)
       resume_point_uri.user = @upstream_username if @upstream_username
       resume_point_uri.password = @upstream_password if @upstream_password
 
-      Anemone.crawl(resume_point_uri, @proxy) do |anemone|
+      Anemone.crawl(resume_point_uri, anemone_proxy_details) do |anemone|
         anemone.focus_crawl do |page|
           @crawled << page.url.path
 

data/app/lib/katello/resources/cdn/katello_cdn.rb

@@ -4,6 +4,8 @@ module Katello
       class KatelloCdn < CdnResource
         def initialize(url, options)
           @organization_label = options.delete(:organization_label)
+          @content_view_label = options.delete(:content_view_label)
+          @lifecycle_environment_label = options.delete(:lifecycle_environment_label)
           fail ArgumentError, "No upstream organization was specified" if @organization_label.nil?
 
           super
@@ -38,14 +40,50 @@ module Katello
           get("/katello/api/v2/organizations/#{organization['id']}/download_debug_certificate")
         end
 
+        def content_view_id
+          rs = get("/katello/api/v2/organizations/#{organization['id']}/content_views?name=#{CGI.escape(@content_view_label)}")
+          content_view = JSON.parse(rs)['results']&.first
+          if content_view.blank?
+            fail _("Upstream organization %{org_label} does not have a content view with the label %{cv_label}") % { org_label: @organization_label,
+                                                                                                                     cv_label: @content_view_label }
+          end
+          content_view["id"]
+        end
+
+        def lifecycle_environment_id
+          rs = get("/katello/api/v2/organizations/#{organization['id']}/environments")
+          env = JSON.parse(rs)['results'].find { |lce| lce['label'] == @lifecycle_environment_label }
+
+          if env.blank?
+            fail _("Upstream organization %{org_label} does not have a lifecycle environment with the label %{lce_label}") % { org_label: @organization_label,
+                                                                                                                               lce_label: @lifecycle_environment_label }
+          end
+          env["id"]
+        end
+
         def repository_url(content_label:)
-          response = get("/katello/api/v2/organizations/#{organization['id']}/repositories?search=#{CGI.escape("content_label = #{content_label}")}")
+          params = {
+            search: CGI.escape("content_label = #{content_label}")
+          }
+
+          params[:content_view_id] = content_view_id if @content_view_label
+          params[:environment_id] = lifecycle_environment_id if @lifecycle_environment_label
+
+          query_params = params.map { |key, value| "#{key}=#{value}" }
+          url = "/katello/api/v2/organizations/#{organization['id']}/repositories?#{query_params.join('&')}"
+          response = get(url)
           repository = JSON.parse(response)['results'].first
 
           if repository.nil?
-            fail _("Repository with content label %{content_label} was not found in upstream organization %{org_label}") % { content_label: content_label, org_label: @organization_label }
-          end
+            msg_params = { content_label: content_label,
+                           org_label: @organization_label,
+                           cv_label: @content_view_label || Katello::OrganizationCreator::DEFAULT_CONTENT_VIEW_LABEL,
+                           env_label: @lifecycle_environment_label || Katello::OrganizationCreator::DEFAULT_LIFECYCLE_ENV_LABEL
+            }
 
+            fail _("Repository with content label %{content_label} was not found in upstream organization %{org_label},"\
+                   " content view %{cv_label} and lifecycle environment %{env_label} ") % msg_params
+          end
           repository['full_path']
         end
 

data/app/lib/katello/resources/cdn.rb

@@ -58,7 +58,7 @@ module Katello
 
         def self.create(product: nil, cdn_configuration:)
           options = {}
-          if cdn_configuration.redhat?
+          if cdn_configuration.redhat_cdn?
             options[:ssl_client_cert] = OpenSSL::X509::Certificate.new(product.certificate)
             options[:ssl_client_key] = OpenSSL::PKey::RSA.new(product.key)
             options[:ssl_ca_file] = self.ca_file
@@ -67,6 +67,8 @@ module Katello
             options[:username] = cdn_configuration.username
             options[:password] = cdn_configuration.password
             options[:organization_label] = cdn_configuration.upstream_organization_label
+            options[:content_view_label] = cdn_configuration.upstream_content_view_label
+            options[:lifecycle_environment_label] = cdn_configuration.upstream_lifecycle_environment_label
             options[:ssl_ca_cert] = cdn_configuration.ssl_ca
             CDN::KatelloCdn.new(cdn_configuration.url, options)
           end
@@ -184,7 +186,7 @@ module Katello
         def net_http_class
           if proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
-            Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
+            Net::HTTP::Proxy("#{uri.host}", uri.port, proxy.username, proxy.password)
           else
             Net::HTTP
           end

data/app/lib/katello/util/deduplication_migrator.rb

@@ -0,0 +1,105 @@
+module Katello
+  module Util
+    class DeduplicationMigrator # used in db/migrate/20211201154845_add_unique_indexes.rb
+      include ActionView::Helpers::TextHelper
+
+      def models_to_clean
+        [
+          {
+            :model => ::Katello::CapsuleLifecycleEnvironment,
+            :fields => [:lifecycle_environment_id, :capsule_id]
+          },
+          {
+            :model => ::Katello::ContentViewErratumFilterRule,
+            :fields => [:errata_id, :content_view_filter_id]
+          },
+          {
+            :model => ::Katello::ContentViewModuleStreamFilterRule,
+            :fields => [:module_stream_id, :content_view_filter_id]
+          },
+          {
+            :model => ::Katello::ContentViewPackageGroupFilterRule,
+            :fields => [:uuid, :content_view_filter_id]
+          },
+          {
+            :model =>
+            ::Katello::ContentViewRepository,
+            :fields => [:content_view_id, :repository_id]
+          }
+        ]
+      end
+
+      def models_to_rename
+        [
+          {
+            :model => ::Katello::ContentView,
+            :fields => [:name, :organization_id]
+          }
+        ]
+      end
+
+      # example ---- (IDs 1/7 and 6/8 are duplicates ) ::Katello::CapsuleLifecycleEnvironment.all
+      # => [#<Katello::CapsuleLifecycleEnvironment:0x0000000017e11060 id: 1, capsule_id: 1, lifecycle_environment_id: 1>,
+      # #<Katello::CapsuleLifecycleEnvironment:0x0000000017e10f98 id: 2, capsule_id: 1, lifecycle_environment_id: 2>,
+      # #<Katello::CapsuleLifecycleEnvironment:0x0000000017e10ed0 id: 3, capsule_id: 1, lifecycle_environment_id: 3>,
+      # ...
+      # #<Katello::CapsuleLifecycleEnvironment:0x0000000017e10c78 id: 6, capsule_id: 1, lifecycle_environment_id: 6>,
+      # #<Katello::CapsuleLifecycleEnvironment:0x0000000017e10bb0 id: 7, capsule_id: 1, lifecycle_environment_id: 1>,
+      # #<Katello::CapsuleLifecycleEnvironment:0x0000000017e10ac0 id: 8, capsule_id: 1, lifecycle_environment_id: 6>]
+      def execute!
+        models_to_clean.each do |model_to_clean|
+          rows_deleted = 0
+          model = model_to_clean[:model]
+          cleaning_queries(model_to_clean).each { |query| rows_deleted += clean_duplicates(query, model) }
+          if rows_deleted > 0
+            Rails.logger.info("Deleted #{pluralize(rows_deleted, 'duplicate table row')} from #{model.table_name}")
+          end
+        end
+        Rails.logger.info("Finished cleaning duplicate table rows")
+
+        models_to_rename.each do |model_to_rename|
+          rows_renamed = 0
+          model = model_to_rename[:model]
+          cleaning_queries(model_to_rename).each { |query| rows_renamed += rename_duplicates(query, model) }
+          if rows_renamed > 0
+            Rails.logger.info("Renamed #{pluralize(rows_renamed, 'duplicate table row')} from #{model.table_name}")
+          end
+        end
+        Rails.logger.info("Finished renaming duplicate table rows")
+      end
+
+      def cleaning_queries(model_to_clean)
+        model = model_to_clean[:model]
+        fields = model_to_clean[:fields]
+        dup_query = model.group(fields).having("count(*) > 1")
+        duplicate_entries = dup_query.count.try(:keys) #  [[1, 1], [6, 1]] - the set of duplicate combinations
+        return [] if duplicate_entries.blank?
+        min_ids = dup_query.pluck('min(id)') # [1, 6] - the ids of the duplicate entries with the lowest id, in the same order as duplicate_entries
+        duplicate_entries.map.with_index do |entry, idx|
+          # [{:lifecycle_environment_id=>1, :capsule_id=>1, :min_id=>1},
+          # {:lifecycle_environment_id=>6, :capsule_id=>1, :min_id=>6}]
+          Hash[fields.zip(entry)].merge({ min_id: min_ids[idx] })
+        end
+      end
+
+      def clean_duplicates(query, model)
+        min_id = query.delete(:min_id)
+        model.where(query).where.not(id: min_id).delete_all # returns quantity deleted
+      end
+
+      def rename_duplicates(query, model)
+        min_id = query.delete(:min_id)
+        counter = 0
+        model.where(query).where.not(id: min_id).each do |duplicate|
+          counter += 1
+          old_name = duplicate.name
+          new_name = "#{duplicate.name}_#{duplicate.id}"
+          duplicate.name = new_name
+          duplicate.save(validate: false) # skip validation since migrations should never fail
+          Rails.logger.info("Content view #{old_name} (id #{duplicate.id}) renamed to #{new_name}")
+        end
+        counter
+      end
+    end
+  end
+end

data/app/models/katello/candlepin/repository_mapper.rb

@@ -44,7 +44,7 @@ module Katello
       end
 
       def validate!
-        return if katello_content_type == Repository::OSTREE_TYPE || Setting[:content_disconnected]
+        return if katello_content_type == Repository::OSTREE_TYPE || product.organization.cdn_configuration.airgapped?
         substitutor.validate_substitutions(content, substitutions)
       end
 

data/app/models/katello/cdn_configuration.rb

@@ -1,31 +1,63 @@
 module Katello
   class CdnConfiguration < Katello::Model
     include Encryptable
+    self.inheritance_column = nil
+    CDN_TYPE = 'redhat_cdn'.freeze
+    UPSTREAM_SERVER_TYPE = 'upstream_server'.freeze
+    AIRGAPPED_TYPE = 'airgapped'.freeze
+
+    TYPES = [CDN_TYPE, UPSTREAM_SERVER_TYPE, AIRGAPPED_TYPE].freeze
 
     belongs_to :organization, :inverse_of => :cdn_configuration
 
     belongs_to :ssl_ca_credential, :class_name => "Katello::ContentCredential", :inverse_of => :ssl_ca_cdn_configurations
 
     encrypts :password
+    validates :password, presence: true, if: :upstream_server?
+    validates :username, presence: true, if: :upstream_server?
+    validates :upstream_organization_label, presence: true, if: :upstream_server?
 
-    validates :url, presence: true
-    validates_with Validators::KatelloUrlFormatValidator, attributes: :url
+    validates :url, presence: true, unless: :airgapped?
+    validates_with Validators::KatelloUrlFormatValidator, attributes: :url, unless: :airgapped?
     validates_with Validators::KatelloLabelFormatValidator, attributes: :upstream_organization_label, if: proc { upstream_organization_label.present? }
-    validate :non_redhat_configuration, unless: :redhat?
+    validate :non_redhat_configuration, if: :upstream_server?
+
+    before_validation :reset_fields
 
     def ssl_ca
       ssl_ca_credential&.content
     end
 
-    def redhat?
-      username.blank? && password.blank? && upstream_organization_label.blank? && ssl_ca_credential_id.blank?
+    def redhat_cdn?
+      type == CDN_TYPE
+    end
+
+    def airgapped?
+      type == AIRGAPPED_TYPE
+    end
+
+    def upstream_server?
+      type == UPSTREAM_SERVER_TYPE
     end
 
     private
 
+    def reset_fields
+      return if upstream_server?
+
+      self.url = nil if airgapped?
+      self.url ||= SETTINGS[:katello][:redhat_repository_url] if redhat_cdn?
+      self.username = nil
+      self.password = nil
+      self.upstream_organization_label = nil
+      self.ssl_ca_credential_id = nil
+      self.upstream_content_view_label = nil
+      self.upstream_lifecycle_environment_label = nil
+    end
+
     def non_redhat_configuration
       if username.blank? || password.blank? || upstream_organization_label.blank? || ssl_ca_credential_id.blank?
-        errors.add(:base, _("Username, Password, Upstream Organization Label, and SSL CA Credential are required when using a non-Red Hat CDN."))
+        errors.add(:base, _("Username, Password, Upstream Organization Label, and SSL CA Credential are required when using an upstream Foreman server."))
       end
     end
   end

data/app/models/katello/concerns/host_managed_extensions.rb

@@ -28,9 +28,21 @@ module Katello
             content_facet.kickstart_repository_id = valid_repos.first
           end
         end
+
+        def remote_execution_proxies(provider, *_rest)
+          proxies = super
+          if (name = subscription_facet&.registered_through)
+            registered_through = SmartProxy.with_features(provider)
+                                           .authorized
+                                           .where(name: name)
+          end
+          proxies[:registered_through] = registered_through || []
+          proxies
+        end
       end
 
       included do
+        prepend ::ForemanRemoteExecution::HostExtensions if ::Katello.with_remote_execution?
         prepend Overrides
 
         delegate :content_source_id, :content_view_id, :lifecycle_environment_id, :kickstart_repository_id, to: :content_facet, allow_nil: true
@@ -388,6 +400,16 @@ module Katello
         ::Katello::HostTracer.helpers_for(traces)
       end
 
+      def package_names_for_job_template(action:, search:)
+        actions = ['install']
+        case action
+        when 'install'
+          ::Katello::Rpm.yum_installable_for_host(self).search_for(search).distinct.pluck(:name)
+        else
+          fail ::Foreman::Exception.new(N_("package_names_for_job_template: Action must be one of %s"), actions.join(', '))
+        end
+      end
+
       def advisory_ids(search:)
         ::Katello::Erratum.installable_for_hosts([self]).search_for(search).pluck(:errata_id)
       end
@@ -414,7 +436,7 @@ end
 class ::Host::Managed::Jail < Safemode::Jail
   allow :content_source, :subscription_manager_configuration_url, :rhsm_organization_label,
         :host_collections, :pools, :hypervisor_host, :lifecycle_environment, :content_view,
-        :installed_packages, :traces_helpers, :advisory_ids
+        :installed_packages, :traces_helpers, :advisory_ids, :package_names_for_job_template
 end
 
 class ActiveRecord::Associations::CollectionProxy::Jail < Safemode::Jail

data/app/models/katello/concerns/organization_extensions.rb

@@ -55,6 +55,10 @@ module Katello
         # so that it will execute after :dependent => :destroy
         before_destroy :destroy_taxable_taxonomies
 
+        def self.non_audited_columns
+          super | [:created_in_katello]
+        end
+
         def default_content_view
           ContentView.default.where(:organization_id => self.id).first
         end
@@ -193,7 +197,7 @@ module Katello
 
         def latest_repo_discovery
           ForemanTasks::Task::DynflowTask.for_action(::Actions::Katello::Repository::Discover)
-            .for_resource(::User.current).order("started_at").last
+            .where(user: ::User.current).order("started_at").last
         end
 
         def cancel_repo_discovery