katello 4.12.0.rc3 → 4.13.0.rc1

data/app/lib/katello/resources/cdn.rb

@@ -1,8 +1,6 @@
 module Katello
   module Resources
     module CDN
-      SUPPORTED_SSL_VERSIONS = ['SSLv23', 'TLSv1'].freeze
-
       class Utils
         # takes releasever from contentUrl (e.g. 6Server, 6.0, 6.1)
         # returns hash e.g. {:major => 6, :minor => "6.1"}
@@ -24,11 +22,6 @@ module Katello
         end
 
         def initialize(url, options = {})
-          @ssl_version = Setting[:cdn_ssl_version]
-          if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
-            fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
-          end
-
           options.reverse_merge!(:verify_ssl => 9)
           options.assert_valid_keys(:ssl_client_key,
                                     :ssl_client_cert,
@@ -106,12 +99,10 @@ module Katello
             net.cert_store = @cert_store
           end
 
-          # NOTE: This was added because some proxies dont support SSLv23 and do not handle TLS 1.2
-          # Valid values in ruby 1.9.3 are 'SSLv23' or 'TLSV1'
-          # Run the following command in rails console to figure out other
-          # valid constants in other ruby versions
-          # "OpenSSL::SSL::SSLContext::METHODS"
-          net.ssl_version = @ssl_version
+          # NOTE: This is only here due to https://github.com/ruby/openssl/issues/709, otherwise the
+          # system-wide crypto policy could be used. Enforcing TLS version >= 1.2  will prevent using
+          # very old infrastructure for now, but that was considered better than having an insecure default.
+          net.min_version = OpenSSL::SSL::TLS1_2_VERSION
 
           if (@options[:verify_ssl] == false) || (@options[:verify_ssl] == OpenSSL::SSL::VERIFY_NONE)
             net.verify_mode = OpenSSL::SSL::VERIFY_NONE

data/app/lib/katello/resources/registry.rb

@@ -18,10 +18,35 @@ module Katello
           client.options.merge!(options)
           client.get(headers)
         end
+
+        def self.put(path, body, headers)
+          logger.debug "Sending PUT request to Registry: #{path}"
+          resource = RegistryResource.load_class
+          joined_path = resource.prefix.chomp("/") + path
+          resource.issue_request(method: :put, path: joined_path, headers: headers, payload: body)
+        end
+
+        def self.patch(path, body, headers)
+          logger.debug "Sending PATCH request to Registry: #{path}"
+          resource = RegistryResource.load_class
+          joined_path = resource.prefix.chomp("/") + path
+          resource.issue_request(method: :patch, path: joined_path, headers: headers, payload: body)
+        end
+
+        def self.post(path, body, headers)
+          logger.debug "Sending PUT request to Registry: #{path}"
+          resource = RegistryResource.load_class
+          joined_path = resource.prefix.chomp("/") + path
+          resource.issue_request(method: :post, path: joined_path, headers: headers, payload: body)
+        end
       end
 
       class RegistryResource < HttpResource
         class << self
+          def logger
+            ::Foreman::Logging.logger('katello/registry_proxy')
+          end
+
           def load_class
             pulp_primary = ::SmartProxy.pulp_primary
             content_app_url = pulp_primary.setting(SmartProxy::PULP3_FEATURE, 'content_app_url')

data/app/mailers/katello/subscription_mailer.rb

@@ -10,14 +10,11 @@ module Katello
 
       ::User.as(user.login) do
         @pools = Katello::Pool.readable.expiring_in_days(days_from_now)
-        @affected_hosts = ::Host::Managed.with_pools_expiring_in_days(days_from_now)
       end
 
-      if @affected_hosts.any?
-        start_report_task(days_from_now)
-        @report_url = report_url
-        @report_link = report_link
-      end
+      start_report_task(days_from_now)
+      @report_url = report_url
+      @report_link = report_link
 
       set_locale_for(user) do
         mail(:to => user.mail, :subject => _("You have subscriptions expiring within %s days") % days_from_now)

data/app/models/katello/candlepin/repository_mapper.rb

@@ -74,7 +74,7 @@ module Katello
       end
 
       def relative_path
-        ::Katello::Glue::Pulp::Repos.repo_path_from_content_path(product.organization.library, path)
+        ::Katello::Repository.repo_path_from_content_path(product.organization.library, path)
       end
 
       def prune_substitutions(subs, url)

data/app/models/katello/concerns/organization_extensions.rb

@@ -100,16 +100,55 @@ module Katello
           end
         end
 
-        def manifest_expired?
-          manifest_expiry = owner_details.dig(:upstreamConsumer, :idCert, :serial, :expiration)
+        def manifest_expiration_date(cached: true)
+          Rails.cache.fetch("#{self.label}_manifest_expiration_date", expires_in: 1.minute, force: !cached) do
+            unless manifest_imported?(cached: cached)
+              Rails.logger.error "Manifest not imported for organization #{self.label}"
+              return nil
+            end
+            manifest_expiry = owner_details.dig(:upstreamConsumer, :idCert, :serial, :expiration)
+
+            if manifest_expiry.present?
+              DateTime.parse(manifest_expiry)
+            else
+              Rails.logger.error "Unable to parse manifest expiration date from owner details"
+              nil
+            end
+          end
+        end
+
+        def manifest_expired?(cached: true)
+          manifest_expiry = manifest_expiration_date(cached: cached)
 
           if manifest_expiry
-            DateTime.parse(manifest_expiry) < DateTime.now
+            manifest_expiry < DateTime.now
           else
             false
           end
         end
 
+        def manifest_expiring_soon?(days = Setting[:expire_soon_days])
+          return false if !manifest_imported? || manifest_expired?
+          manifest_expiry = manifest_expiration_date
+
+          if manifest_expiry
+            manifest_expiry < DateTime.now + days.days
+          else
+            false
+          end
+        end
+
+        def manifest_expire_days_remaining
+          manifest_expiry = manifest_expiration_date
+          return 0 if manifest_expired?
+
+          if manifest_expiry
+            (manifest_expiry - DateTime.now).to_i
+          else
+            0
+          end
+        end
+
         def manifest_history
           imports.map { |i| OpenStruct.new(i) }
         end

data/app/models/katello/content_view.rb

@@ -634,6 +634,7 @@ module Katello
         check_ready_to_import!
       else
         fail _("Import-only content views can not be published directly") if import_only? && !syncable
+        check_repositories_blocking_publish!
         check_composite_action_allowed!(organization.library)
         check_docker_repository_names!([organization.library])
         check_orphaned_content_facets!(environments: self.environments)
@@ -642,6 +643,16 @@ module Katello
       true
     end
 
+    def check_repositories_blocking_publish!
+      blocking_tasks = repositories&.map { |repo| repo.blocking_task }&.compact
+
+      if blocking_tasks&.any?
+        errored_tasks = blocking_tasks.uniq.map { |task| "- #{Setting['foreman_url']}/foreman_tasks/tasks/#{task&.id}" }.join("\n")
+        fail _("Pending tasks detected in repositories of this content view. Please wait for the tasks: " +
+                 errored_tasks + " before publishing.")
+      end
+    end
+
     def check_docker_repository_names!(environments)
       environments.each do |environment|
         repositories = []
@@ -789,6 +800,10 @@ module Katello
       repositories.any? { |repo| repo.last_indexed && repo.last_indexed > latest_version_object.created_at }
     end
 
+    def unpublishable?
+      default? || import_only? || generated?
+    end
+
     def needs_publish?
       #Returns
       # True:
@@ -808,7 +823,9 @@ module Katello
       #     a) No changes were detected via audits *and*
       #        Audit for CV publish exists (Audits haven't been cleaned up)
       #        *and* applied_filters field is set(Published after upgrade)
+      #     b) Default, import only and generated CVs can not be published, hence these will always return false.
       #
+      return false if unpublishable?
       return true unless latest_version_object
       return nil unless last_publish_task_success?
       return composite_cv_components_changed? if composite?
@@ -868,6 +885,17 @@ module Katello
       filters.present?
     end
 
+    def blocking_task
+      blocking_task_labels = [
+        ::Actions::Katello::ContentView::Publish.name
+      ]
+      ForemanTasks::Task::DynflowTask.where(:label => blocking_task_labels)
+                                     .where.not(state: 'stopped')
+                                     .for_resource(self)
+                                     .order(:started_at)
+                                     .last
+    end
+
     protected
 
     def remove_repository(repository)

data/app/models/katello/content_view_environment_content_facet.rb

@@ -5,7 +5,7 @@ module Katello
 
     validates :content_view_environment_id, presence: true
     validates :content_facet_id, presence: true, unless: :new_record?
-    validate :ensure_valid_content_source
+    validate :ensure_valid_content_source, if: proc { Setting['validate_host_lce_content_source_coherence'] }
 
     def ensure_valid_content_source
       source = self.content_facet&.content_source
@@ -14,7 +14,9 @@ module Katello
       hostname = self.content_facet&.host&.name
       return unless [source, env].all? { |x| x.present? }
       unless source.lifecycle_environments.include?(env)
-        errors.add(:base, _("Host %{hostname}: Cannot add content view environment to content facet. The host's content source '%{content_source}' does not sync lifecycle environment '%{lce}'.") % { hostname: hostname, content_source: source.name, lce: env.name })
+        error_msg = _("Host %{hostname}: Cannot add content view environment to content facet. The host's content source '%{content_source}' does not sync lifecycle environment '%{lce}'.") % { hostname: hostname, content_source: source.name, lce: env.name }
+        Rails.logger.warn error_msg
+        errors.add(:base, error_msg)
       end
     end
   end

data/app/models/katello/glue/provider.rb

@@ -32,6 +32,10 @@ module Katello
 
     module InstanceMethods
       API_URL = 'https://subscription.rhsm.redhat.com/subscription/consumers/'.freeze
+      def api_url(upstream = {})
+        # Default to Red Hat
+        upstream['apiUrl'] || API_URL
+      end
 
       def sync
         Rails.logger.debug "Syncing provider #{name}"
@@ -62,34 +66,37 @@ module Katello
           fail _("Upstream identity certificate not available")
         end
 
-        # Default to Red Hat
-        url = upstream['apiUrl'] || API_URL
-
         params = {}
         params[:capabilities] = Resources::Candlepin::CandlepinPing.ping['managerCapabilities'].inject([]) do |result, element|
           result << {'name' => element}
         end
         params[:facts] = {:distributor_version => DISTRIBUTOR_VERSION }
-        Resources::Candlepin::UpstreamConsumer.update("#{url}#{upstream['uuid']}", upstream['idCert']['cert'],
+        Resources::Candlepin::UpstreamConsumer.update("#{api_url(upstream)}#{upstream['uuid']}", upstream['idCert']['cert'],
                                                       upstream['idCert']['key'], ca_file, params)
       end
 
-      def start_owner_upstream_export(upstream)
+      def owner_upstream_regenerate_identity_cert(upstream)
         validate_upstream_identity_cert!(upstream)
-        url = upstream['apiUrl'] || API_URL
+        Rails.logger.debug "Sending request to regenerate identity certificate for upstream consumer: #{upstream['uuid']}"
+        response = Resources::Candlepin::UpstreamConsumer.regenerate_upstream_identity("#{api_url(upstream)}#{upstream['uuid']}", upstream['idCert']['cert'],
+          upstream['idCert']['key'], ca_file)
+        JSON.parse(response)
+      end
 
-        response = Resources::Candlepin::UpstreamConsumer.get_export("#{url}#{upstream['uuid']}/export/async", upstream['idCert']['cert'],
+      def start_owner_upstream_export(upstream)
+        validate_upstream_identity_cert!(upstream)
+        response = Resources::Candlepin::UpstreamConsumer.start_upstream_export("#{api_url(upstream)}#{upstream['uuid']}/export/async", upstream['idCert']['cert'],
           upstream['idCert']['key'], ca_file)
         JSON.parse(response)
       end
 
       def retrieve_owner_upstream_export(upstream, zip_file_path, export_id)
         validate_upstream_identity_cert!(upstream)
-        url = upstream['apiUrl'] || API_URL
-
-        data = Resources::Candlepin::UpstreamConsumer.get_export("#{url}#{upstream['uuid']}/export/#{export_id}", upstream['idCert']['cert'],
-                                                             upstream['idCert']['key'], ca_file)
-
+        data = Resources::Candlepin::UpstreamConsumer.retrieve_upstream_export(
+          "#{api_url(upstream)}#{upstream['uuid']}/export/#{export_id}",
+          upstream['idCert']['cert'],
+          upstream['idCert']['key'], ca_file
+        )
         File.write(zip_file_path, data, mode: 'wb')
 
         true

data/app/models/katello/glue/pulp/repos.rb

@@ -6,12 +6,6 @@ module Katello
       base.send :include, InstanceMethods
     end
 
-    def self.repo_path_from_content_path(environment, content_path)
-      path = content_path.sub(%r|^/|, '')
-      path_prefix = [environment.organization.label, environment.label].join('/')
-      "#{path_prefix}/#{path}"
-    end
-
     module InstanceMethods
       def distributions(env)
         to_ret = []
@@ -56,7 +50,7 @@ module Katello
       end
 
       def last_sync_audit
-        Audited::Audit.where(:auditable_id => self.repositories, :auditable_type => Katello::Repository.name).order(:created_at).last
+        Audited::Audit.where(:auditable_id => self.repositories, :auditable_type => Katello::Repository.name, :action => "sync").order(:created_at).last
       end
 
       def last_sync
@@ -64,13 +58,14 @@ module Katello
       end
 
       def last_repo_sync_task
-        @last_sync_task ||= last_repo_sync_tasks.first
+        @last_sync_task ||= last_repo_sync_tasks&.first
       end
 
       def last_repo_sync_tasks
         ids = repos(self.library, nil, false).pluck(:id).join(',')
         label = ::Actions::Katello::Repository::Sync.name
         type = ::Katello::Repository.name
+        return nil if ids.empty?
         ForemanTasks::Task.search_for("label = #{label} and resource_type = #{type} and resource_id ^ (#{ids})")
           .order("started_at desc")
       end

data/app/models/katello/host/content_facet.rb

@@ -44,7 +44,7 @@ module Katello
 
       validates_with ::AssociationExistsValidator, attributes: [:content_source]
       validates_with Katello::Validators::GeneratedContentViewValidator
-      validates_associated :content_view_environment_content_facets
+      validates_associated :content_view_environment_content_facets, :message => _("invalid: The content source must sync the lifecycle environment assigned to the host. See the logs for more information.")
       validates :host, :presence => true, :allow_blank => false
 
       attr_accessor :cves_changed

data/app/models/katello/host/subscription_facet.rb

@@ -67,7 +67,7 @@ module Katello
           self.purpose_addon_ids = consumer_params['addOns'].map { |addon_name| ::Katello::PurposeAddon.find_or_create_by(name: addon_name).id }
         end
 
-        unless consumer_params['releaseVer'].blank?
+        unless consumer_params['releaseVer'].nil?
           release = consumer_params['releaseVer']
           release = release['releaseVer'] if release.is_a?(Hash)
           self.release_version = release

data/app/models/katello/host_collection.rb

@@ -29,7 +29,10 @@ module Katello
     scoped_search :relation => :hosts, :on => :name, :rename => :host, :complete_value => true
 
     def max_hosts_check
-      if !unlimited_hosts && (hosts.length > 0 && (hosts.length.to_i > max_hosts.to_i)) && max_hosts_changed?
+      # NOTE: max_hosts_check and max_hosts_no_exceeded use size() instead of count() because
+      # the host list exists as an array rather than a DB query when run as a validation.
+      host_count = hosts.size
+      if !unlimited_hosts && (host_count > 0 && (host_count.to_i > max_hosts.to_i)) && max_hosts_changed?
         errors.add :max_host, N_("may not be less than the number of hosts associated with the host collection.")
       end
     end
@@ -60,8 +63,14 @@ module Katello
       type ? query.of_type(type) : query
     end
 
-    def total_hosts
-      hosts.length
+    def cache_key
+      "#{self.class.name}/#{self.id}"
+    end
+
+    def total_hosts(cached: false)
+      Rails.cache.fetch("#{cache_key}/total_hosts", expires_in: 1.minute, force: !cached) do
+        hosts.count
+      end
     end
 
     # Retrieve the list of accessible host collections in the organization specified, returning

data/app/models/katello/ping.rb

@@ -2,7 +2,7 @@ module Katello
   class Ping
     OK_RETURN_CODE = 'ok'.freeze
     FAIL_RETURN_CODE = 'FAIL'.freeze
-    PACKAGES = %w(katello candlepin pulp foreman hammer).freeze
+    PACKAGES = %w(katello candlepin pulp foreman hammer dynflow).freeze
 
     class << self
       def services(capsule_id = nil)

data/app/models/katello/repository.rb

@@ -201,6 +201,12 @@ module Katello
       joins(:root).where("#{Katello::RootRepository.table_name}.product_id" => products)
     end
 
+    def self.repo_path_from_content_path(environment, content_path)
+      path = content_path.sub(%r|^/|, '')
+      path_prefix = [environment.organization.label, environment.label].join('/')
+      "#{path_prefix}/#{path}"
+    end
+
     def to_label
       name
     end
@@ -645,6 +651,33 @@ module Katello
                                 for_resource(self).order(:started_at).last
     end
 
+    def blocking_task
+      blocking_task_labels = [
+        ::Actions::Katello::Repository::Sync.name,
+        ::Actions::Katello::Repository::UploadFiles.name,
+        ::Actions::Katello::Repository::RemoveContent.name,
+        ::Actions::Katello::Repository::MetadataGenerate.name
+      ]
+      ForemanTasks::Task::DynflowTask.where(:label => blocking_task_labels)
+                                     .where.not(state: 'stopped')
+                                     .for_resource(self)
+                                     .order(:started_at)
+                                     .last
+    end
+
+    def check_ready_to_act!
+      blocking_tasks = content_views&.map { |cv| cv.blocking_task }&.compact
+
+      if blocking_tasks&.any?
+        errored_tasks = blocking_tasks
+                          .uniq
+                          .map { |task| "- #{Setting['foreman_url']}/foreman_tasks/tasks/#{task&.id}" }
+                          .join("\n")
+        fail _("This repository has pending tasks in associated content views. Please wait for the tasks: " + errored_tasks +
+               " to complete before proceeding.")
+      end
+    end
+
     # returns other instances of this repo with the same library
     # equivalent of repo
     def environmental_instances(view)

data/app/models/katello/root_repository.rb

@@ -387,10 +387,6 @@ module Katello
       Katello::RepositoryTypeManager.generic_repository_types(false).values.map(&:id).map(&:to_s).flatten.include? self.content_type
     end
 
-    def metadata_generate_needed?
-      (%w(unprotected checksum_type container_repsoitory_name) & previous_changes.keys).any?
-    end
-
     def using_mirrored_content?
       self.mirroring_policy != Katello::RootRepository::MIRRORING_POLICY_ADDITIVE
     end

data/app/services/katello/content_unit_indexer.rb

@@ -55,6 +55,15 @@ module Katello
       end
     end
 
+    def reimport_units
+      units_from_pulp.each do |units|
+        to_update = units.map do |unit|
+          @service_class.generate_model_row(unit)
+        end
+        @model_class.upsert_all(to_update, unique_by: :pulp_id)
+      end
+    end
+
     def import_associations(units)
       pulp_id_to_id = self.class.pulp_id_to_id_map(@content_type, units.map { |unit| unit[@service_class.unit_identifier] })
       @service_class.insert_child_associations(units, pulp_id_to_id) if @service_class.respond_to?(:insert_child_associations)

data/app/services/katello/pulp3/alternate_content_source.rb

@@ -88,18 +88,16 @@ module Katello
       end
 
       def get_remote(href = smart_proxy_acs.remote_href)
-        acs.base_url&.start_with?('uln') ? api.remotes_uln_api.read(href) : api.remotes_api.read(href)
+        api.get_remotes_api(href: href).read(href)
       end
 
-      def update_remote
-        api.remotes_api.partial_update(smart_proxy_acs.remote_href, remote_options)
+      def update_remote(href = smart_proxy_acs.remote_href)
+        api.get_remotes_api(href: href).partial_update(href, remote_options)
       end
 
-      # The old repo URL is needed to determine which remote API to use.
       def delete_remote(options = {})
         options[:href] ||= smart_proxy_acs.remote_href
-        options[:old_url] ||= remote_options[:url]
-        ignore_404_exception { options[:old_url]&.start_with?('uln') ? api.remotes_uln_api.delete(options[:href]) : api.remotes_api.delete(options[:href]) } if options[:href]
+        ignore_404_exception { api.get_remotes_api(href: options[:href]).delete(options[:href]) } if options[:href]
       end
 
       def create
@@ -108,7 +106,7 @@ module Katello
           if acs.content_type == ::Katello::Repository::FILE_TYPE && acs.subpaths.present?
             paths = insert_pulp_manifest!(paths)
           end
-          response = api.alternate_content_source_api.create(name: generate_backend_object_name, paths: paths,
+          response = api.alternate_content_source_api.create(name: generate_backend_object_name, paths: paths.sort,
                                                              remote: smart_proxy_acs.remote_href)
           smart_proxy_acs.update!(alternate_content_source_href: response.pulp_href)
           return response
@@ -125,7 +123,7 @@ module Katello
         if acs.content_type == ::Katello::Repository::FILE_TYPE && acs.subpaths.present?
           paths = insert_pulp_manifest!(paths)
         end
-        api.alternate_content_source_api.update(href, name: generate_backend_object_name, paths: paths, remote: smart_proxy_acs.remote_href)
+        api.alternate_content_source_api.update(href, name: generate_backend_object_name, paths: paths.sort, remote: smart_proxy_acs.remote_href)
       end
 
       def delete_alternate_content_source

data/app/services/katello/pulp3/api/core.rb

@@ -47,6 +47,11 @@ module Katello
           fail NotImplementedError
         end
 
+        # Method is called with either :url or :href parameters for the sake of yum content.
+        def get_remotes_api(*)
+          remotes_api
+        end
+
         def publications_api
           repository_type.publications_api_class.new(api_client) #Optional
         end
@@ -136,6 +141,10 @@ module Katello
           client
         end
 
+        def repair_api
+          PulpcoreClient::RepairApi.new(core_api_client)
+        end
+
         def uploads_api
           PulpcoreClient::UploadsApi.new(core_api_client)
         end
@@ -230,6 +239,10 @@ module Katello
           end
         end
 
+        def repair
+          repair_api.post(PulpcoreClient::Repair.new(verify_checksums: true))
+        end
+
         def self.fetch_from_list
           page_size = Setting[:bulk_load_size]
           page_opts = { "offset" => 0, limit: page_size }

data/app/services/katello/pulp3/api/yum.rb

@@ -32,6 +32,17 @@ module Katello
           PulpRpmClient::RemotesUlnApi.new(api_client)
         end
 
+        def get_remotes_api(href: nil, url: nil)
+          fail 'Provide exactly one of href or url for yum remote selection!' if url.blank? && href.blank?
+          fail 'The href must be a pulp_rpm remote href!' if href && !href.start_with?('/pulp/api/v3/remotes/rpm/')
+
+          if href&.start_with?('/pulp/api/v3/remotes/rpm/uln/') || url&.start_with?('uln')
+            remotes_uln_api
+          else
+            remotes_api
+          end
+        end
+
         def copy_api
           PulpRpmClient::RpmCopyApi.new(api_client)
         end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -26,7 +26,11 @@ module Katello
         {
           schema_version: unit['schema_version'],
           digest: unit['digest'],
-          pulp_id: unit[unit_identifier]
+          pulp_id: unit[unit_identifier],
+          annotations: unit['annotations'],
+          labels: unit['labels'],
+          is_bootable: unit['is_bootable'],
+          is_flatpak: unit['is_flatpak']
         }
       end
     end

data/app/services/katello/pulp3/repository/generic.rb

@@ -3,7 +3,7 @@ module Katello
     class Repository
       class Generic < ::Katello::Pulp3::Repository
         def copy_content_for_source(source_repository, _options = {})
-          copy_units_by_href(source_repository.files.pluck(:pulp_id))
+          copy_units_by_href(source_repository.generic_content_units&.pluck(:pulp_id))
         end
 
         def distribution_options(path)