katello 4.12.0 → 4.13.0.rc1

data/app/models/katello/concerns/organization_extensions.rb

@@ -100,16 +100,55 @@ module Katello
           end
         end
 
-        def manifest_expired?
-          manifest_expiry = owner_details.dig(:upstreamConsumer, :idCert, :serial, :expiration)
+        def manifest_expiration_date(cached: true)
+          Rails.cache.fetch("#{self.label}_manifest_expiration_date", expires_in: 1.minute, force: !cached) do
+            unless manifest_imported?(cached: cached)
+              Rails.logger.error "Manifest not imported for organization #{self.label}"
+              return nil
+            end
+            manifest_expiry = owner_details.dig(:upstreamConsumer, :idCert, :serial, :expiration)
+
+            if manifest_expiry.present?
+              DateTime.parse(manifest_expiry)
+            else
+              Rails.logger.error "Unable to parse manifest expiration date from owner details"
+              nil
+            end
+          end
+        end
+
+        def manifest_expired?(cached: true)
+          manifest_expiry = manifest_expiration_date(cached: cached)
 
           if manifest_expiry
-            DateTime.parse(manifest_expiry) < DateTime.now
+            manifest_expiry < DateTime.now
           else
             false
           end
         end
 
+        def manifest_expiring_soon?(days = Setting[:expire_soon_days])
+          return false if !manifest_imported? || manifest_expired?
+          manifest_expiry = manifest_expiration_date
+
+          if manifest_expiry
+            manifest_expiry < DateTime.now + days.days
+          else
+            false
+          end
+        end
+
+        def manifest_expire_days_remaining
+          manifest_expiry = manifest_expiration_date
+          return 0 if manifest_expired?
+
+          if manifest_expiry
+            (manifest_expiry - DateTime.now).to_i
+          else
+            0
+          end
+        end
+
         def manifest_history
           imports.map { |i| OpenStruct.new(i) }
         end

data/app/models/katello/content_view.rb

@@ -634,6 +634,7 @@ module Katello
         check_ready_to_import!
       else
         fail _("Import-only content views can not be published directly") if import_only? && !syncable
+        check_repositories_blocking_publish!
         check_composite_action_allowed!(organization.library)
         check_docker_repository_names!([organization.library])
         check_orphaned_content_facets!(environments: self.environments)
@@ -642,6 +643,16 @@ module Katello
       true
     end
 
+    def check_repositories_blocking_publish!
+      blocking_tasks = repositories&.map { |repo| repo.blocking_task }&.compact
+
+      if blocking_tasks&.any?
+        errored_tasks = blocking_tasks.uniq.map { |task| "- #{Setting['foreman_url']}/foreman_tasks/tasks/#{task&.id}" }.join("\n")
+        fail _("Pending tasks detected in repositories of this content view. Please wait for the tasks: " +
+                 errored_tasks + " before publishing.")
+      end
+    end
+
     def check_docker_repository_names!(environments)
       environments.each do |environment|
         repositories = []
@@ -789,6 +800,10 @@ module Katello
       repositories.any? { |repo| repo.last_indexed && repo.last_indexed > latest_version_object.created_at }
     end
 
+    def unpublishable?
+      default? || import_only? || generated?
+    end
+
     def needs_publish?
       #Returns
       # True:
@@ -808,7 +823,9 @@ module Katello
       #     a) No changes were detected via audits *and*
       #        Audit for CV publish exists (Audits haven't been cleaned up)
       #        *and* applied_filters field is set(Published after upgrade)
+      #     b) Default, import only and generated CVs can not be published, hence these will always return false.
       #
+      return false if unpublishable?
       return true unless latest_version_object
       return nil unless last_publish_task_success?
       return composite_cv_components_changed? if composite?
@@ -868,6 +885,17 @@ module Katello
       filters.present?
     end
 
+    def blocking_task
+      blocking_task_labels = [
+        ::Actions::Katello::ContentView::Publish.name
+      ]
+      ForemanTasks::Task::DynflowTask.where(:label => blocking_task_labels)
+                                     .where.not(state: 'stopped')
+                                     .for_resource(self)
+                                     .order(:started_at)
+                                     .last
+    end
+
     protected
 
     def remove_repository(repository)

data/app/models/katello/content_view_environment_content_facet.rb

@@ -5,7 +5,7 @@ module Katello
 
     validates :content_view_environment_id, presence: true
     validates :content_facet_id, presence: true, unless: :new_record?
-    validate :ensure_valid_content_source
+    validate :ensure_valid_content_source, if: proc { Setting['validate_host_lce_content_source_coherence'] }
 
     def ensure_valid_content_source
       source = self.content_facet&.content_source
@@ -14,7 +14,9 @@ module Katello
       hostname = self.content_facet&.host&.name
       return unless [source, env].all? { |x| x.present? }
       unless source.lifecycle_environments.include?(env)
-        errors.add(:base, _("Host %{hostname}: Cannot add content view environment to content facet. The host's content source '%{content_source}' does not sync lifecycle environment '%{lce}'.") % { hostname: hostname, content_source: source.name, lce: env.name })
+        error_msg = _("Host %{hostname}: Cannot add content view environment to content facet. The host's content source '%{content_source}' does not sync lifecycle environment '%{lce}'.") % { hostname: hostname, content_source: source.name, lce: env.name }
+        Rails.logger.warn error_msg
+        errors.add(:base, error_msg)
       end
     end
   end

data/app/models/katello/glue/provider.rb

@@ -32,6 +32,10 @@ module Katello
 
     module InstanceMethods
       API_URL = 'https://subscription.rhsm.redhat.com/subscription/consumers/'.freeze
+      def api_url(upstream = {})
+        # Default to Red Hat
+        upstream['apiUrl'] || API_URL
+      end
 
       def sync
         Rails.logger.debug "Syncing provider #{name}"
@@ -62,34 +66,37 @@ module Katello
           fail _("Upstream identity certificate not available")
         end
 
-        # Default to Red Hat
-        url = upstream['apiUrl'] || API_URL
-
         params = {}
         params[:capabilities] = Resources::Candlepin::CandlepinPing.ping['managerCapabilities'].inject([]) do |result, element|
           result << {'name' => element}
         end
         params[:facts] = {:distributor_version => DISTRIBUTOR_VERSION }
-        Resources::Candlepin::UpstreamConsumer.update("#{url}#{upstream['uuid']}", upstream['idCert']['cert'],
+        Resources::Candlepin::UpstreamConsumer.update("#{api_url(upstream)}#{upstream['uuid']}", upstream['idCert']['cert'],
                                                       upstream['idCert']['key'], ca_file, params)
       end
 
-      def start_owner_upstream_export(upstream)
+      def owner_upstream_regenerate_identity_cert(upstream)
         validate_upstream_identity_cert!(upstream)
-        url = upstream['apiUrl'] || API_URL
+        Rails.logger.debug "Sending request to regenerate identity certificate for upstream consumer: #{upstream['uuid']}"
+        response = Resources::Candlepin::UpstreamConsumer.regenerate_upstream_identity("#{api_url(upstream)}#{upstream['uuid']}", upstream['idCert']['cert'],
+          upstream['idCert']['key'], ca_file)
+        JSON.parse(response)
+      end
 
-        response = Resources::Candlepin::UpstreamConsumer.get_export("#{url}#{upstream['uuid']}/export/async", upstream['idCert']['cert'],
+      def start_owner_upstream_export(upstream)
+        validate_upstream_identity_cert!(upstream)
+        response = Resources::Candlepin::UpstreamConsumer.start_upstream_export("#{api_url(upstream)}#{upstream['uuid']}/export/async", upstream['idCert']['cert'],
           upstream['idCert']['key'], ca_file)
         JSON.parse(response)
       end
 
       def retrieve_owner_upstream_export(upstream, zip_file_path, export_id)
         validate_upstream_identity_cert!(upstream)
-        url = upstream['apiUrl'] || API_URL
-
-        data = Resources::Candlepin::UpstreamConsumer.get_export("#{url}#{upstream['uuid']}/export/#{export_id}", upstream['idCert']['cert'],
-                                                             upstream['idCert']['key'], ca_file)
-
+        data = Resources::Candlepin::UpstreamConsumer.retrieve_upstream_export(
+          "#{api_url(upstream)}#{upstream['uuid']}/export/#{export_id}",
+          upstream['idCert']['cert'],
+          upstream['idCert']['key'], ca_file
+        )
         File.write(zip_file_path, data, mode: 'wb')
 
         true

data/app/models/katello/glue/pulp/repos.rb

@@ -6,12 +6,6 @@ module Katello
       base.send :include, InstanceMethods
     end
 
-    def self.repo_path_from_content_path(environment, content_path)
-      path = content_path.sub(%r|^/|, '')
-      path_prefix = [environment.organization.label, environment.label].join('/')
-      "#{path_prefix}/#{path}"
-    end
-
     module InstanceMethods
       def distributions(env)
         to_ret = []
@@ -56,7 +50,7 @@ module Katello
       end
 
       def last_sync_audit
-        Audited::Audit.where(:auditable_id => self.repositories, :auditable_type => Katello::Repository.name).order(:created_at).last
+        Audited::Audit.where(:auditable_id => self.repositories, :auditable_type => Katello::Repository.name, :action => "sync").order(:created_at).last
       end
 
       def last_sync
@@ -64,13 +58,14 @@ module Katello
       end
 
       def last_repo_sync_task
-        @last_sync_task ||= last_repo_sync_tasks.first
+        @last_sync_task ||= last_repo_sync_tasks&.first
       end
 
       def last_repo_sync_tasks
         ids = repos(self.library, nil, false).pluck(:id).join(',')
         label = ::Actions::Katello::Repository::Sync.name
         type = ::Katello::Repository.name
+        return nil if ids.empty?
         ForemanTasks::Task.search_for("label = #{label} and resource_type = #{type} and resource_id ^ (#{ids})")
           .order("started_at desc")
       end

data/app/models/katello/host/content_facet.rb

@@ -44,7 +44,7 @@ module Katello
 
       validates_with ::AssociationExistsValidator, attributes: [:content_source]
       validates_with Katello::Validators::GeneratedContentViewValidator
-      validates_associated :content_view_environment_content_facets
+      validates_associated :content_view_environment_content_facets, :message => _("invalid: The content source must sync the lifecycle environment assigned to the host. See the logs for more information.")
       validates :host, :presence => true, :allow_blank => false
 
       attr_accessor :cves_changed

data/app/models/katello/host/subscription_facet.rb

@@ -67,7 +67,7 @@ module Katello
           self.purpose_addon_ids = consumer_params['addOns'].map { |addon_name| ::Katello::PurposeAddon.find_or_create_by(name: addon_name).id }
         end
 
-        unless consumer_params['releaseVer'].blank?
+        unless consumer_params['releaseVer'].nil?
           release = consumer_params['releaseVer']
           release = release['releaseVer'] if release.is_a?(Hash)
           self.release_version = release

data/app/models/katello/host_collection.rb

@@ -29,7 +29,10 @@ module Katello
     scoped_search :relation => :hosts, :on => :name, :rename => :host, :complete_value => true
 
     def max_hosts_check
-      if !unlimited_hosts && (hosts.length > 0 && (hosts.length.to_i > max_hosts.to_i)) && max_hosts_changed?
+      # NOTE: max_hosts_check and max_hosts_no_exceeded use size() instead of count() because
+      # the host list exists as an array rather than a DB query when run as a validation.
+      host_count = hosts.size
+      if !unlimited_hosts && (host_count > 0 && (host_count.to_i > max_hosts.to_i)) && max_hosts_changed?
         errors.add :max_host, N_("may not be less than the number of hosts associated with the host collection.")
       end
     end
@@ -60,8 +63,14 @@ module Katello
       type ? query.of_type(type) : query
     end
 
-    def total_hosts
-      hosts.length
+    def cache_key
+      "#{self.class.name}/#{self.id}"
+    end
+
+    def total_hosts(cached: false)
+      Rails.cache.fetch("#{cache_key}/total_hosts", expires_in: 1.minute, force: !cached) do
+        hosts.count
+      end
     end
 
     # Retrieve the list of accessible host collections in the organization specified, returning

data/app/models/katello/ping.rb

@@ -2,7 +2,7 @@ module Katello
   class Ping
     OK_RETURN_CODE = 'ok'.freeze
     FAIL_RETURN_CODE = 'FAIL'.freeze
-    PACKAGES = %w(katello candlepin pulp foreman hammer).freeze
+    PACKAGES = %w(katello candlepin pulp foreman hammer dynflow).freeze
 
     class << self
       def services(capsule_id = nil)

data/app/models/katello/repository.rb

@@ -201,6 +201,12 @@ module Katello
       joins(:root).where("#{Katello::RootRepository.table_name}.product_id" => products)
     end
 
+    def self.repo_path_from_content_path(environment, content_path)
+      path = content_path.sub(%r|^/|, '')
+      path_prefix = [environment.organization.label, environment.label].join('/')
+      "#{path_prefix}/#{path}"
+    end
+
     def to_label
       name
     end
@@ -645,6 +651,33 @@ module Katello
                                 for_resource(self).order(:started_at).last
     end
 
+    def blocking_task
+      blocking_task_labels = [
+        ::Actions::Katello::Repository::Sync.name,
+        ::Actions::Katello::Repository::UploadFiles.name,
+        ::Actions::Katello::Repository::RemoveContent.name,
+        ::Actions::Katello::Repository::MetadataGenerate.name
+      ]
+      ForemanTasks::Task::DynflowTask.where(:label => blocking_task_labels)
+                                     .where.not(state: 'stopped')
+                                     .for_resource(self)
+                                     .order(:started_at)
+                                     .last
+    end
+
+    def check_ready_to_act!
+      blocking_tasks = content_views&.map { |cv| cv.blocking_task }&.compact
+
+      if blocking_tasks&.any?
+        errored_tasks = blocking_tasks
+                          .uniq
+                          .map { |task| "- #{Setting['foreman_url']}/foreman_tasks/tasks/#{task&.id}" }
+                          .join("\n")
+        fail _("This repository has pending tasks in associated content views. Please wait for the tasks: " + errored_tasks +
+               " to complete before proceeding.")
+      end
+    end
+
     # returns other instances of this repo with the same library
     # equivalent of repo
     def environmental_instances(view)

data/app/models/katello/root_repository.rb

@@ -387,10 +387,6 @@ module Katello
       Katello::RepositoryTypeManager.generic_repository_types(false).values.map(&:id).map(&:to_s).flatten.include? self.content_type
     end
 
-    def metadata_generate_needed?
-      (%w(unprotected checksum_type container_repsoitory_name) & previous_changes.keys).any?
-    end
-
     def using_mirrored_content?
       self.mirroring_policy != Katello::RootRepository::MIRRORING_POLICY_ADDITIVE
     end

data/app/services/katello/content_unit_indexer.rb

@@ -55,6 +55,15 @@ module Katello
       end
     end
 
+    def reimport_units
+      units_from_pulp.each do |units|
+        to_update = units.map do |unit|
+          @service_class.generate_model_row(unit)
+        end
+        @model_class.upsert_all(to_update, unique_by: :pulp_id)
+      end
+    end
+
     def import_associations(units)
       pulp_id_to_id = self.class.pulp_id_to_id_map(@content_type, units.map { |unit| unit[@service_class.unit_identifier] })
       @service_class.insert_child_associations(units, pulp_id_to_id) if @service_class.respond_to?(:insert_child_associations)

data/app/services/katello/pulp3/alternate_content_source.rb

@@ -88,18 +88,16 @@ module Katello
       end
 
       def get_remote(href = smart_proxy_acs.remote_href)
-        acs.base_url&.start_with?('uln') ? api.remotes_uln_api.read(href) : api.remotes_api.read(href)
+        api.get_remotes_api(href: href).read(href)
       end
 
-      def update_remote
-        api.remotes_api.partial_update(smart_proxy_acs.remote_href, remote_options)
+      def update_remote(href = smart_proxy_acs.remote_href)
+        api.get_remotes_api(href: href).partial_update(href, remote_options)
       end
 
-      # The old repo URL is needed to determine which remote API to use.
       def delete_remote(options = {})
         options[:href] ||= smart_proxy_acs.remote_href
-        options[:old_url] ||= remote_options[:url]
-        ignore_404_exception { options[:old_url]&.start_with?('uln') ? api.remotes_uln_api.delete(options[:href]) : api.remotes_api.delete(options[:href]) } if options[:href]
+        ignore_404_exception { api.get_remotes_api(href: options[:href]).delete(options[:href]) } if options[:href]
       end
 
       def create

data/app/services/katello/pulp3/api/core.rb

@@ -47,6 +47,11 @@ module Katello
           fail NotImplementedError
         end
 
+        # Method is called with either :url or :href parameters for the sake of yum content.
+        def get_remotes_api(*)
+          remotes_api
+        end
+
         def publications_api
           repository_type.publications_api_class.new(api_client) #Optional
         end
@@ -136,6 +141,10 @@ module Katello
           client
         end
 
+        def repair_api
+          PulpcoreClient::RepairApi.new(core_api_client)
+        end
+
         def uploads_api
           PulpcoreClient::UploadsApi.new(core_api_client)
         end
@@ -230,6 +239,10 @@ module Katello
           end
         end
 
+        def repair
+          repair_api.post(PulpcoreClient::Repair.new(verify_checksums: true))
+        end
+
         def self.fetch_from_list
           page_size = Setting[:bulk_load_size]
           page_opts = { "offset" => 0, limit: page_size }

data/app/services/katello/pulp3/api/yum.rb

@@ -32,6 +32,17 @@ module Katello
           PulpRpmClient::RemotesUlnApi.new(api_client)
         end
 
+        def get_remotes_api(href: nil, url: nil)
+          fail 'Provide exactly one of href or url for yum remote selection!' if url.blank? && href.blank?
+          fail 'The href must be a pulp_rpm remote href!' if href && !href.start_with?('/pulp/api/v3/remotes/rpm/')
+
+          if href&.start_with?('/pulp/api/v3/remotes/rpm/uln/') || url&.start_with?('uln')
+            remotes_uln_api
+          else
+            remotes_api
+          end
+        end
+
         def copy_api
           PulpRpmClient::RpmCopyApi.new(api_client)
         end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -26,7 +26,11 @@ module Katello
         {
           schema_version: unit['schema_version'],
           digest: unit['digest'],
-          pulp_id: unit[unit_identifier]
+          pulp_id: unit[unit_identifier],
+          annotations: unit['annotations'],
+          labels: unit['labels'],
+          is_bootable: unit['is_bootable'],
+          is_flatpak: unit['is_flatpak']
         }
       end
     end

data/app/services/katello/pulp3/repository/generic.rb

@@ -3,7 +3,7 @@ module Katello
     class Repository
       class Generic < ::Katello::Pulp3::Repository
         def copy_content_for_source(source_repository, _options = {})
-          copy_units_by_href(source_repository.files.pluck(:pulp_id))
+          copy_units_by_href(source_repository.generic_content_units&.pluck(:pulp_id))
         end
 
         def distribution_options(path)

data/app/services/katello/pulp3/repository.rb

@@ -91,16 +91,21 @@ module Katello
       end
 
       def remote_partial_update
-        if remote_options[:url]&.start_with?('uln')
-          api.remotes_uln_api.partial_update(repo.remote_href, remote_options)
-        else
-          api.remotes_api.partial_update(repo.remote_href, remote_options)
+        url_type = remote_options[:url]&.start_with?('uln') ? 'uln' : 'default'
+        remote_type = repo.remote_href.start_with?('/pulp/api/v3/remotes/rpm/uln/') ? 'uln' : 'default'
+        href = repo.remote_href
+
+        if url_type == remote_type
+          api.get_remotes_api(href: href).partial_update(href, remote_options)
+        else # We need to recreate a remote of the correct type!
+          create_remote
+          delete_remote(href: href)
         end
       end
 
       def delete_remote(options = {})
         options[:href] ||= repo.remote_href
-        ignore_404_exception { remote_options[:url]&.start_with?('uln') ? api.remotes_uln_api.delete(options[:href]) : api.remotes_api.delete(options[:href]) } if options[:href]
+        ignore_404_exception { api.get_remotes_api(href: options[:href]).delete(options[:href]) } if options[:href]
       end
 
       def self.instance_for_type(repo, smart_proxy)
@@ -139,7 +144,7 @@ module Katello
       end
 
       def get_remote(href = repo.remote_href)
-        repo.url&.start_with?('uln') ? api.remotes_uln_api.read(href) : api.remotes_api.read(href)
+        api.get_remotes_api(href: href).read(href)
       end
 
       def get_distribution(href = distribution_reference.href)
@@ -210,6 +215,10 @@ module Katello
         api.publications_api.create(publication_data)
       end
 
+      def delete_publication
+        ignore_404_exception { api.publications_api.delete(repo.publication_href) } if repo.publication_href
+      end
+
       def publication_options(repository_version)
         {
           repository_version: repository_version
@@ -329,6 +338,17 @@ module Katello
 
       def delete_version
         ignore_404_exception { api.repository_versions_api.delete(repo.version_href) } unless version_zero?
+      rescue api.api_exception_class => e
+        if e.message.include?("are currently being used to distribute content")
+          Rails.logger.warn "Exception when calling repository_versions_api->delete: #{e}"
+          publication_href = repo.publication_href
+          Rails.logger.warn "Trying to delete publication #{publication_href} for repository #{repo.id}}"
+          Rails.logger.error "Could not delete version: #{repo.version_href} because conflicting publication could not be looked up" unless publication_href
+          if publication_href
+            ignore_404_exception { api.publications_api.delete(publication_href) }
+            ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
+          end
+        end
       end
 
       def create_version(options = {})

data/app/services/katello/pulp3/repository_mirror.rb

@@ -24,11 +24,9 @@ module Katello
       def refresh_entities
         href = remote_href
         if href
-          if remote_options[:url]&.start_with?('uln')
-            [api.remotes_uln_api.partial_update(href, remote_options)]
-          else
-            [api.remotes_api.partial_update(href, remote_options)]
-          end
+          # Do not consider remotes_uln_api, since the Katello server is not a ULN server. Even if the sync
+          # to Katello used ULN, the sync from Katello server to smart proxy will use a normal RPM remote!
+          [api.remotes_api.partial_update(href, remote_options)]
         else
           create_remote
           []
@@ -111,13 +109,10 @@ module Katello
       end
 
       def create_remote
-        if remote_options[:url]&.start_with?('uln')
-          remote_file_data = @repo_service.api.class.remote_uln_class.new(remote_options)
-          api.remotes_uln_api.create(remote_file_data)
-        else
-          remote_file_data = @repo_service.api.remote_class.new(remote_options)
-          api.remotes_api.create(remote_file_data)
-        end
+        # Do not consider remotes_uln_api, since the Katello server is not a ULN server. Even if the sync
+        # to Katello used ULN, the sync from Katello server to smart proxy will use a normal RPM remote!
+        remote_file_data = @repo_service.api.remote_class.new(remote_options)
+        api.remotes_api.create(remote_file_data)
       end
 
       def compute_remote_options
@@ -240,6 +235,12 @@ module Katello
         distribution_data = api.distribution_class.new(distribution_options(path))
         repo_service.distributions_api.create(distribution_data)
       end
+
+      def repair
+        data = api.repair_class.new
+        fail "Could not lookup a version_href for repo #{repo_service.repo.id}" if version_href.nil?
+        api.repository_versions_api.repair(version_href, data)
+      end
     end
   end
 end

data/app/services/katello/pulp3/service_common.rb

@@ -9,11 +9,7 @@ module Katello
           remote_file_data = api.remote_class.new(remote_options)
         end
         reformat_api_exception do
-          if remote_options[:url]&.start_with?('uln')
-            response = api.remotes_uln_api.create(remote_file_data)
-          else
-            response = api.remotes_api.create(remote_file_data)
-          end
+          response = api.get_remotes_api(url: remote_options[:url]).create(remote_file_data)
         end
         response
       end
@@ -37,11 +33,7 @@ module Katello
         end
 
         reformat_api_exception do
-          if remote_options[:url]&.start_with?('uln')
-            response = api.remotes_uln_api.create(remote_file_data)
-          else
-            response = api.remotes_api.create(remote_file_data)
-          end
+          response = api.get_remotes_api(url: remote_options[:url]).create(remote_file_data)
           #delete is async, but if its not properly deleted, orphan cleanup will take care of it later
           delete_remote(href: response.pulp_href)
         end

data/app/services/katello/pulp3/smart_proxy_repository.rb

@@ -35,13 +35,11 @@ module Katello
 
       def delete_orphan_repository_versions
         tasks = []
-
         orphan_repository_versions.each do |api, version_hrefs|
           tasks << version_hrefs.collect do |href|
             api.repository_versions_api.delete(href)
           end
         end
-
         tasks.flatten
       end