katello 3.16.0 → 3.16.1

data/app/lib/actions/pulp3/capsule_content/refresh_content_guard.rb

@@ -0,0 +1,17 @@
+module Actions
+  module Pulp3
+    module CapsuleContent
+      class RefreshContentGuard < Pulp3::AbstractAsyncTask
+        def plan(smart_proxy, options = {})
+          content_guard_api = ::Katello::Pulp3::Api::ContentGuard.new(smart_proxy)
+          content_guard_href = content_guard_api.list&.results&.first&.pulp_href
+          if content_guard_href && options.try(:[], :update)
+            content_guard_api.partial_update content_guard_href
+          else
+            content_guard_api.create
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/capsule_content/sync.rb

@@ -11,7 +11,9 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
-          output[:pulp_tasks] = repo.backend_service(smart_proxy).with_mirror_adapter.sync
+          sync_options = {}
+          sync_options[:optimize] = false if sync_options[:skip_metadata_check]
+          output[:pulp_tasks] = repo.backend_service(smart_proxy).with_mirror_adapter.sync(sync_options)
         end
 
         def rescue_strategy_for_self

data/app/lib/actions/pulp3/orchestration/repository/copy_all_units.rb

@@ -16,8 +16,7 @@ module Actions
                   copy_action = plan_action(Actions::Pulp3::Repository::CopyContent, source_repositories.first, smart_proxy, target_repo,
                                             filter_ids: filter_ids, solve_dependencies: solve_dependencies,
                                             rpm_filenames: rpm_filenames)
-                  plan_action(Actions::Pulp3::Repository::SaveVersion, target_repo,
-                              repository_details: { latest_version_href: copy_action.output[:latest_version_href] }, tasks: copy_action.output[:pulp_tasks])
+                  plan_action(Actions::Pulp3::Repository::SaveVersion, target_repo, tasks: copy_action.output[:pulp_tasks])
                 else
                   #if we are not filtering, copy the version to the cv repository, and the units for each additional repo
                   action = plan_action(Actions::Pulp3::Repository::CopyVersion, source_repositories.first, smart_proxy, target_repo)

data/app/lib/actions/pulp3/orchestration/repository/sync.rb

@@ -7,7 +7,9 @@ module Actions
           def plan(repository, smart_proxy, options)
             sequence do
               action_output = plan_action(Actions::Pulp3::Repository::Sync, repository, smart_proxy, options).output
-              version_output = plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
+
+              force_fetch_version = true if options[:optimize] == false
+              version_output = plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks], :force_fetch_version => force_fetch_version).output
               plan_action(Pulp3::Orchestration::Repository::GenerateMetadata, repository, smart_proxy, :contents_changed => version_output[:contents_changed])
               plan_self(:subaction_output => version_output)
             end

data/app/lib/actions/pulp3/orchestration/repository/trigger_update_repo_cert_guard.rb

@@ -0,0 +1,22 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module Repository
+        class TriggerUpdateRepoCertGuard < Pulp3::Abstract
+          def plan(repository, smart_proxy)
+            plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id)
+          end
+
+          def run
+            repository = ::Katello::Repository.find(input[:repository_id])
+            ForemanTasks.async_task(::Actions::Pulp3::Repository::UpdateCVRepositoryCertGuard, repository, smart_proxy)
+          end
+
+          def humanized_name
+            _("Updating repository authentication configuration")
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/copy_content.rb

@@ -11,7 +11,6 @@ module Actions
         def invoke_external_task
           source = ::Katello::Repository.find(input[:source_repository_id])
           target = ::Katello::Repository.find(input[:target_repository_id] || input[:target_repository])
-          output[:latest_version_href] = target.backend_service(smart_proxy).read.latest_version_href
           output[:pulp_tasks] = target.backend_service(smart_proxy).copy_content_for_source(source, input)
         end
       end

data/app/lib/actions/pulp3/repository/multi_copy_content.rb

@@ -17,7 +17,7 @@ module Actions
           repo_id_map = {}
 
           input[:repo_id_map].each do |source_repo_ids, dest_repo_map|
-            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map
+            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map.deep_dup
           end
 
           output[:pulp_tasks] = ::Katello::Repository.find(repo_id_map.values.first[:dest_repo]).backend_service(smart_proxy).copy_content_from_mapping(repo_id_map, input)

data/app/lib/actions/pulp3/repository/presenters/repair_presenter.rb

@@ -0,0 +1,85 @@
+module Actions
+  module Pulp3
+    module Repository
+      module Presenters
+        class RepairPresenter < Helpers::Presenter::Base
+          def humanized_output
+            if action.external_task
+              humanized_details
+            end
+          end
+
+          def progress
+            total_units == 0 ? 0.1 : finished_units.to_f / total_units
+          end
+
+          private
+
+          def repair_task
+            tasks = action.external_task.select do |task|
+              if task.key? 'name'
+                task['name'].include?("repair_version")
+              end
+            end
+            tasks.first
+          end
+
+          def cancelled?
+            repair_task && repair_task['state'] == 'cancelled'
+          end
+
+          def task_result
+            repair_task['result']
+          end
+
+          def task_result_details
+            task_result && task_result['details']
+          end
+
+          def humanized_details
+            ret = []
+            ret << _("Cancelled.") if cancelled?
+            ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
+            ret << "--------------------------------"
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports = progress_reports.sort_by { |pr| pr.try(:[], 'message') }
+            progress_reports.each do |pr|
+              done = pr.try(:[], 'done')
+              total = pr.try(:[], 'total') || pr.try(:[], 'done')
+              unless done.nil? || total.nil?
+                ret << _(pr.try(:[], 'message') + ": #{done}/#{total}")
+              end
+            end
+
+            ret.join("\n")
+          end
+
+          def total_units
+            total_unit = 0
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports.each do |pr|
+              total = pr.try(:[], 'total')
+              total = pr.try(:[], 'done') if total.nil?
+              unless total.nil?
+                total_unit += total.to_i
+              end
+            end
+            total_unit
+          end
+
+          def finished_units
+            finished_unit = 0
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports.each do |pr|
+              done = pr.try(:[], 'done')
+              unless done.nil?
+                finished_unit += done.to_i
+              end
+            end
+            finished_unit
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/repair.rb

@@ -0,0 +1,29 @@
+module Actions
+  module Pulp3
+    module Repository
+      class Repair < Pulp3::AbstractAsyncTask
+        include Helpers::Presenter
+        def plan(repository_id, smart_proxy)
+          plan_self(:repository_id => repository_id, :smart_proxy_id => smart_proxy.id)
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          output[:response] = repo.backend_service(smart_proxy).repair(repo.version_href)
+        end
+
+        def run_progress
+          presenter.progress
+        end
+
+        def run_progress_weight
+          10
+        end
+
+        def presenter
+          Presenters::RepairPresenter.new(self)
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -3,35 +3,39 @@ module Actions
     module Repository
       class SaveVersion < Pulp3::Abstract
         def plan(repository, options)
-          plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details])
+          fail "Cannot accept tasks and repository_details into Save Version." if options[:tasks].present? && options[:repository_details].present?
+          plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details], :force_fetch_version => options.fetch(:force_fetch_version, false))
         end
 
         def run
           repo = ::Katello::Repository.find(input[:repository_id])
 
-          if input[:tasks].present?
+          if input[:force_fetch_version]
+            version_href = fetch_version_href(repo)
+          elsif input[:repository_details].present?
+            version_href = input[:repository_details][:latest_version_href]
+          elsif input[:tasks].present?
             version_href = input[:tasks].last[:created_resources].first
-          end
-
-          unless version_href
-            if input[:repository_details]
-              version_href = input[:repository_details][:latest_version_href]
-            elsif repo.version_href.nil?
-              # Fetch latest Pulp 3 repo version
-              repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
-              version_href ||= repo_backend_service.api.
-                repositories_api.read(repo_backend_service.
-                repository_reference.repository_href).latest_version_href
-            end
+          else
+            version_href = fetch_version_href(repo)
           end
 
           if version_href
-            repo.update(:version_href => version_href)
-            output[:contents_changed] = true
+            if repo.version_href != version_href || input[:force_fetch_version]
+              output[:contents_changed] = true
+              repo.update(:version_href => version_href)
+            end
           else
             output[:contents_changed] = false
           end
         end
+
+        def fetch_version_href(repo)
+          # Fetch latest Pulp 3 repo version
+          repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+          repo_href = repo_backend_service.repository_reference.repository_href
+          repo_backend_service.api.repositories_api.read(repo_href).latest_version_href
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/sync.rb

@@ -10,7 +10,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repo_id])
-          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync
+          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync(input[:options])
         end
 
         def external_task=(tasks)

data/app/lib/actions/pulp3/repository/update_cv_repository_cert_guard.rb

@@ -2,13 +2,17 @@ module Actions
   module Pulp3
     module Repository
       class UpdateCVRepositoryCertGuard < Pulp3::Abstract
-        def plan(repository, _smart_proxy)
+        def plan(repository, smart_proxy)
           root = repository.root
           cv_repositories = root.repositories - [root.library_instance]
           cv_repositories.each do |repo|
-            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, SmartProxy.pulp_master)
+            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, smart_proxy)
           end
         end
+
+        def humanized_name
+          _("Updating repository authentication configuration")
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/upload_file.rb

@@ -25,7 +25,7 @@ module Actions
                 filechunk.flush
                 actual_chunk_size = File.size(filechunk)
                 response = uploads_api.update(upload_href, content_range(offset, offset + actual_chunk_size - 1, total_size), filechunk)
-                offset += actual_chunk_size - 1
+                offset += actual_chunk_size
               ensure
                 filechunk.close
                 filechunk.unlink

data/app/lib/katello/concerns/base_template_scope_extensions.rb

@@ -154,6 +154,10 @@ module Katello
       end
       # rubocop:enable Metrics/MethodLength
 
+      def sortable_version(version)
+        Util::Package.sortable_version(version)
+      end
+
       private
 
       def host_subscription_facet(host)

data/app/lib/katello/resources/cdn.rb

@@ -18,7 +18,7 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product, :options
+        attr_reader :url, :product, :options, :proxy
 
         def substitutor(logger = nil)
           @logger = logger
@@ -26,6 +26,7 @@ module Katello
         end
 
         def initialize(url, options = {})
+          @proxy = ::HttpProxy.default_global_content_proxy
           @ssl_version = Setting[:cdn_ssl_version]
           if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
             fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
@@ -140,7 +141,7 @@ module Katello
         end
 
         def net_http_class
-          if (proxy = ::HttpProxy.default_global_content_proxy)
+          if self.proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
             Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -59,14 +59,17 @@ module Katello
 
         return resolved if to_resolve.empty?
 
-        futures = to_resolve.map do |path_with_substitution|
-          Concurrent::Promises.future do
-            path_with_substitution.resolve_substitutions(@resource)
+        to_resolve.in_groups_of(8) do |group|
+          futures = group.map do |path_with_substitution|
+            Concurrent::Promises.future do
+              path_with_substitution.resolve_substitutions(@resource)
+            end
           end
-        end
 
-        futures.each do |future|
-          resolved << future.value
+          futures.each do |future|
+            resolved << future.value
+            Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          end
         end
 
         find_substitutions(resolved.compact.flatten)

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -120,8 +120,7 @@ module Katello
           uri = pulp3_uri!
           config.host = uri.host
           config.scheme = uri.scheme
-          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
-          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+          pulp3_ssl_configuration(config)
           config.debugging = true
           config.logger = ::Foreman::Logging.logger('katello/pulp_rest')
           config.username = self.setting(PULP3_FEATURE, 'username')
@@ -129,6 +128,18 @@ module Katello
         end
       end
 
+      def pulp3_ssl_configuration(config)
+        if Faraday.default_adapter == :excon
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert_filename
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key_filename
+        elsif Faraday.default_adapter == :net_http
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+        else
+          fail "Unexpected faraday default_adapter #{Faraday.default_adapter}!  Cannot continue, this is likely a bug."
+        end
+      end
+
       def backend_service_type(repository)
         if pulp3_support?(repository)
           Actions::Pulp3::Abstract::BACKEND_SERVICE_TYPE
@@ -142,7 +153,7 @@ module Katello
       end
 
       def pulp3_support?(repository)
-        pulp3_repository_type_support?(repository.content_type)
+        repository ? pulp3_repository_type_support?(repository.try(:content_type)) : false
       end
 
       def pulp2_preferred_for_type?(repository_type)

data/app/models/katello/content_view_package_filter.rb

@@ -37,7 +37,7 @@ module Katello
       self.package_rules.each do |rule|
         package_filenames.concat(query_rpms(repo, rule))
       end
-
+      package_filenames.concat(repo.packages_without_errata.map(&:filename)) if self.original_packages
       rpms = Rpm.in_repositories(repo)
       rpms.where(filename: package_filenames).where(:modular => false).pluck(:pulp_id).flatten.uniq
     end

data/app/models/katello/host/content_facet.rb

@@ -126,6 +126,7 @@ module Katello
         ::Katello::Applicability::ApplicableContentHelper.new(self, ::Katello::Erratum, bound_repos).calculate_and_import
         ::Katello::Applicability::ApplicableContentHelper.new(self, ::Katello::ModuleStream, bound_repos).calculate_and_import
         update_applicability_counts
+        self.update_errata_status
       end
 
       def import_applicability(partial = false)

data/app/models/katello/ping.rb

@@ -31,9 +31,7 @@ module Katello
 
         # set overall status result code
         result = {:services => result}
-        result[:services].each_value do |v|
-          result[:status] = v[:status] == OK_RETURN_CODE ? OK_RETURN_CODE : FAIL_RETURN_CODE
-        end
+        result[:status] = result[:services].each_value.any? { |v| v[:status] == FAIL_RETURN_CODE } ? FAIL_RETURN_CODE : OK_RETURN_CODE
         result
       end
 

data/app/models/katello/repository.rb

@@ -133,6 +133,7 @@ module Katello
     scope :in_published_environments, -> { in_content_views(Katello::ContentView.non_default).where.not(:environment_id => nil) }
     scope :order_by_root, ->(attr) { joins(:root).order("#{Katello::RootRepository.table_name}.#{attr}") }
     scope :with_content, ->(content) { joins(Katello::RepositoryTypeManager.find_content_type(content).model_class.repository_association_class.name.demodulize.underscore.pluralize.to_sym).distinct }
+    scope :by_rpm_count, -> { left_joins(:repository_rpms).group(:id).order("count(katello_repository_rpms.id) ASC") } # smallest count first
 
     scoped_search :on => :name, :relation => :root, :complete_value => true
     scoped_search :rename => :product, :on => :name, :relation => :product, :complete_value => true
@@ -592,6 +593,10 @@ module Katello
       environment
     end
 
+    def self.smart_proxy_syncable
+      where.not(:environment_id => nil)
+    end
+
     def exist_for_environment?(environment, content_view, attribute = nil)
       if environment.present? && content_view.in_environment?(environment)
         repos = content_view.version(environment).repos(environment)