katello 3.16.0.rc5.1 → 3.16.2

data/app/lib/actions/pulp3/repository/presenters/repair_presenter.rb

@@ -0,0 +1,85 @@
+module Actions
+  module Pulp3
+    module Repository
+      module Presenters
+        class RepairPresenter < Helpers::Presenter::Base
+          def humanized_output
+            if action.external_task
+              humanized_details
+            end
+          end
+
+          def progress
+            total_units == 0 ? 0.1 : finished_units.to_f / total_units
+          end
+
+          private
+
+          def repair_task
+            tasks = action.external_task.select do |task|
+              if task.key? 'name'
+                task['name'].include?("repair_version")
+              end
+            end
+            tasks.first
+          end
+
+          def cancelled?
+            repair_task && repair_task['state'] == 'cancelled'
+          end
+
+          def task_result
+            repair_task['result']
+          end
+
+          def task_result_details
+            task_result && task_result['details']
+          end
+
+          def humanized_details
+            ret = []
+            ret << _("Cancelled.") if cancelled?
+            ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
+            ret << "--------------------------------"
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports = progress_reports.sort_by { |pr| pr.try(:[], 'message') }
+            progress_reports.each do |pr|
+              done = pr.try(:[], 'done')
+              total = pr.try(:[], 'total') || pr.try(:[], 'done')
+              unless done.nil? || total.nil?
+                ret << _(pr.try(:[], 'message') + ": #{done}/#{total}")
+              end
+            end
+
+            ret.join("\n")
+          end
+
+          def total_units
+            total_unit = 0
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports.each do |pr|
+              total = pr.try(:[], 'total')
+              total = pr.try(:[], 'done') if total.nil?
+              unless total.nil?
+                total_unit += total.to_i
+              end
+            end
+            total_unit
+          end
+
+          def finished_units
+            finished_unit = 0
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports.each do |pr|
+              done = pr.try(:[], 'done')
+              unless done.nil?
+                finished_unit += done.to_i
+              end
+            end
+            finished_unit
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/repair.rb

@@ -0,0 +1,29 @@
+module Actions
+  module Pulp3
+    module Repository
+      class Repair < Pulp3::AbstractAsyncTask
+        include Helpers::Presenter
+        def plan(repository_id, smart_proxy)
+          plan_self(:repository_id => repository_id, :smart_proxy_id => smart_proxy.id)
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          output[:response] = repo.backend_service(smart_proxy).repair(repo.version_href)
+        end
+
+        def run_progress
+          presenter.progress
+        end
+
+        def run_progress_weight
+          10
+        end
+
+        def presenter
+          Presenters::RepairPresenter.new(self)
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -3,27 +3,39 @@ module Actions
     module Repository
       class SaveVersion < Pulp3::Abstract
         def plan(repository, options)
-          plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details])
+          fail "Cannot accept tasks and repository_details into Save Version." if options[:tasks].present? && options[:repository_details].present?
+          plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details], :force_fetch_version => options.fetch(:force_fetch_version, false))
         end
 
         def run
           repo = ::Katello::Repository.find(input[:repository_id])
 
-          if input[:tasks]
-            version_href = input[:tasks].last[:created_resources].first
-          end
-
-          if !version_href && input[:repository_details]
+          if input[:force_fetch_version]
+            version_href = fetch_version_href(repo)
+          elsif input[:repository_details].present?
             version_href = input[:repository_details][:latest_version_href]
+          elsif input[:tasks].present?
+            version_href = input[:tasks].last[:created_resources].first
+          else
+            version_href = fetch_version_href(repo)
           end
 
           if version_href
-            repo.update(:version_href => version_href)
-            output[:contents_changed] = true
+            if repo.version_href != version_href || input[:force_fetch_version]
+              output[:contents_changed] = true
+              repo.update(:version_href => version_href)
+            end
           else
             output[:contents_changed] = false
           end
         end
+
+        def fetch_version_href(repo)
+          # Fetch latest Pulp 3 repo version
+          repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+          repo_href = repo_backend_service.repository_reference.repository_href
+          repo_backend_service.api.repositories_api.read(repo_href).latest_version_href
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/save_versions.rb

@@ -7,23 +7,34 @@ module Actions
         end
 
         def run
+          return if input[:tasks].empty?
           version_hrefs = input[:tasks].last[:created_resources]
-          repositories = input[:repository_ids].collect do |repo_id|
-            ::Katello::Repository.find(repo_id)
-          end
+          repositories = find_repositories(input[:repository_ids])
 
-          output[:contents_changed] = false
-          output[:updated_repositories] = []
+          output.merge!(contents_changed: false, updated_repositories: [])
           repositories.each do |repo|
-            # Chop off the version number to compare base repo strings
-            unversioned_href = repo.version_href[0..-2].rpartition('/').first
-            new_version_href = version_hrefs.detect do |version_href|
-              unversioned_href == version_href[0..-2].rpartition('/').first
+            repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+            if repo.version_href
+              # Chop off the version number to compare base repo strings
+              unversioned_href = repo.version_href[0..-2].rpartition('/').first
+              # Could have multiple version_hrefs for the same repo depending on the copy task
+              new_version_hrefs = version_hrefs.collect do |version_href|
+                version_href if unversioned_href == version_href[0..-2].rpartition('/').first
+              end
+
+              new_version_hrefs.compact!
+              if new_version_hrefs.size > 1
+                # Find latest version_href by its version number
+                new_version_href = version_map(new_version_hrefs).max_by { |_href, version| version }.first
+              else
+                new_version_href = new_version_hrefs.first
+              end
+
+              # Successive incremental updates won't generate a new repo version, so fetch the latest Pulp 3 repo version
+              new_version_href ||= latest_version_href(repo_backend_service)
+            else
+              new_version_href = latest_version_href(repo_backend_service)
             end
-            # Successive incremental updates won't generate a new repo version, so fetch the latest Pulp 3 repo version
-            new_version_href ||= ::Katello::Pulp3::Api::Yum.new(SmartProxy.pulp_master!).
-              repositories_api.read(repo.backend_service(SmartProxy.pulp_master).
-              repository_reference.repository_href).latest_version_href
 
             unless new_version_href == repo.version_href
               repo.update(version_href: new_version_href)
@@ -33,6 +44,29 @@ module Actions
             end
           end
         end
+
+        def version_map(version_hrefs)
+          version_map = {}
+          version_hrefs.each do |href|
+            version_map[href] = href.split("/")[-1].to_i
+          end
+          version_map
+        end
+
+        def latest_version_href(repo_backend_service)
+          repo_backend_service.api.repositories_api.
+            read(repo_backend_service.repository_reference.repository_href).latest_version_href
+        end
+
+        def find_repositories(repository_ids)
+          repository_ids.collect do |repo_id|
+            if repo_id.is_a?(Hash)
+              ::Katello::Repository.find(repo_id.with_indifferent_access[:id])
+            else
+              ::Katello::Repository.find(repo_id)
+            end
+          end
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/sync.rb

@@ -10,7 +10,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repo_id])
-          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync
+          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync(input[:options])
         end
 
         def external_task=(tasks)

data/app/lib/actions/pulp3/repository/update_cv_repository_cert_guard.rb

@@ -2,13 +2,17 @@ module Actions
   module Pulp3
     module Repository
       class UpdateCVRepositoryCertGuard < Pulp3::Abstract
-        def plan(repository, _smart_proxy)
+        def plan(repository, smart_proxy)
           root = repository.root
           cv_repositories = root.repositories - [root.library_instance]
           cv_repositories.each do |repo|
-            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, SmartProxy.pulp_master)
+            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, smart_proxy)
           end
         end
+
+        def humanized_name
+          _("Updating repository authentication configuration")
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/upload_file.rb

@@ -25,7 +25,7 @@ module Actions
                 filechunk.flush
                 actual_chunk_size = File.size(filechunk)
                 response = uploads_api.update(upload_href, content_range(offset, offset + actual_chunk_size - 1, total_size), filechunk)
-                offset += actual_chunk_size - 1
+                offset += actual_chunk_size
               ensure
                 filechunk.close
                 filechunk.unlink

data/app/lib/katello/concerns/base_template_scope_extensions.rb

@@ -154,6 +154,10 @@ module Katello
       end
       # rubocop:enable Metrics/MethodLength
 
+      def sortable_version(version)
+        Util::Package.sortable_version(version)
+      end
+
       private
 
       def host_subscription_facet(host)

data/app/lib/katello/errors.rb

@@ -112,21 +112,7 @@ module Katello
       end
     end
 
-    class Pulp3Error < StandardError
-      def self.from_task(task)
-        if task[:state] == 'canceled'
-          self.new(_("Task canceled"))
-        elsif task[:state] == 'failed'
-          message = if task[:error][:description].blank?
-                      _("Pulp task error")
-                    else
-                      task[:error][:description]
-                    end
-          self.new(message)
-        end
-      end
-    end
-
+    class Pulp3Error < StandardError; end
     class Pulp3MigrationError < StandardError; end
 
     class PulpError < StandardError

data/app/lib/katello/resources/cdn.rb

@@ -18,7 +18,7 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product, :options
+        attr_reader :url, :product, :options, :proxy
 
         def substitutor(logger = nil)
           @logger = logger
@@ -26,6 +26,7 @@ module Katello
         end
 
         def initialize(url, options = {})
+          @proxy = ::HttpProxy.default_global_content_proxy
           @ssl_version = Setting[:cdn_ssl_version]
           if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
             fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
@@ -140,7 +141,7 @@ module Katello
         end
 
         def net_http_class
-          if (proxy = ::HttpProxy.default_global_content_proxy)
+          if self.proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
             Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -59,14 +59,17 @@ module Katello
 
         return resolved if to_resolve.empty?
 
-        futures = to_resolve.map do |path_with_substitution|
-          Concurrent::Promises.future do
-            path_with_substitution.resolve_substitutions(@resource)
+        to_resolve.in_groups_of(8) do |group|
+          futures = group.map do |path_with_substitution|
+            Concurrent::Promises.future do
+              path_with_substitution.resolve_substitutions(@resource)
+            end
           end
-        end
 
-        futures.each do |future|
-          resolved << future.value
+          futures.each do |future|
+            resolved << future.value
+            Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          end
         end
 
         find_substitutions(resolved.compact.flatten)

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -120,8 +120,7 @@ module Katello
           uri = pulp3_uri!
           config.host = uri.host
           config.scheme = uri.scheme
-          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
-          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+          pulp3_ssl_configuration(config)
           config.debugging = true
           config.logger = ::Foreman::Logging.logger('katello/pulp_rest')
           config.username = self.setting(PULP3_FEATURE, 'username')
@@ -129,6 +128,18 @@ module Katello
         end
       end
 
+      def pulp3_ssl_configuration(config)
+        if Faraday.default_adapter == :excon
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert_filename
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key_filename
+        elsif Faraday.default_adapter == :net_http
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+        else
+          fail "Unexpected faraday default_adapter #{Faraday.default_adapter}!  Cannot continue, this is likely a bug."
+        end
+      end
+
       def backend_service_type(repository)
         if pulp3_support?(repository)
           Actions::Pulp3::Abstract::BACKEND_SERVICE_TYPE
@@ -142,7 +153,7 @@ module Katello
       end
 
       def pulp3_support?(repository)
-        pulp3_repository_type_support?(repository.content_type)
+        repository ? pulp3_repository_type_support?(repository.try(:content_type)) : false
       end
 
       def pulp2_preferred_for_type?(repository_type)

data/app/models/katello/content_view.rb

@@ -289,9 +289,21 @@ module Katello
         where("#{Katello::ContentViewVersion.table_name}.content_view_id" => self.id)
     end
 
-    def repositories_to_publish
+    def repositories_to_publish(override_components = nil)
       if composite?
-        ids = components.flat_map { |version| version.repositories.archived }.map(&:id)
+        components_to_publish = []
+        components.each do |component|
+          override_component = override_components&.detect do |override_cvv|
+            override_cvv.content_view == component.content_view
+          end
+
+          if override_component
+            components_to_publish << override_component
+          else
+            components_to_publish << component
+          end
+        end
+        ids = components_to_publish.flat_map { |version| version.repositories.archived }.map(&:id)
         Repository.where(:id => ids)
       else
         repositories
@@ -302,11 +314,11 @@ module Katello
       composite? ? repositories_to_publish.pluck(&:id) : repository_ids
     end
 
-    def repositories_to_publish_by_library_instance
+    def repositories_to_publish_by_library_instance(override_components = nil)
       # retrieve the list of repositories in a hash, where the key
       # is the library instance id, and the value is an array
       # of the repositories for that instance.
-      repositories_to_publish.inject({}) do |result, repo|
+      repositories_to_publish(override_components).inject({}) do |result, repo|
         result[repo.library_instance] ||= []
         result[repo.library_instance] << repo
         result
@@ -326,8 +338,8 @@ module Katello
       component_composites.where(latest: true).joins(:composite_content_view).where(self.class.table_name => {auto_publish: true})
     end
 
-    def publish_repositories
-      repositories = composite? ? repositories_to_publish_by_library_instance.values : repositories_to_publish
+    def publish_repositories(override_components = nil)
+      repositories = composite? ? repositories_to_publish_by_library_instance(override_components).values : repositories_to_publish
       repositories.each do |repos|
         if repos.is_a? Array
           yield repos