katello 3.16.0.rc5 → 3.16.1.2

data/app/lib/actions/pulp3/repository/multi_copy_units.rb

@@ -3,7 +3,7 @@ module Actions
     module Repository
       class MultiCopyUnits < Pulp3::AbstractAsyncTask
         # repo_map example: {
-        #   <source_repo_id>: {
+        #   [<source_repo_ids>]: {
         #     dest_repo: <dest_repo_id>,
         #     base_version: <base_version>
         #   }
@@ -22,7 +22,11 @@ module Actions
 
         def invoke_external_task
           unit_hrefs = []
-          repo_map = input[:repo_map]
+          repo_map = {}
+
+          input[:repo_map].each do |source_repo_ids, dest_repo_map|
+            repo_map[JSON.parse(source_repo_ids)] = dest_repo_map
+          end
 
           if input[:unit_map][:errata].any?
             unit_hrefs << ::Katello::RepositoryErratum.
@@ -34,13 +38,16 @@ module Actions
           if input[:unit_map][:rpms].any?
             unit_hrefs << ::Katello::Rpm.where(:id => input[:unit_map][:rpms]).map(&:pulp_id)
           end
+          unit_hrefs.flatten!
+
+          repo_map.each do |_source_repos, dest_repo_map|
+            dest_repo_map[:content_unit_hrefs] = unit_hrefs
+          end
 
-          # TODO: Fix this workaround by refactoring copy_units after general content view dep solving is refactored
-          source_repo = ::Katello::Repository.find(repo_map.keys.first)
           target_repo = ::Katello::Repository.find(repo_map.values.first[:dest_repo])
-          dest_base_version = repo_map.values.first[:base_version]
-          repo_map.delete(repo_map.keys.first)
-          output[:pulp_tasks] = target_repo.backend_service(SmartProxy.pulp_master).copy_units(source_repo, unit_hrefs.flatten, input[:dependency_solving], dest_base_version, repo_map)
+          unless unit_hrefs.flatten.empty?
+            output[:pulp_tasks] = target_repo.backend_service(SmartProxy.pulp_master).multi_copy_units(repo_map, input[:dependency_solving])
+          end
         end
       end
     end

data/app/lib/actions/pulp3/repository/presenters/content_unit_presenter.rb

@@ -12,7 +12,7 @@ module Actions
           def humanized_details
             ret = []
             ret << _("Cancelled.") if cancelled?
-            ret << _("Total tasks: ") + ": #{finished_units}/#{total_units}"
+            ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
             ret << "--------------------------------"
             progress_reports = sync_task.try(:[], 'progress_reports') || []
             progress_reports = progress_reports.sort_by { |pr| pr.try(:[], 'message') }

data/app/lib/actions/pulp3/repository/presenters/repair_presenter.rb

@@ -0,0 +1,85 @@
+module Actions
+  module Pulp3
+    module Repository
+      module Presenters
+        class RepairPresenter < Helpers::Presenter::Base
+          def humanized_output
+            if action.external_task
+              humanized_details
+            end
+          end
+
+          def progress
+            total_units == 0 ? 0.1 : finished_units.to_f / total_units
+          end
+
+          private
+
+          def repair_task
+            tasks = action.external_task.select do |task|
+              if task.key? 'name'
+                task['name'].include?("repair_version")
+              end
+            end
+            tasks.first
+          end
+
+          def cancelled?
+            repair_task && repair_task['state'] == 'cancelled'
+          end
+
+          def task_result
+            repair_task['result']
+          end
+
+          def task_result_details
+            task_result && task_result['details']
+          end
+
+          def humanized_details
+            ret = []
+            ret << _("Cancelled.") if cancelled?
+            ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
+            ret << "--------------------------------"
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports = progress_reports.sort_by { |pr| pr.try(:[], 'message') }
+            progress_reports.each do |pr|
+              done = pr.try(:[], 'done')
+              total = pr.try(:[], 'total') || pr.try(:[], 'done')
+              unless done.nil? || total.nil?
+                ret << _(pr.try(:[], 'message') + ": #{done}/#{total}")
+              end
+            end
+
+            ret.join("\n")
+          end
+
+          def total_units
+            total_unit = 0
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports.each do |pr|
+              total = pr.try(:[], 'total')
+              total = pr.try(:[], 'done') if total.nil?
+              unless total.nil?
+                total_unit += total.to_i
+              end
+            end
+            total_unit
+          end
+
+          def finished_units
+            finished_unit = 0
+            progress_reports = repair_task.try(:[], 'progress_reports') || []
+            progress_reports.each do |pr|
+              done = pr.try(:[], 'done')
+              unless done.nil?
+                finished_unit += done.to_i
+              end
+            end
+            finished_unit
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/repair.rb

@@ -0,0 +1,29 @@
+module Actions
+  module Pulp3
+    module Repository
+      class Repair < Pulp3::AbstractAsyncTask
+        include Helpers::Presenter
+        def plan(repository_id, smart_proxy)
+          plan_self(:repository_id => repository_id, :smart_proxy_id => smart_proxy.id)
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          output[:response] = repo.backend_service(smart_proxy).repair(repo.version_href)
+        end
+
+        def run_progress
+          presenter.progress
+        end
+
+        def run_progress_weight
+          10
+        end
+
+        def presenter
+          Presenters::RepairPresenter.new(self)
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -3,27 +3,39 @@ module Actions
     module Repository
       class SaveVersion < Pulp3::Abstract
         def plan(repository, options)
-          plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details])
+          fail "Cannot accept tasks and repository_details into Save Version." if options[:tasks].present? && options[:repository_details].present?
+          plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details], :force_fetch_version => options.fetch(:force_fetch_version, false))
         end
 
         def run
           repo = ::Katello::Repository.find(input[:repository_id])
 
-          if input[:tasks]
-            version_href = input[:tasks].last[:created_resources].first
-          end
-
-          if !version_href && input[:repository_details]
+          if input[:force_fetch_version]
+            version_href = fetch_version_href(repo)
+          elsif input[:repository_details].present?
             version_href = input[:repository_details][:latest_version_href]
+          elsif input[:tasks].present?
+            version_href = input[:tasks].last[:created_resources].first
+          else
+            version_href = fetch_version_href(repo)
           end
 
           if version_href
-            repo.update(:version_href => version_href)
-            output[:contents_changed] = true
+            if repo.version_href != version_href || input[:force_fetch_version]
+              output[:contents_changed] = true
+              repo.update(:version_href => version_href)
+            end
           else
             output[:contents_changed] = false
           end
         end
+
+        def fetch_version_href(repo)
+          # Fetch latest Pulp 3 repo version
+          repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+          repo_href = repo_backend_service.repository_reference.repository_href
+          repo_backend_service.api.repositories_api.read(repo_href).latest_version_href
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/save_versions.rb

@@ -7,23 +7,34 @@ module Actions
         end
 
         def run
+          return if input[:tasks].empty?
           version_hrefs = input[:tasks].last[:created_resources]
-          repositories = input[:repository_ids].collect do |repo_id|
-            ::Katello::Repository.find(repo_id)
-          end
+          repositories = find_repositories(input[:repository_ids])
 
-          output[:contents_changed] = false
-          output[:updated_repositories] = []
+          output.merge!(contents_changed: false, updated_repositories: [])
           repositories.each do |repo|
-            # Chop off the version number to compare base repo strings
-            unversioned_href = repo.version_href[0..-2].rpartition('/').first
-            new_version_href = version_hrefs.detect do |version_href|
-              unversioned_href == version_href[0..-2].rpartition('/').first
+            repo_backend_service = repo.backend_service(SmartProxy.pulp_master)
+            if repo.version_href
+              # Chop off the version number to compare base repo strings
+              unversioned_href = repo.version_href[0..-2].rpartition('/').first
+              # Could have multiple version_hrefs for the same repo depending on the copy task
+              new_version_hrefs = version_hrefs.collect do |version_href|
+                version_href if unversioned_href == version_href[0..-2].rpartition('/').first
+              end
+
+              new_version_hrefs.compact!
+              if new_version_hrefs.size > 1
+                # Find latest version_href by its version number
+                new_version_href = version_map(new_version_hrefs).max_by { |_href, version| version }.first
+              else
+                new_version_href = new_version_hrefs.first
+              end
+
+              # Successive incremental updates won't generate a new repo version, so fetch the latest Pulp 3 repo version
+              new_version_href ||= latest_version_href(repo_backend_service)
+            else
+              new_version_href = latest_version_href(repo_backend_service)
             end
-            # Successive incremental updates won't generate a new repo version, so fetch the latest Pulp 3 repo version
-            new_version_href ||= ::Katello::Pulp3::Api::Yum.new(SmartProxy.pulp_master!).
-              repositories_api.read(repo.backend_service(SmartProxy.pulp_master).
-              repository_reference.repository_href).latest_version_href
 
             unless new_version_href == repo.version_href
               repo.update(version_href: new_version_href)
@@ -33,6 +44,29 @@ module Actions
             end
           end
         end
+
+        def version_map(version_hrefs)
+          version_map = {}
+          version_hrefs.each do |href|
+            version_map[href] = href.split("/")[-1].to_i
+          end
+          version_map
+        end
+
+        def latest_version_href(repo_backend_service)
+          repo_backend_service.api.repositories_api.
+            read(repo_backend_service.repository_reference.repository_href).latest_version_href
+        end
+
+        def find_repositories(repository_ids)
+          repository_ids.collect do |repo_id|
+            if repo_id.is_a?(Hash)
+              ::Katello::Repository.find(repo_id.with_indifferent_access[:id])
+            else
+              ::Katello::Repository.find(repo_id)
+            end
+          end
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/sync.rb

@@ -10,7 +10,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repo_id])
-          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync
+          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync(input[:options])
         end
 
         def external_task=(tasks)

data/app/lib/actions/pulp3/repository/update_cv_repository_cert_guard.rb

@@ -2,13 +2,17 @@ module Actions
   module Pulp3
     module Repository
       class UpdateCVRepositoryCertGuard < Pulp3::Abstract
-        def plan(repository, _smart_proxy)
+        def plan(repository, smart_proxy)
           root = repository.root
           cv_repositories = root.repositories - [root.library_instance]
           cv_repositories.each do |repo|
-            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, SmartProxy.pulp_master)
+            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, smart_proxy)
           end
         end
+
+        def humanized_name
+          _("Updating repository authentication configuration")
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/upload_file.rb

@@ -25,7 +25,7 @@ module Actions
                 filechunk.flush
                 actual_chunk_size = File.size(filechunk)
                 response = uploads_api.update(upload_href, content_range(offset, offset + actual_chunk_size - 1, total_size), filechunk)
-                offset += actual_chunk_size - 1
+                offset += actual_chunk_size
               ensure
                 filechunk.close
                 filechunk.unlink

data/app/lib/katello/concerns/base_template_scope_extensions.rb

@@ -154,6 +154,10 @@ module Katello
       end
       # rubocop:enable Metrics/MethodLength
 
+      def sortable_version(version)
+        Util::Package.sortable_version(version)
+      end
+
       private
 
       def host_subscription_facet(host)

data/app/lib/katello/errors.rb

@@ -112,21 +112,7 @@ module Katello
       end
     end
 
-    class Pulp3Error < StandardError
-      def self.from_task(task)
-        if task[:state] == 'canceled'
-          self.new(_("Task canceled"))
-        elsif task[:state] == 'failed'
-          message = if task[:error][:description].blank?
-                      _("Pulp task error")
-                    else
-                      task[:error][:description]
-                    end
-          self.new(message)
-        end
-      end
-    end
-
+    class Pulp3Error < StandardError; end
     class Pulp3MigrationError < StandardError; end
 
     class PulpError < StandardError

data/app/lib/katello/resources/cdn.rb

@@ -18,7 +18,7 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product, :options
+        attr_reader :url, :product, :options, :proxy
 
         def substitutor(logger = nil)
           @logger = logger
@@ -26,6 +26,7 @@ module Katello
         end
 
         def initialize(url, options = {})
+          @proxy = ::HttpProxy.default_global_content_proxy
           @ssl_version = Setting[:cdn_ssl_version]
           if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
             fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
@@ -140,7 +141,7 @@ module Katello
         end
 
         def net_http_class
-          if (proxy = ::HttpProxy.default_global_content_proxy)
+          if self.proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
             Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -59,14 +59,17 @@ module Katello
 
         return resolved if to_resolve.empty?
 
-        futures = to_resolve.map do |path_with_substitution|
-          Concurrent::Promises.future do
-            path_with_substitution.resolve_substitutions(@resource)
+        to_resolve.in_groups_of(8) do |group|
+          futures = group.map do |path_with_substitution|
+            Concurrent::Promises.future do
+              path_with_substitution.resolve_substitutions(@resource)
+            end
           end
-        end
 
-        futures.each do |future|
-          resolved << future.value
+          futures.each do |future|
+            resolved << future.value
+            Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          end
         end
 
         find_substitutions(resolved.compact.flatten)

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -120,8 +120,7 @@ module Katello
           uri = pulp3_uri!
           config.host = uri.host
           config.scheme = uri.scheme
-          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
-          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+          pulp3_ssl_configuration(config)
           config.debugging = true
           config.logger = ::Foreman::Logging.logger('katello/pulp_rest')
           config.username = self.setting(PULP3_FEATURE, 'username')
@@ -129,6 +128,18 @@ module Katello
         end
       end
 
+      def pulp3_ssl_configuration(config)
+        if Faraday.default_adapter == :excon
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert_filename
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key_filename
+        elsif Faraday.default_adapter == :net_http
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+        else
+          fail "Unexpected faraday default_adapter #{Faraday.default_adapter}!  Cannot continue, this is likely a bug."
+        end
+      end
+
       def backend_service_type(repository)
         if pulp3_support?(repository)
           Actions::Pulp3::Abstract::BACKEND_SERVICE_TYPE
@@ -142,7 +153,7 @@ module Katello
       end
 
       def pulp3_support?(repository)
-        pulp3_repository_type_support?(repository.content_type)
+        repository ? pulp3_repository_type_support?(repository.try(:content_type)) : false
       end
 
       def pulp2_preferred_for_type?(repository_type)