katello 3.17.0.rc2.1 → 3.17.3

data/app/services/katello/host_trace_manager.rb

@@ -0,0 +1,38 @@
+module Katello
+  class HostTraceManager
+    def self.resolve_traces(traces)
+      traces.each do |trace|
+        if trace.reboot_required?
+          trace.helper = 'reboot'
+        end
+      end
+
+      traces_by_host_id = traces.group_by(&:host_id)
+      traces_by_helper = traces.group_by(&:helper)
+
+      composers = []
+
+      if traces_by_host_id.size < traces_by_helper.size
+        traces_by_host_id.each do |host_id, trace|
+          needed_traces = trace.map(&:helper).join(',')
+          joined_helpers = { :helper => needed_traces }
+          composers << ::JobInvocationComposer.for_feature(:katello_service_restart, [host_id], joined_helpers)
+        end
+      else
+        traces_by_helper.each do |helper, trace|
+          helpers = { :helper => helper }
+          composers << ::JobInvocationComposer.for_feature(:katello_service_restart, trace.map(&:host_id), helpers)
+        end
+      end
+
+      job_invocations = []
+
+      composers.each do |composer|
+        composer.trigger
+        job_invocations << composer.job_invocation
+      end
+
+      job_invocations
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -2,20 +2,15 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class Export
+        include ImportExportCommon
+        METADATA_FILE = "metadata.json".freeze
+
         def initialize(smart_proxy:, content_view_version: nil, destination_server: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
         end
 
-        def exporter_name
-          @content_view_version.name.gsub(/\s/, '_')
-        end
-
-        def generate_exporter_id
-          "#{@content_view_version.organization.label}_#{exporter_name}"
-        end
-
         def generate_exporter_path
           export_path = "#{@content_view_version.content_view}/#{@content_view_version.version}/#{@destination_server}/#{date_dir}".gsub(/\s/, '_')
           "#{@content_view_version.organization.label}/#{export_path}"
@@ -25,28 +20,8 @@ module Katello
           DateTime.now.to_s.gsub(/\W/, '-')
         end
 
-        def api
-          ::Katello::Pulp3::Api::Core.new(@smart_proxy)
-        end
-
-        def repository_hrefs
-          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
-        end
-
-        def version_hrefs
-          if @content_view_version.default?
-            @content_view_version.repositories.yum_type.pluck(:version_href).compact
-          else
-            @content_view_version.archived_repos.yum_type.pluck(:version_href).compact
-          end
-        end
-
-        def version_href_to_repository_href(version_href)
-          version_href.split("/")[0..-3].join("/") + "/"
-        end
-
         def create_exporter(export_base_dir: Setting['pulpcore_export_destination'])
-          api.exporter_api.create(name: generate_exporter_id,
+          api.exporter_api.create(name: generate_id,
                                   path: "#{export_base_dir}/#{generate_exporter_path}",
                                   repositories: repository_hrefs)
         end
@@ -65,6 +40,27 @@ module Katello
           api.export_api.delete(export_data.pulp_href) unless export_data.blank?
           api.exporter_api.delete(exporter_href)
         end
+
+        def generate_metadata
+          ret = { organization: @content_view_version.organization.name,
+                  repository_mapping: {},
+                  content_view: @content_view_version.content_view.name,
+                  content_view_version: {
+                    major: @content_view_version.major,
+                    minor: @content_view_version.minor
+                  }
+          }
+          repositories.each do |repo|
+            next if repo.version_href.blank?
+            pulp3_repo = fetch_repository_info(repo.version_href).name
+            ret[:repository_mapping][pulp3_repo] = {
+              repository: repo.root.name,
+              product: repo.root.product.name,
+              redhat: repo.redhat?
+            }
+          end
+          ret
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -0,0 +1,87 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class Import
+        include ImportExportCommon
+        BASEDIR = '/var/lib/pulp'.freeze
+
+        def initialize(smart_proxy:, content_view_version: nil, path: nil)
+          @smart_proxy = smart_proxy
+          @content_view_version = content_view_version
+          @path = path
+        end
+
+        def repository_mapping
+          mapping = {}
+          metadata[:repository_mapping].each do |key, value|
+            repo = @content_view_version.importable_repositories.joins(:root, :product).
+                        where("#{::Katello::Product.table_name}" => {:name => value[:product]},
+                                                  "#{::Katello::RootRepository.table_name}" => {:name => value[:repository]}).first
+            next unless repo&.version_href
+            repo_info = fetch_repository_info(repo.version_href)
+            mapping[key] = repo_info.name
+          end
+          mapping
+        end
+
+        def create_importer
+          api.importer_api.create(name: generate_id,
+                                  repo_mapping: repository_mapping)
+        end
+
+        def create_import(importer_href)
+          [api.import_api.create(importer_href, toc: "#{@path}/#{metadata[:toc]}")]
+        end
+
+        def fetch_import(importer_href)
+          api.import_api.list(importer_href).results.first
+        end
+
+        def destroy_importer(importer_href)
+          import_data = fetch_import(importer_href)
+          api.import_api.delete(import_data.pulp_href) unless import_data.blank?
+          api.importer_api.delete(importer_href)
+        end
+
+        def metadata
+          @metadata ||= self.class.metadata(@path)
+        end
+
+        class << self
+          def metadata(path)
+            JSON.parse(File.read("#{path}/#{Export::METADATA_FILE}")).with_indifferent_access
+          end
+
+          def check_permissions!(path)
+            fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
+            fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
+            metadata_file = "#{path}/#{::Katello::Pulp3::ContentViewVersion::Export::METADATA_FILE}"
+            fail _("Could not find metadata.json at '%s'." % metadata_file) unless File.exist?(metadata_file)
+            fail _("Unable to read the metadata.json at '%s'." % metadata_file) unless File.readable?(metadata_file)
+            fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
+            Dir.glob("#{path}/*").each do |file|
+              next if file == metadata_file
+              fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
+            end
+          end
+
+          def pulp_user_accessible?(path)
+            pulp_info = fetch_pulp_user_info
+            return false if pulp_info.blank?
+
+            stat = File.stat(path)
+            stat.gid.to_s == pulp_info.gid ||
+              stat.uid.to_s == pulp_info.uid ||
+              stat.mode.to_s(8)[-1].to_i >= 4
+          end
+
+          def fetch_pulp_user_info
+            pulp_user = nil
+            Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
+            pulp_user
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -0,0 +1,44 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      module ImportExportCommon
+        def generate_name
+          @content_view_version.name.gsub(/\s/, '_')
+        end
+
+        def generate_id
+          "#{@content_view_version.organization.label}_#{generate_name}"
+        end
+
+        def api
+          ::Katello::Pulp3::Api::Core.new(@smart_proxy)
+        end
+
+        def fetch_repository_info(version_href)
+          repo_api = ::Katello::Pulp3::Api::Yum.new(@smart_proxy).repositories_api
+          repo_api.read(version_href_to_repository_href(version_href))
+        end
+
+        def repository_hrefs
+          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
+        end
+
+        def version_hrefs
+          repositories.pluck(:version_href).compact
+        end
+
+        def repositories
+          if @content_view_version.default?
+            @content_view_version.repositories.yum_type
+          else
+            @content_view_version.archived_repos.yum_type
+          end
+        end
+
+        def version_href_to_repository_href(version_href)
+          version_href.split("/")[0..-3].join("/") + "/"
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/erratum.rb

@@ -35,7 +35,8 @@ module Katello
         custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])
         custom_json["updated"] = convert_date_if_epoch(custom_json["updated"]) unless custom_json["updated"].blank?
 
-        if model.updated.blank? || (custom_json['updated'].to_datetime != model.updated.to_datetime)
+        if model.updated.blank? ||
+            (custom_json['updated'] && (custom_json['updated'].to_datetime != model.updated.to_datetime))
           custom_json['errata_id'] = custom_json.delete('id')
           custom_json['errata_type'] = custom_json.delete('type')
           custom_json['updated'] = custom_json['updated'].blank? ? custom_json['issued'] : custom_json['updated']

data/app/services/katello/pulp3/repository.rb

@@ -1,5 +1,4 @@
 require "pulpcore_client"
-
 module Katello
   module Pulp3
     class Repository
@@ -188,8 +187,9 @@ module Katello
         api.repositories_api.read(repository_reference.try(:repository_href))
       end
 
-      def delete(href = repository_reference.try(:repository_href))
-        repository_reference.try(:destroy)
+      def delete_repository(repo_reference = repository_reference)
+        href = repo_reference.try(:repository_href)
+        repo_reference.try(:destroy)
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
@@ -251,8 +251,12 @@ module Katello
         create_version(:base_version => from_repository.version_href)
       end
 
+      def version_zero?
+        repo.version_href.ends_with?('/versions/0/')
+      end
+
       def delete_version
-        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
+        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) } unless version_zero?
       end
 
       def create_version(options = {})

data/app/services/katello/pulp3/repository/yum.rb

@@ -3,10 +3,13 @@ require 'pulp_rpm_client'
 module Katello
   module Pulp3
     class Repository
+      # rubocop:disable Metrics/ClassLength
       class Yum < ::Katello::Pulp3::Repository
         include Katello::Util::Errata
         include Katello::Util::PulpcoreContentFilters
 
+        UNIT_LIMIT = 10_000
+
         def remote_options
           url, sles_token = extract_sles_token
           options = common_remote_options
@@ -109,14 +112,70 @@ module Katello
                 data.config << config
               end
             end
-            # FIXME: data's content being [] causes all content to be copied back
-            tasks << api.copy_api.copy_content(data)
+            tasks << copy_content_chunked(data)
           else
             tasks << remove_all_content_from_mapping(repo_id_map)
           end
           tasks.flatten
         end
 
+        def copy_api_data_dup(data)
+          data_dup = PulpRpmClient::Copy.new
+          data_dup.dependency_solving = data.dependency_solving
+          data_dup.config = []
+          data.config.each do |repo_config|
+            config_hash = {
+              source_repo_version: repo_config[:source_repo_version],
+              dest_repo: repo_config[:dest_repo],
+              content: []
+            }
+            config_hash[:dest_base_version] = repo_config[:dest_base_version] if repo_config[:dest_base_version]
+            data_dup.config << config_hash
+          end
+          data_dup
+        end
+
+        def copy_content_chunked(data)
+          tasks = []
+          # Don't chunk if there aren't enough content units
+          if data.config.sum { |repo_config| repo_config[:content].size } <= UNIT_LIMIT
+            return api.copy_api.copy_content(data)
+          end
+
+          unit_copy_counter = 0
+          i = 0
+          leftover_units = data.config.first[:content].deep_dup
+
+          # Copy data and clear its content fields
+          data_dup = copy_api_data_dup(data)
+
+          while i < data_dup.config.size
+            # Copy all units within repo or only some?
+            if leftover_units.length < UNIT_LIMIT - unit_copy_counter
+              copy_amount = leftover_units.length
+            else
+              copy_amount = UNIT_LIMIT - unit_copy_counter
+            end
+
+            data_dup.config[i][:content] = leftover_units.pop(copy_amount)
+            unit_copy_counter += copy_amount
+            if unit_copy_counter != 0
+              tasks << api.copy_api.copy_content(data_dup)
+              unit_copy_counter = 0
+            end
+
+            if leftover_units.empty?
+              # Nothing more to copy -- clear current config's content
+              data_dup.config[i][:content] = []
+              i += 1
+              # Fetch unit list for next data config
+              leftover_units = data.config[i][:content].deep_dup unless i == data_dup.config.size
+            end
+          end
+
+          tasks
+        end
+
         def remove_all_content_from_mapping(repo_id_map)
           tasks = []
           repo_id_map.each do |_source_repo_ids, dest_repo_id_map|
@@ -148,6 +207,16 @@ module Katello
           tasks
         end
 
+        def copy_all(source_repository)
+          data = PulpRpmClient::Copy.new
+          data.config = [{
+            source_repo_version: source_repository.version_href,
+            dest_repo: repository_reference.repository_href
+          }]
+
+          [api.copy_api.copy_content(data)]
+        end
+
         def remove_all_content_from_repo(repo_href)
           data = PulpRpmClient::RepositoryAddRemoveContent.new(
             remove_content_units: ['*'])
@@ -271,7 +340,6 @@ module Katello
           end
 
           filters.flatten!.compact!
-
           whitelist_ids = []
           blacklist_ids = []
           filters.each do |filter|
@@ -298,7 +366,6 @@ module Katello
             content_unit_hrefs += additional_content_hrefs(source_repository, content_unit_hrefs)
           end
           content_unit_hrefs += source_repository.srpms.pluck(:pulp_id)
-
           dependency_solving = options[:solve_dependencies] || false
           copy_units(source_repository, content_unit_hrefs.uniq, dependency_solving)
         end

data/app/services/katello/pulp3/task.rb

@@ -61,7 +61,7 @@ module Katello
       end
 
       def task_group_href
-        task_data[:task_group]
+        task_data[:task_group] || task_data[:created_resources].find { |href| href.starts_with?("/pulp/api/v3/task-groups/") }
       end
 
       def done?
@@ -79,7 +79,7 @@ module Katello
 
       def error
         if task_data[:state] == CANCELED
-          self.new(_("Task canceled"))
+          _("Task canceled")
         elsif task_data[:state] == FAILED
           if task_data[:error][:description].blank?
             _("Pulp task error")
@@ -90,8 +90,8 @@ module Katello
       end
 
       def cancel
-        data = PulpcoreClient::Task.new(state: 'canceled')
-        tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
+        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
+        tasks_api.tasks_cancel(task_data['pulp_href'], data)
         #the main task may have completed, so cancel spawned tasks too
         task_data['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
       end

data/app/services/katello/pulp3/task_group.rb

@@ -73,6 +73,12 @@ module Katello
       end
 
       def cancel
+        tasks_api = ::Katello::Pulp3::Api::Core.new(@smart_proxy).tasks_api
+        tasks_response = tasks_api.list(task_group: task_group_data['pulp_href'])
+        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
+        tasks_response.results.collect do |result|
+          tasks_api.tasks_cancel(result.pulp_href, data)
+        end
       end
     end
   end