katello 3.17.0.rc2.2 → 3.17.0

data/app/lib/actions/pulp3/content_view_version/destroy_importer.rb

@@ -0,0 +1,16 @@
+module Actions
+  module Pulp3
+    module ContentViewVersion
+      class DestroyImporter < Pulp3::Abstract
+        input_format do
+          param :smart_proxy_id, Integer
+          param :importer_data, Hash
+        end
+
+        def run
+          ::Katello::Pulp3::ContentViewVersion::Import.new(smart_proxy: smart_proxy).destroy_importer(input[:importer_data][:pulp_href])
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/content_view_version/import.rb

@@ -0,0 +1,21 @@
+module Actions
+  module Pulp3
+    module ContentViewVersion
+      class Import < Pulp3::AbstractAsyncTask
+        input_format do
+          param :content_view_version_id, Integer
+          param :smart_proxy_id, Integer
+          param :importer_data, Hash
+          param :path, String
+        end
+
+        def invoke_external_task
+          cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+          output[:pulp_tasks] = ::Katello::Pulp3::ContentViewVersion::Import.new(smart_proxy: smart_proxy,
+                                                   content_view_version: cvv,
+                                                   path: input[:path]).create_import(input[:importer_data][:pulp_href])
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/content_view_version/copy_version_units_to_library.rb

@@ -0,0 +1,25 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module ContentViewVersion
+        class CopyVersionUnitsToLibrary < Actions::EntryAction
+          def plan(content_view_version)
+            concurrence do
+              content_view_version.importable_repositories.each do |repo|
+                sequence do
+                  copy_action = plan_action(Actions::Pulp3::Repository::CopyContent, repo, SmartProxy.pulp_primary!,
+                                                    repo.library_instance,
+                                                    copy_all: true)
+                  plan_action(Actions::Pulp3::Repository::SaveVersion, repo.library_instance,
+                                tasks: copy_action.output[:pulp_tasks])
+                  plan_action(Katello::Repository::IndexContent, id: repo.library_instance_id)
+                  plan_action(Katello::Repository::MetadataGenerate, repo.library_instance, :force => true)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/content_view_version/export.rb

@@ -50,9 +50,17 @@ module Actions
             api = ::Katello::Pulp3::Api::Core.new(smart_proxy)
             export_data = api.export_api.list(input[:exporter_data][:pulp_href]).results.first
             output[:exported_file_name], output[:exported_file_checksum] = export_data.output_file_info.first
+            path = File.dirname(output[:exported_file_name].to_s)
             ::Katello::ContentViewVersionExportHistory.create!(content_view_version_id: input[:content_view_version_id],
                                                                destination_server: input[:destination_server],
-                                                               path: File.dirname(output[:exported_file_name].to_s))
+                                                               path: path)
+            cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+
+            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(:content_view_version => cvv,
+                                                               :smart_proxy => smart_proxy).generate_metadata
+            toc = Dir.glob("#{path}/*toc.json").first
+            export_metadata[:toc] = File.basename(toc) if toc
+            File.write("#{path}/#{::Katello::Pulp3::ContentViewVersion::Export::METADATA_FILE}", export_metadata.to_json)
           end
 
           def humanized_name

data/app/lib/actions/pulp3/orchestration/content_view_version/import.rb

@@ -0,0 +1,41 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module ContentViewVersion
+        class Import < Actions::EntryAction
+          def plan(content_view_version, path:)
+            action_subject(content_view_version)
+            sequence do
+              smart_proxy = SmartProxy.pulp_primary!
+              importer_output = plan_action(::Actions::Pulp3::ContentViewVersion::CreateImporter,
+                                           content_view_version_id: content_view_version.id,
+                                           smart_proxy_id: smart_proxy.id,
+                                           path: path).output
+
+              import_output = plan_action(::Actions::Pulp3::ContentViewVersion::Import,
+                                           content_view_version_id: content_view_version.id,
+                                           smart_proxy_id: smart_proxy.id,
+                                           importer_data: importer_output[:importer_data],
+                                           path: path).output
+
+              plan_action(Actions::Pulp3::Repository::SaveVersions, content_view_version.importable_repositories.pluck(:id),
+                          tasks: import_output[:pulp_tasks])
+
+              plan_action(::Actions::Pulp3::ContentViewVersion::DestroyImporter,
+                            smart_proxy_id: smart_proxy.id,
+                            importer_data: importer_output[:importer_data])
+            end
+          end
+
+          def humanized_name
+            _("Import")
+          end
+
+          def rescue_strategy
+            Dynflow::Action::Rescue::Skip
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/copy_content.rb

@@ -11,7 +11,12 @@ module Actions
         def invoke_external_task
           source = ::Katello::Repository.find(input[:source_repository_id])
           target = ::Katello::Repository.find(input[:target_repository_id] || input[:target_repository])
-          output[:pulp_tasks] = target.backend_service(smart_proxy).copy_content_for_source(source, input)
+          service = target.backend_service(smart_proxy)
+          output[:pulp_tasks] = if input[:copy_all]
+                                  service.copy_all(source)
+                                else
+                                  service.copy_content_for_source(source, input)
+                                end
         end
       end
     end

data/app/lib/actions/pulp3/repository/multi_copy_content.rb

@@ -17,7 +17,7 @@ module Actions
           repo_id_map = {}
 
           input[:repo_id_map].each do |source_repo_ids, dest_repo_map|
-            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map
+            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map.deep_dup
           end
 
           output[:pulp_tasks] = ::Katello::Repository.find(repo_id_map.values.first[:dest_repo]).backend_service(smart_proxy).copy_content_from_mapping(repo_id_map, input)

data/app/lib/katello/resources/cdn.rb

@@ -18,7 +18,7 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product, :options
+        attr_reader :url, :product, :options, :proxy
 
         def substitutor(logger = nil)
           @logger = logger
@@ -26,6 +26,7 @@ module Katello
         end
 
         def initialize(url, options = {})
+          @proxy = ::HttpProxy.default_global_content_proxy
           @ssl_version = Setting[:cdn_ssl_version]
           if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
             fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
@@ -140,7 +141,7 @@ module Katello
         end
 
         def net_http_class
-          if (proxy = ::HttpProxy.default_global_content_proxy)
+          if self.proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
             Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -59,15 +59,17 @@ module Katello
 
         return resolved if to_resolve.empty?
 
-        futures = to_resolve.map do |path_with_substitution|
-          Concurrent::Promises.future do
-            path_with_substitution.resolve_substitutions(@resource)
+        to_resolve.in_groups_of(8) do |group|
+          futures = group.map do |path_with_substitution|
+            Concurrent::Promises.future do
+              path_with_substitution.resolve_substitutions(@resource)
+            end
           end
-        end
 
-        futures.each do |future|
-          resolved << future.value
-          Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          futures.each do |future|
+            resolved << future.value
+            Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          end
         end
 
         find_substitutions(resolved.compact.flatten)

data/app/models/katello/content_view.rb

@@ -579,6 +579,12 @@ module Katello
       PuppetModule.group_by_repoid(puppet_modules.flatten)
     end
 
+    def check_ready_to_import!
+      fail _("User must be logged in.") if ::User.current.nil?
+      fail _("Cannot import a composite content view") if composite?
+      true
+    end
+
     def check_ready_to_publish!
       fail _("User must be logged in.") if ::User.current.nil?
       fail _("Cannot publish default content view") if default?

data/app/models/katello/content_view_version.rb

@@ -1,4 +1,5 @@
 module Katello
+  # rubocop:disable Metrics/ClassLength
   class ContentViewVersion < Katello::Model
     include Authorization::ContentViewVersion
     include ForemanTasks::Concerns::ActionSubject
@@ -25,7 +26,7 @@ module Katello
     has_many :export_histories, :class_name => "Katello::ContentViewVersionExportHistory", :dependent => :destroy,
              :inverse_of => :content_view_version, :foreign_key => :content_view_version_id
 
-    has_many :repositories, :class_name => "Katello::Repository", :dependent => :destroy
+    has_many :repositories, :class_name => "::Katello::Repository", :dependent => :destroy
     has_many :content_view_puppet_environments, :class_name => "Katello::ContentViewPuppetEnvironment",
                                                 :dependent => :destroy
     has_one :task_status, :class_name => "Katello::TaskStatus", :as => :task_owner, :dependent => :destroy
@@ -357,6 +358,14 @@ module Katello
       true
     end
 
+    def importable_repositories
+      if default?
+        repositories.yum_type
+      else
+        archived_repos.yum_type
+      end
+    end
+
     def before_promote_hooks
       run_callbacks :sync do
         logger.debug "custom hook before_promote on #{name} will be executed if defined."

data/app/models/katello/glue/pulp/repo.rb

@@ -369,7 +369,7 @@ module Katello
       pulp_uri = URI.parse(smart_proxy ? smart_proxy.url : SETTINGS[:katello][:pulp][:url])
       scheme   = (self.unprotected && !force_https) ? 'http' : 'https'
       if docker?
-        "#{pulp_uri.host.downcase}:#{Setting['pulp_docker_registry_port']}/#{container_repository_name}"
+        "#{pulp_uri.host.downcase}/#{container_repository_name}"
       elsif file?
         "#{scheme}://#{pulp_uri.host.downcase}/pulp/isos/#{relative_path}/"
       elsif puppet?

data/app/models/katello/root_repository.rb

@@ -321,7 +321,11 @@ module Katello
 
     def format_arches
       if content_type == ::Katello::Repository::DEB_TYPE
-        self.deb_architectures
+        # FIXME: This should be set to self.deb_architectures but it needs to have the
+        # subscription-manager PR https://github.com/candlepin/subscription-manager/pull/2213
+        # merged. Otherwise subscription-manager returns _NO_ debian repository as described in
+        # https://community.theforeman.org/t/katello-3-16-1-1-el7-subscription-manager-doesnt-create-rhsm-repos-for-ubuntu/20928/38
+        nil
       else
         self.arch == "noarch" ? nil : self.arch
       end

data/app/overrides/add_organization_attributes.rb

@@ -22,3 +22,15 @@ Deface::Override.new(:virtual_path => "taxonomies/_form",
                      :name => "add_organization_attributes_on_edit",
                      :insert_after => 'erb[loud]:contains("text_f"):contains(":name")',
                      :partial => 'overrides/organizations/edit_override')
+
+# Add Simple Content Access column to org table
+Deface::Override.new(:virtual_path => "taxonomies/index",
+                     :name => "add_sca_column_on_index",
+                     :insert_before => 'table th:last-child', # make it the second-to-last column
+                     :partial => 'overrides/organizations/index_header_override')
+
+# Add Simple Content Access cells to org table
+Deface::Override.new(:virtual_path => "taxonomies/index",
+                    :name => "add_sca_attributes_on_index",
+                    :insert_before => 'tbody td:last-child',
+                    :partial => 'overrides/organizations/index_row_override')

data/app/services/katello/host_trace_manager.rb

@@ -0,0 +1,38 @@
+module Katello
+  class HostTraceManager
+    def self.resolve_traces(traces)
+      traces.each do |trace|
+        if trace.reboot_required?
+          trace.helper = 'reboot'
+        end
+      end
+
+      traces_by_host_id = traces.group_by(&:host_id)
+      traces_by_helper = traces.group_by(&:helper)
+
+      composers = []
+
+      if traces_by_host_id.size < traces_by_helper.size
+        traces_by_host_id.each do |host_id, trace|
+          needed_traces = trace.map(&:helper).join(',')
+          joined_helpers = { :helper => needed_traces }
+          composers << ::JobInvocationComposer.for_feature(:katello_service_restart, [host_id], joined_helpers)
+        end
+      else
+        traces_by_helper.each do |helper, trace|
+          helpers = { :helper => helper }
+          composers << ::JobInvocationComposer.for_feature(:katello_service_restart, trace.map(&:host_id), helpers)
+        end
+      end
+
+      job_invocations = []
+
+      composers.each do |composer|
+        composer.trigger
+        job_invocations << composer.job_invocation
+      end
+
+      job_invocations
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -2,20 +2,15 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class Export
+        include ImportExportCommon
+        METADATA_FILE = "metadata.json".freeze
+
         def initialize(smart_proxy:, content_view_version: nil, destination_server: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
         end
 
-        def exporter_name
-          @content_view_version.name.gsub(/\s/, '_')
-        end
-
-        def generate_exporter_id
-          "#{@content_view_version.organization.label}_#{exporter_name}"
-        end
-
         def generate_exporter_path
           export_path = "#{@content_view_version.content_view}/#{@content_view_version.version}/#{@destination_server}/#{date_dir}".gsub(/\s/, '_')
           "#{@content_view_version.organization.label}/#{export_path}"
@@ -25,28 +20,8 @@ module Katello
           DateTime.now.to_s.gsub(/\W/, '-')
         end
 
-        def api
-          ::Katello::Pulp3::Api::Core.new(@smart_proxy)
-        end
-
-        def repository_hrefs
-          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
-        end
-
-        def version_hrefs
-          if @content_view_version.default?
-            @content_view_version.repositories.yum_type.pluck(:version_href).compact
-          else
-            @content_view_version.archived_repos.yum_type.pluck(:version_href).compact
-          end
-        end
-
-        def version_href_to_repository_href(version_href)
-          version_href.split("/")[0..-3].join("/") + "/"
-        end
-
         def create_exporter(export_base_dir: Setting['pulpcore_export_destination'])
-          api.exporter_api.create(name: generate_exporter_id,
+          api.exporter_api.create(name: generate_id,
                                   path: "#{export_base_dir}/#{generate_exporter_path}",
                                   repositories: repository_hrefs)
         end
@@ -65,6 +40,27 @@ module Katello
           api.export_api.delete(export_data.pulp_href) unless export_data.blank?
           api.exporter_api.delete(exporter_href)
         end
+
+        def generate_metadata
+          ret = { organization: @content_view_version.organization.name,
+                  repository_mapping: {},
+                  content_view: @content_view_version.content_view.name,
+                  content_view_version: {
+                    major: @content_view_version.major,
+                    minor: @content_view_version.minor
+                  }
+          }
+          repositories.each do |repo|
+            next if repo.version_href.blank?
+            pulp3_repo = fetch_repository_info(repo.version_href).name
+            ret[:repository_mapping][pulp3_repo] = {
+              repository: repo.root.name,
+              product: repo.root.product.name,
+              redhat: repo.redhat?
+            }
+          end
+          ret
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -0,0 +1,87 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class Import
+        include ImportExportCommon
+        BASEDIR = '/var/lib/pulp'.freeze
+
+        def initialize(smart_proxy:, content_view_version: nil, path: nil)
+          @smart_proxy = smart_proxy
+          @content_view_version = content_view_version
+          @path = path
+        end
+
+        def repository_mapping
+          mapping = {}
+          metadata[:repository_mapping].each do |key, value|
+            repo = @content_view_version.importable_repositories.joins(:root, :product).
+                        where("#{::Katello::Product.table_name}" => {:name => value[:product]},
+                                                  "#{::Katello::RootRepository.table_name}" => {:name => value[:repository]}).first
+            next unless repo&.version_href
+            repo_info = fetch_repository_info(repo.version_href)
+            mapping[key] = repo_info.name
+          end
+          mapping
+        end
+
+        def create_importer
+          api.importer_api.create(name: generate_id,
+                                  repo_mapping: repository_mapping)
+        end
+
+        def create_import(importer_href)
+          [api.import_api.create(importer_href, toc: "#{@path}/#{metadata[:toc]}")]
+        end
+
+        def fetch_import(importer_href)
+          api.import_api.list(importer_href).results.first
+        end
+
+        def destroy_importer(importer_href)
+          import_data = fetch_import(importer_href)
+          api.import_api.delete(import_data.pulp_href) unless import_data.blank?
+          api.importer_api.delete(importer_href)
+        end
+
+        def metadata
+          @metadata ||= self.class.metadata(@path)
+        end
+
+        class << self
+          def metadata(path)
+            JSON.parse(File.read("#{path}/#{Export::METADATA_FILE}")).with_indifferent_access
+          end
+
+          def check_permissions!(path)
+            fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
+            fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
+            metadata_file = "#{path}/#{::Katello::Pulp3::ContentViewVersion::Export::METADATA_FILE}"
+            fail _("Could not find metadata.json at '%s'." % metadata_file) unless File.exist?(metadata_file)
+            fail _("Unable to read the metadata.json at '%s'." % metadata_file) unless File.readable?(metadata_file)
+            fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
+            Dir.glob("#{path}/*").each do |file|
+              next if file == metadata_file
+              fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
+            end
+          end
+
+          def pulp_user_accessible?(path)
+            pulp_info = fetch_pulp_user_info
+            return false if pulp_info.blank?
+
+            stat = File.stat(path)
+            stat.gid.to_s == pulp_info.gid ||
+              stat.uid.to_s == pulp_info.uid ||
+              stat.mode.to_s(8)[-1].to_i >= 4
+          end
+
+          def fetch_pulp_user_info
+            pulp_user = nil
+            Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
+            pulp_user
+          end
+        end
+      end
+    end
+  end
+end