katello 3.17.0.rc2.1 → 3.17.3

data/app/lib/actions/katello/organization/simple_content_access/toggle.rb

@@ -4,6 +4,7 @@ module Actions
       module SimpleContentAccess
         class Toggle < Actions::AbstractAsyncTask
           middleware.use Actions::Middleware::PropagateCandlepinErrors
+          include Helpers::Notifications
 
           SIMPLE_CONTENT_ACCESS_DISABLED_VALUE = "entitlement".freeze
           SIMPLE_CONTENT_ACCESS_ENABLED_VALUE = "org_environment".freeze
@@ -12,7 +13,7 @@ module Actions
 
           def plan(organization_id)
             @organization = ::Organization.find(organization_id)
-
+            action_subject organization
             ::Katello::Resources::Candlepin::UpstreamConsumer.update(
               "#{consumer['apiUrl']}#{consumer['uuid']}",
               consumer['idCert']['cert'],
@@ -24,10 +25,23 @@ module Actions
             plan_action(::Actions::Katello::Organization::ManifestRefresh, organization)
           end
 
+          def failure_notification(plan)
+            task_error_notification.deliver!(
+              :subject => subject_organization,
+              :task => get_foreman_task(plan)
+            )
+          end
+
+          def success_notification(_plan)
+            task_success_notification.deliver!(
+              subject_organization
+            )
+          end
+
           private
 
           def consumer
-            @consumer ||= ::Katello::Resources::Candlepin::UpstreamCandlepinResource.upstream_consumer
+            @consumer ||= @organization.owner_details['upstreamConsumer']
           end
         end
       end

data/app/lib/actions/katello/repository/multi_clone_contents.rb

@@ -16,14 +16,16 @@ module Actions
                           solve_dependencies: solve_dependencies)
             end
 
-            extended_repo_mapping.each do |source_repos, dest_repo_map|
-              if generate_metadata
-                metadata_generate(source_repos, dest_repo_map[:dest_repo], dest_repo_map[:filters])
+            concurrence do
+              extended_repo_mapping.each do |source_repos, dest_repo_map|
+                if generate_metadata
+                  metadata_generate(source_repos, dest_repo_map[:dest_repo], dest_repo_map[:filters])
+                end
               end
-            end
 
-            extended_repo_mapping.values.each do |dest_repo_map|
-              plan_action(Katello::Repository::IndexContent, id: dest_repo_map[:dest_repo].id)
+              extended_repo_mapping.values.each do |dest_repo_map|
+                plan_action(Katello::Repository::IndexContent, id: dest_repo_map[:dest_repo].id)
+              end
             end
           end
         end

data/app/lib/actions/pulp3/abstract_async_task.rb

@@ -74,6 +74,7 @@ module Actions
 
       def cancel
         pulp_tasks.each { |task| task.cancel }
+        task_groups.each { |task_group| task_group.cancel }
       end
 
       def rescue_external_task(error)

data/app/lib/actions/pulp3/content_view/delete_repository_references.rb

@@ -14,7 +14,7 @@ module Actions
           content_view.repository_references.each do |repository_reference|
             repo = repository_reference.root_repository.library_instance
             #force pulp3 in case we've done migrations, but haven't switched over yet
-            tasks << repo.backend_service(smart_proxy, true).delete(repository_reference.repository_href)
+            tasks << repo.backend_service(smart_proxy, true).delete_repository(repository_reference)
           end
           content_view.repository_references.destroy_all
 

data/app/lib/actions/pulp3/content_view_version/create_importer.rb

@@ -0,0 +1,20 @@
+module Actions
+  module Pulp3
+    module ContentViewVersion
+      class CreateImporter < Pulp3::Abstract
+        input_format do
+          param :smart_proxy_id, Integer
+          param :content_view_version_id, Integer
+          param :path, String
+        end
+
+        def run
+          cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+          output[:importer_data] = ::Katello::Pulp3::ContentViewVersion::Import.new(smart_proxy: smart_proxy,
+                                                                            content_view_version: cvv,
+                                                                            path: input[:path]).create_importer
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/content_view_version/destroy_importer.rb

@@ -0,0 +1,16 @@
+module Actions
+  module Pulp3
+    module ContentViewVersion
+      class DestroyImporter < Pulp3::Abstract
+        input_format do
+          param :smart_proxy_id, Integer
+          param :importer_data, Hash
+        end
+
+        def run
+          ::Katello::Pulp3::ContentViewVersion::Import.new(smart_proxy: smart_proxy).destroy_importer(input[:importer_data][:pulp_href])
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/content_view_version/import.rb

@@ -0,0 +1,21 @@
+module Actions
+  module Pulp3
+    module ContentViewVersion
+      class Import < Pulp3::AbstractAsyncTask
+        input_format do
+          param :content_view_version_id, Integer
+          param :smart_proxy_id, Integer
+          param :importer_data, Hash
+          param :path, String
+        end
+
+        def invoke_external_task
+          cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+          output[:pulp_tasks] = ::Katello::Pulp3::ContentViewVersion::Import.new(smart_proxy: smart_proxy,
+                                                   content_view_version: cvv,
+                                                   path: input[:path]).create_import(input[:importer_data][:pulp_href])
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/content_view_version/copy_version_units_to_library.rb

@@ -0,0 +1,25 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module ContentViewVersion
+        class CopyVersionUnitsToLibrary < Actions::EntryAction
+          def plan(content_view_version)
+            concurrence do
+              content_view_version.importable_repositories.each do |repo|
+                sequence do
+                  copy_action = plan_action(Actions::Pulp3::Repository::CopyContent, repo, SmartProxy.pulp_primary!,
+                                                    repo.library_instance,
+                                                    copy_all: true)
+                  plan_action(Actions::Pulp3::Repository::SaveVersion, repo.library_instance,
+                                tasks: copy_action.output[:pulp_tasks])
+                  plan_action(Katello::Repository::IndexContent, id: repo.library_instance_id)
+                  plan_action(Katello::Repository::MetadataGenerate, repo.library_instance, :force => true)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/content_view_version/export.rb

@@ -50,9 +50,17 @@ module Actions
             api = ::Katello::Pulp3::Api::Core.new(smart_proxy)
             export_data = api.export_api.list(input[:exporter_data][:pulp_href]).results.first
             output[:exported_file_name], output[:exported_file_checksum] = export_data.output_file_info.first
+            path = File.dirname(output[:exported_file_name].to_s)
             ::Katello::ContentViewVersionExportHistory.create!(content_view_version_id: input[:content_view_version_id],
                                                                destination_server: input[:destination_server],
-                                                               path: File.dirname(output[:exported_file_name].to_s))
+                                                               path: path)
+            cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+
+            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(:content_view_version => cvv,
+                                                               :smart_proxy => smart_proxy).generate_metadata
+            toc = Dir.glob("#{path}/*toc.json").first
+            export_metadata[:toc] = File.basename(toc) if toc
+            File.write("#{path}/#{::Katello::Pulp3::ContentViewVersion::Export::METADATA_FILE}", export_metadata.to_json)
           end
 
           def humanized_name

data/app/lib/actions/pulp3/orchestration/content_view_version/import.rb

@@ -0,0 +1,41 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module ContentViewVersion
+        class Import < Actions::EntryAction
+          def plan(content_view_version, path:)
+            action_subject(content_view_version)
+            sequence do
+              smart_proxy = SmartProxy.pulp_primary!
+              importer_output = plan_action(::Actions::Pulp3::ContentViewVersion::CreateImporter,
+                                           content_view_version_id: content_view_version.id,
+                                           smart_proxy_id: smart_proxy.id,
+                                           path: path).output
+
+              import_output = plan_action(::Actions::Pulp3::ContentViewVersion::Import,
+                                           content_view_version_id: content_view_version.id,
+                                           smart_proxy_id: smart_proxy.id,
+                                           importer_data: importer_output[:importer_data],
+                                           path: path).output
+
+              plan_action(Actions::Pulp3::Repository::SaveVersions, content_view_version.importable_repositories.pluck(:id),
+                          tasks: import_output[:pulp_tasks])
+
+              plan_action(::Actions::Pulp3::ContentViewVersion::DestroyImporter,
+                            smart_proxy_id: smart_proxy.id,
+                            importer_data: importer_output[:importer_data])
+            end
+          end
+
+          def humanized_name
+            _("Import")
+          end
+
+          def rescue_strategy
+            Dynflow::Action::Rescue::Skip
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/repository/copy_content.rb

@@ -11,7 +11,12 @@ module Actions
         def invoke_external_task
           source = ::Katello::Repository.find(input[:source_repository_id])
           target = ::Katello::Repository.find(input[:target_repository_id] || input[:target_repository])
-          output[:pulp_tasks] = target.backend_service(smart_proxy).copy_content_for_source(source, input)
+          service = target.backend_service(smart_proxy)
+          output[:pulp_tasks] = if input[:copy_all]
+                                  service.copy_all(source)
+                                else
+                                  service.copy_content_for_source(source, input)
+                                end
         end
       end
     end

data/app/lib/actions/pulp3/repository/delete.rb

@@ -8,7 +8,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
-          output[:response] = repo.backend_service(smart_proxy).delete
+          output[:response] = repo.backend_service(smart_proxy).delete_repository
         end
       end
     end

data/app/lib/actions/pulp3/repository/multi_copy_content.rb

@@ -17,7 +17,7 @@ module Actions
           repo_id_map = {}
 
           input[:repo_id_map].each do |source_repo_ids, dest_repo_map|
-            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map
+            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map.deep_dup
           end
 
           output[:pulp_tasks] = ::Katello::Repository.find(repo_id_map.values.first[:dest_repo]).backend_service(smart_proxy).copy_content_from_mapping(repo_id_map, input)

data/app/lib/katello/resources/cdn.rb

@@ -18,7 +18,7 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product, :options
+        attr_reader :url, :product, :options, :proxy
 
         def substitutor(logger = nil)
           @logger = logger
@@ -26,6 +26,7 @@ module Katello
         end
 
         def initialize(url, options = {})
+          @proxy = ::HttpProxy.default_global_content_proxy
           @ssl_version = Setting[:cdn_ssl_version]
           if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
             fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
@@ -140,7 +141,7 @@ module Katello
         end
 
         def net_http_class
-          if (proxy = ::HttpProxy.default_global_content_proxy)
+          if self.proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
             Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -59,15 +59,17 @@ module Katello
 
         return resolved if to_resolve.empty?
 
-        futures = to_resolve.map do |path_with_substitution|
-          Concurrent::Promises.future do
-            path_with_substitution.resolve_substitutions(@resource)
+        to_resolve.in_groups_of(8) do |group|
+          futures = group.map do |path_with_substitution|
+            Concurrent::Promises.future do
+              path_with_substitution.resolve_substitutions(@resource)
+            end
           end
-        end
 
-        futures.each do |future|
-          resolved << future.value
-          Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          futures.each do |future|
+            resolved << future.value
+            Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          end
         end
 
         find_substitutions(resolved.compact.flatten)

data/app/models/katello/concerns/redhat_extensions.rb

@@ -44,8 +44,8 @@ module Katello
 
       def kickstart_repos(host)
         distros = distribution_repositories(host).where(distribution_bootable: true)
-        if distros && host.content_source
-          distros.map { |distro| distro.to_hash(host.content_source) }
+        if distros && host&.content_facet&.content_source
+          distros.map { |distro| distro.to_hash(host.content_facet.content_source) }
         else
           []
         end

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -177,7 +177,9 @@ module Katello
       end
 
       def fix_pulp3_capabilities(type)
-        if missing_pulp3_capabilities? && !pulp2_preferred_for_type?(type)
+        repository_type_obj = type.is_a?(String) ||  type.is_a?(Symbol) ? Katello::RepositoryTypeManager.repository_types[type] : type
+
+        if missing_pulp3_capabilities? && repository_type_obj.pulp3_plugin && !pulp2_preferred_for_type?(repository_type_obj.id)
           self.refresh
           if self.capabilities(::SmartProxy::PULP3_FEATURE).empty?
             fail Katello::Errors::PulpcoreMissingCapabilities

data/app/models/katello/content_view.rb

@@ -579,6 +579,12 @@ module Katello
       PuppetModule.group_by_repoid(puppet_modules.flatten)
     end
 
+    def check_ready_to_import!
+      fail _("User must be logged in.") if ::User.current.nil?
+      fail _("Cannot import a composite content view") if composite?
+      true
+    end
+
     def check_ready_to_publish!
       fail _("User must be logged in.") if ::User.current.nil?
       fail _("Cannot publish default content view") if default?

data/app/models/katello/content_view_version.rb

@@ -1,4 +1,5 @@
 module Katello
+  # rubocop:disable Metrics/ClassLength
   class ContentViewVersion < Katello::Model
     include Authorization::ContentViewVersion
     include ForemanTasks::Concerns::ActionSubject
@@ -25,7 +26,7 @@ module Katello
     has_many :export_histories, :class_name => "Katello::ContentViewVersionExportHistory", :dependent => :destroy,
              :inverse_of => :content_view_version, :foreign_key => :content_view_version_id
 
-    has_many :repositories, :class_name => "Katello::Repository", :dependent => :destroy
+    has_many :repositories, :class_name => "::Katello::Repository", :dependent => :destroy
     has_many :content_view_puppet_environments, :class_name => "Katello::ContentViewPuppetEnvironment",
                                                 :dependent => :destroy
     has_one :task_status, :class_name => "Katello::TaskStatus", :as => :task_owner, :dependent => :destroy
@@ -357,6 +358,14 @@ module Katello
       true
     end
 
+    def importable_repositories
+      if default?
+        repositories.yum_type
+      else
+        archived_repos.yum_type
+      end
+    end
+
     def before_promote_hooks
       run_callbacks :sync do
         logger.debug "custom hook before_promote on #{name} will be executed if defined."

data/app/models/katello/glue/pulp/repo.rb

@@ -369,7 +369,7 @@ module Katello
       pulp_uri = URI.parse(smart_proxy ? smart_proxy.url : SETTINGS[:katello][:pulp][:url])
       scheme   = (self.unprotected && !force_https) ? 'http' : 'https'
       if docker?
-        "#{pulp_uri.host.downcase}:#{Setting['pulp_docker_registry_port']}/#{container_repository_name}"
+        "#{pulp_uri.host.downcase}/#{container_repository_name}"
       elsif file?
         "#{scheme}://#{pulp_uri.host.downcase}/pulp/isos/#{relative_path}/"
       elsif puppet?

data/app/models/katello/root_repository.rb

@@ -321,7 +321,11 @@ module Katello
 
     def format_arches
       if content_type == ::Katello::Repository::DEB_TYPE
-        self.deb_architectures
+        # FIXME: This should be set to self.deb_architectures but it needs to have the
+        # subscription-manager PR https://github.com/candlepin/subscription-manager/pull/2213
+        # merged. Otherwise subscription-manager returns _NO_ debian repository as described in
+        # https://community.theforeman.org/t/katello-3-16-1-1-el7-subscription-manager-doesnt-create-rhsm-repos-for-ubuntu/20928/38
+        nil
       else
         self.arch == "noarch" ? nil : self.arch
       end

data/app/overrides/add_organization_attributes.rb

@@ -22,3 +22,15 @@ Deface::Override.new(:virtual_path => "taxonomies/_form",
                      :name => "add_organization_attributes_on_edit",
                      :insert_after => 'erb[loud]:contains("text_f"):contains(":name")',
                      :partial => 'overrides/organizations/edit_override')
+
+# Add Simple Content Access column to org table
+Deface::Override.new(:virtual_path => "taxonomies/index",
+                     :name => "add_sca_column_on_index",
+                     :insert_before => 'table th:last-child', # make it the second-to-last column
+                     :partial => 'overrides/organizations/index_header_override')
+
+# Add Simple Content Access cells to org table
+Deface::Override.new(:virtual_path => "taxonomies/index",
+                    :name => "add_sca_attributes_on_index",
+                    :insert_before => 'tbody td:last-child',
+                    :partial => 'overrides/organizations/index_row_override')

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -78,7 +78,8 @@ module Katello
           joins("INNER JOIN katello_installed_packages ON
             katello_rpms.name = katello_installed_packages.name AND
             katello_rpms.arch = katello_installed_packages.arch AND
-            katello_rpms.evr > katello_installed_packages.evr").
+            katello_rpms.evr > katello_installed_packages.evr AND
+            katello_installed_packages.id in (#{newest_distinct_installed_packages_query})").
           joins("LEFT JOIN katello_module_stream_rpms ON
             katello_rpms.id = katello_module_stream_rpms.rpm_id").
           joins("INNER JOIN katello_host_installed_packages ON
@@ -92,6 +93,16 @@ module Katello
             :enabled_module_streams => enabled_module_stream_ids).pluck(:id).uniq
       end
 
+      def newest_distinct_installed_packages_query
+        "SELECT DISTINCT ON (katello_installed_packages.name) katello_installed_packages.id " \
+          "FROM katello_installed_packages INNER JOIN " \
+          "katello_host_installed_packages ON " \
+          "katello_installed_packages.id = " \
+          "katello_host_installed_packages.installed_package_id " \
+          "WHERE katello_host_installed_packages.host_id = " \
+          "#{content_facet.host.id} ORDER BY katello_installed_packages.name, katello_installed_packages.evr DESC"
+      end
+
       def applicable_differences
         consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
         content_ids = fetch_content_ids