katello 3.16.0.rc4 → 3.16.1

data/app/lib/actions/pulp3/repository/sync.rb

@@ -10,7 +10,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repo_id])
-          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync
+          output[:pulp_tasks] = repo.backend_service(::SmartProxy.unscoped.find(input[:smart_proxy_id])).sync(input[:options])
         end
 
         def external_task=(tasks)

data/app/lib/actions/pulp3/repository/update_cv_repository_cert_guard.rb

@@ -2,13 +2,17 @@ module Actions
   module Pulp3
     module Repository
       class UpdateCVRepositoryCertGuard < Pulp3::Abstract
-        def plan(repository, _smart_proxy)
+        def plan(repository, smart_proxy)
           root = repository.root
           cv_repositories = root.repositories - [root.library_instance]
           cv_repositories.each do |repo|
-            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, SmartProxy.pulp_master)
+            plan_action(::Actions::Pulp3::Repository::RefreshDistribution, repo, smart_proxy)
           end
         end
+
+        def humanized_name
+          _("Updating repository authentication configuration")
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/upload_file.rb

@@ -25,7 +25,7 @@ module Actions
                 filechunk.flush
                 actual_chunk_size = File.size(filechunk)
                 response = uploads_api.update(upload_href, content_range(offset, offset + actual_chunk_size - 1, total_size), filechunk)
-                offset += actual_chunk_size - 1
+                offset += actual_chunk_size
               ensure
                 filechunk.close
                 filechunk.unlink

data/app/lib/katello/concerns/base_template_scope_extensions.rb

@@ -3,20 +3,6 @@ module Katello
     module BaseTemplateScopeExtensions
       extend ActiveSupport::Concern
 
-      module Overrides
-        def allowed_helpers
-          super + [:errata, :host_subscriptions, :host_applicable_errata_ids, :host_applicable_errata_filtered,
-                   :host_latest_applicable_rpm_version, :load_pools, :load_errata_applications, :host_content_facet,
-                   :host_sla, :host_products, :sub_name, :sub_sku, :registered_through, :last_checkin, :host_collections,
-                   :host_subscriptions_names, :host_subscriptions, :host_products_names, :host_collections_names,
-                   :host_redhat_subscription_names, :registered_at]
-        end
-      end
-
-      included do
-        prepend Overrides
-      end
-
       def errata(id)
         Katello::Erratum.in_repositories(Katello::Repository.readable).with_identifiers(id).map(&:attributes).first.slice!('created_at', 'updated_at')
       end
@@ -168,6 +154,10 @@ module Katello
       end
       # rubocop:enable Metrics/MethodLength
 
+      def sortable_version(version)
+        Util::Package.sortable_version(version)
+      end
+
       private
 
       def host_subscription_facet(host)

data/app/lib/katello/errors.rb

@@ -112,21 +112,7 @@ module Katello
       end
     end
 
-    class Pulp3Error < StandardError
-      def self.from_task(task)
-        if task[:state] == 'canceled'
-          self.new(_("Task canceled"))
-        elsif task[:state] == 'failed'
-          message = if task[:error][:description].blank?
-                      _("Pulp task error")
-                    else
-                      task[:error][:description]
-                    end
-          self.new(message)
-        end
-      end
-    end
-
+    class Pulp3Error < StandardError; end
     class Pulp3MigrationError < StandardError; end
 
     class PulpError < StandardError

data/app/lib/katello/resources/cdn.rb

@@ -18,7 +18,7 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product, :options
+        attr_reader :url, :product, :options, :proxy
 
         def substitutor(logger = nil)
           @logger = logger
@@ -26,6 +26,7 @@ module Katello
         end
 
         def initialize(url, options = {})
+          @proxy = ::HttpProxy.default_global_content_proxy
           @ssl_version = Setting[:cdn_ssl_version]
           if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
             fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
@@ -140,7 +141,7 @@ module Katello
         end
 
         def net_http_class
-          if (proxy = ::HttpProxy.default_global_content_proxy)
+          if self.proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
             Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -59,14 +59,17 @@ module Katello
 
         return resolved if to_resolve.empty?
 
-        futures = to_resolve.map do |path_with_substitution|
-          Concurrent::Promises.future do
-            path_with_substitution.resolve_substitutions(@resource)
+        to_resolve.in_groups_of(8) do |group|
+          futures = group.map do |path_with_substitution|
+            Concurrent::Promises.future do
+              path_with_substitution.resolve_substitutions(@resource)
+            end
           end
-        end
 
-        futures.each do |future|
-          resolved << future.value
+          futures.each do |future|
+            resolved << future.value
+            Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected?
+          end
         end
 
         find_substitutions(resolved.compact.flatten)

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -120,8 +120,7 @@ module Katello
           uri = pulp3_uri!
           config.host = uri.host
           config.scheme = uri.scheme
-          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
-          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+          pulp3_ssl_configuration(config)
           config.debugging = true
           config.logger = ::Foreman::Logging.logger('katello/pulp_rest')
           config.username = self.setting(PULP3_FEATURE, 'username')
@@ -129,6 +128,18 @@ module Katello
         end
       end
 
+      def pulp3_ssl_configuration(config)
+        if Faraday.default_adapter == :excon
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert_filename
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key_filename
+        elsif Faraday.default_adapter == :net_http
+          config.ssl_client_cert = ::Cert::Certs.ssl_client_cert
+          config.ssl_client_key = ::Cert::Certs.ssl_client_key
+        else
+          fail "Unexpected faraday default_adapter #{Faraday.default_adapter}!  Cannot continue, this is likely a bug."
+        end
+      end
+
       def backend_service_type(repository)
         if pulp3_support?(repository)
           Actions::Pulp3::Abstract::BACKEND_SERVICE_TYPE
@@ -142,7 +153,7 @@ module Katello
       end
 
       def pulp3_support?(repository)
-        pulp3_repository_type_support?(repository.content_type)
+        repository ? pulp3_repository_type_support?(repository.try(:content_type)) : false
       end
 
       def pulp2_preferred_for_type?(repository_type)

data/app/models/katello/content_view.rb

@@ -289,9 +289,21 @@ module Katello
         where("#{Katello::ContentViewVersion.table_name}.content_view_id" => self.id)
     end
 
-    def repositories_to_publish
+    def repositories_to_publish(override_components = nil)
       if composite?
-        ids = components.flat_map { |version| version.repositories.archived }.map(&:id)
+        components_to_publish = []
+        components.each do |component|
+          override_component = override_components&.detect do |override_cvv|
+            override_cvv.content_view == component.content_view
+          end
+
+          if override_component
+            components_to_publish << override_component
+          else
+            components_to_publish << component
+          end
+        end
+        ids = components_to_publish.flat_map { |version| version.repositories.archived }.map(&:id)
         Repository.where(:id => ids)
       else
         repositories
@@ -302,11 +314,11 @@ module Katello
       composite? ? repositories_to_publish.pluck(&:id) : repository_ids
     end
 
-    def repositories_to_publish_by_library_instance
+    def repositories_to_publish_by_library_instance(override_components = nil)
       # retrieve the list of repositories in a hash, where the key
       # is the library instance id, and the value is an array
       # of the repositories for that instance.
-      repositories_to_publish.inject({}) do |result, repo|
+      repositories_to_publish(override_components).inject({}) do |result, repo|
         result[repo.library_instance] ||= []
         result[repo.library_instance] << repo
         result
@@ -326,8 +338,8 @@ module Katello
       component_composites.where(latest: true).joins(:composite_content_view).where(self.class.table_name => {auto_publish: true})
     end
 
-    def publish_repositories
-      repositories = composite? ? repositories_to_publish_by_library_instance.values : repositories_to_publish
+    def publish_repositories(override_components = nil)
+      repositories = composite? ? repositories_to_publish_by_library_instance(override_components).values : repositories_to_publish
       repositories.each do |repos|
         if repos.is_a? Array
           yield repos

data/app/models/katello/content_view_erratum_filter.rb

@@ -30,6 +30,7 @@ module Katello
       if filter_by_id?
         errata_ids = erratum_rules.map(&:errata_id)
         errata_pulp_ids = errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
+        errata_pulp_ids += errata_module_stream_pulp_ids_from_errata_ids(errata_ids)
       else
         clauses = []
         clauses << errata_from
@@ -37,7 +38,9 @@ module Katello
         clauses << types_clause
         package_filenames = Erratum.list_filenames_by_clauses(repo, clauses.compact)
         errata_pulp_ids = errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
+        errata_pulp_ids += errata_module_stream_pulp_ids_from_clauses(repo, clauses.compact)
       end
+
       errata_pulp_ids
     end
 
@@ -48,10 +51,20 @@ module Katello
       repo.rpms.where("filename ILIKE ANY ( array[?] )", query_params)
     end
 
+    def errata_module_stream_pulp_ids_from_clauses(repo, clauses)
+      module_streams = Erratum.list_modular_streams_by_clauses(repo, clauses)
+      module_streams.pluck(:pulp_id)
+    end
+
     def errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
       rpms_by_filename(repo, package_filenames).pluck(:pulp_id)
     end
 
+    def errata_module_stream_pulp_ids_from_errata_ids(errata_ids)
+      module_streams = Katello::Erratum.where(:errata_id => errata_ids).map(&:module_streams).compact.flatten
+      module_streams.pluck(:pulp_id)
+    end
+
     def errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
       package_filenames = Katello::ErratumPackage.joins(:erratum).where("#{Erratum.table_name}.errata_id" => errata_ids).pluck(:filename)
       rpms_by_filename(repo, package_filenames).pluck(:pulp_id)

data/app/models/katello/content_view_filter.rb

@@ -182,6 +182,10 @@ module Katello
       fail "setting original_packages not supported for #{self.class.name}"
     end
 
+    def original_module_streams=(_include_original)
+      fail "setting original_module_streams not supported for #{self.class.name}"
+    end
+
     protected
 
     def validate_repos

data/app/models/katello/content_view_module_stream_filter.rb

@@ -6,9 +6,36 @@ module Katello
 
     validates_lengths_from_database
 
-    def generate_clauses(_repo)
-      return if module_stream_rules.blank?
-      module_stream_rules.map(&:module_stream_id)
+    def generate_clauses(repo)
+      rules = module_stream_rules || []
+      ids = rules.map(&:module_stream_id)
+      if self.original_module_streams
+        ids.concat(repo.module_streams_without_errata.map(&:id))
+      end
+      ids
+    end
+
+    def original_module_streams=(value)
+      self[:original_module_streams] = value
+    end
+
+    def content_unit_pulp_ids(repo, dependents = false)
+      content_unit_ids = []
+      module_ids = []
+
+      self.module_stream_rules.each do |rule|
+        module_ids << rule.module_stream_id
+      end
+      if self.original_module_streams
+        module_ids.concat(repo.module_streams_without_errata.map(&:id))
+      end
+      modules_streams = ModuleStream.where(id: module_ids).includes(:rpms)
+      content_unit_ids += modules_streams.pluck(:pulp_id).flatten.uniq
+      if dependents && !modules_streams.empty?
+        rpms = modules_streams.map(&:rpms).flatten
+        content_unit_ids += rpms.pluck(:pulp_id).flatten.uniq
+      end
+      content_unit_ids.uniq
     end
   end
 end

data/app/models/katello/content_view_package_filter.rb

@@ -37,7 +37,7 @@ module Katello
       self.package_rules.each do |rule|
         package_filenames.concat(query_rpms(repo, rule))
       end
-
+      package_filenames.concat(repo.packages_without_errata.map(&:filename)) if self.original_packages
       rpms = Rpm.in_repositories(repo)
       rpms.where(filename: package_filenames).where(:modular => false).pluck(:pulp_id).flatten.uniq
     end

data/app/models/katello/glue/pulp/repo.rb

@@ -90,6 +90,7 @@ module Katello
       def pulp_counts_differ?
         pulp_counts = pulp_repo_facts[:content_unit_counts]
         rpms.count != pulp_counts['rpm'].to_i ||
+          srpms.count != pulp_counts['srpm'].to_i ||
           errata.count != pulp_counts['erratum'].to_i ||
           package_groups.count != pulp_counts['package_group'].to_i ||
           puppet_modules.count != pulp_counts['puppet_module'].to_i ||

data/app/models/katello/host/content_facet.rb

@@ -47,11 +47,13 @@ module Katello
         bindable_paths = bindable_types
         relative_paths = []
 
-        paths.each do |path|
+        # paths == ["/pulp/repos/Default_Organization/Library/custom/Test_product/test2",
+        # "/pulp/repos/Default_Organization/Library/custom/Test_product/My_repo"]
+        paths.each do |absolute_path|
           bindable_paths.each do |supported|
-            relative_path = path.gsub(supported[:matcher], '')
-            relative_paths << relative_path
-            if path.starts_with?(supported[:matcher])
+            relative_path = absolute_path.gsub(supported[:matcher], '') # remove e.g. '/pulp/repos/' from beginning of string
+            relative_paths << relative_path unless relative_path == absolute_path
+            if absolute_path.starts_with?(supported[:matcher])
               supported[:paths] << relative_path
               break
             end
@@ -60,10 +62,12 @@ module Katello
 
         repos = bindable_paths.flat_map do |supported|
           repos = Repository.joins(:root).where(RootRepository.table_name => {content_type: supported[:type]}, relative_path: supported[:paths])
-          relative_paths -= repos.pluck(:relative_path)
+          relative_paths -= repos.pluck(:relative_path) # remove relative paths that match our repos
           repos
         end
 
+        # Any leftover relative paths do not match the repos we've just retrieved from the db,
+        # so we should log warnings about them.
         relative_paths.each do |repo_path|
           Rails.logger.warn("System #{self.host.name} (#{self.host.id}) requested binding to unknown repo #{repo_path}")
         end
@@ -121,6 +125,8 @@ module Katello
         ::Katello::Applicability::ApplicableContentHelper.new(self, ::Katello::Rpm, bound_repos).calculate_and_import
         ::Katello::Applicability::ApplicableContentHelper.new(self, ::Katello::Erratum, bound_repos).calculate_and_import
         ::Katello::Applicability::ApplicableContentHelper.new(self, ::Katello::ModuleStream, bound_repos).calculate_and_import
+        update_applicability_counts
+        self.update_errata_status
       end
 
       def import_applicability(partial = false)

data/app/models/katello/module_stream.rb

@@ -65,7 +65,7 @@ module Katello
     end
 
     def module_spec_hash
-      {:name => name, :stream => stream, :version => version, :context => context, :arch => arch, :id => id}.compact
+      {:name => name, :stream => stream, :version => version, :context => context, :arch => arch, :pulp_id => pulp_id, :id => id}.compact
     end
 
     def self.parse_module_spec(module_spec)

data/app/models/katello/ping.rb

@@ -31,9 +31,7 @@ module Katello
 
         # set overall status result code
         result = {:services => result}
-        result[:services].each_value do |v|
-          result[:status] = v[:status] == OK_RETURN_CODE ? OK_RETURN_CODE : FAIL_RETURN_CODE
-        end
+        result[:status] = result[:services].each_value.any? { |v| v[:status] == FAIL_RETURN_CODE } ? FAIL_RETURN_CODE : OK_RETURN_CODE
         result
       end
 

data/app/models/katello/repository.rb

@@ -133,6 +133,7 @@ module Katello
     scope :in_published_environments, -> { in_content_views(Katello::ContentView.non_default).where.not(:environment_id => nil) }
     scope :order_by_root, ->(attr) { joins(:root).order("#{Katello::RootRepository.table_name}.#{attr}") }
     scope :with_content, ->(content) { joins(Katello::RepositoryTypeManager.find_content_type(content).model_class.repository_association_class.name.demodulize.underscore.pluralize.to_sym).distinct }
+    scope :by_rpm_count, -> { left_joins(:repository_rpms).group(:id).order("count(katello_repository_rpms.id) ASC") } # smallest count first
 
     scoped_search :on => :name, :relation => :root, :complete_value => true
     scoped_search :rename => :product, :on => :name, :relation => :product, :complete_value => true
@@ -469,6 +470,17 @@ module Katello
       end
     end
 
+    def module_streams_without_errata
+      module_stream_errata = Katello::ModuleStreamErratumPackage.joins(:erratum_package => {:erratum => :repository_errata})
+                              .where("#{RepositoryErratum.table_name}.repository_id" => self.id)
+                              .pluck("#{Katello::ModuleStreamErratumPackage.table_name}.module_stream_id")
+      if module_stream_errata.any?
+        self.module_streams.where("#{ModuleStream.table_name}.id NOT in (?)", module_stream_errata)
+      else
+        self.module_streams
+      end
+    end
+
     def self.with_errata(errata)
       joins(:repository_errata).where("#{Katello::RepositoryErratum.table_name}.erratum_id" => errata)
     end
@@ -581,6 +593,10 @@ module Katello
       environment
     end
 
+    def self.smart_proxy_syncable
+      where.not(:environment_id => nil)
+    end
+
     def exist_for_environment?(environment, content_view, attribute = nil)
       if environment.present? && content_view.in_environment?(environment)
         repos = content_view.version(environment).repos(environment)

data/app/models/setting/content.rb

@@ -2,7 +2,7 @@ class Setting::Content < Setting
   #rubocop:disable Metrics/MethodLength
   #rubocop:disable Metrics/AbcSize
 
-  validate :content_default_http_proxy, if: proc { |s| s.name == 'content_default_http_proxy' }
+  validate :content_default_http_proxy, if: proc { |s| s.name == 'content_default_http_proxy' && HttpProxy.table_exists? }
 
   after_save :add_organizations_and_locations_if_global_http_proxy
 
@@ -91,6 +91,8 @@ class Setting::Content < Setting
                false, N_('Restrict Composite Content View promotion')),
       self.set('check_services_before_actions', N_("Whether or not to check the status of backend services such as pulp and candlepin prior to performing some actions."),
                true, N_('Check services before actions')),
+      self.set('foreman_proxy_content_batch_size', N_("How many repositories should be synced concurrently on the capsule.  A smaller number may lead to longer sync times.  A larger number will increase dynflow load."),
+               100, N_('Batch size to sync repositories in.')),
       self.set('foreman_proxy_content_auto_sync', N_("Whether or not to auto sync the Smart Proxies after a Content View promotion."),
                true, N_('Sync Smart Proxies after Content View promotion')),
       self.set('default_download_policy', N_("Default download policy for custom repositories (either 'immediate', 'on_demand', or 'background')"), "immediate",