katello 3.16.0.rc1.1 → 3.16.0.rc4

data/app/models/katello/rhsm_fact_parser.rb

@@ -61,6 +61,10 @@ module Katello
           os_attributes[:release_name] = release_name
         end
 
+        if facts['distribution.name'] == 'Red Hat Enterprise Linux Workstation'
+          os_attributes[:name] = os_name + '_Workstation'
+        end
+
         ::Operatingsystem.find_by(os_attributes) || ::Operatingsystem.create!(os_attributes)
       end
     end

data/app/models/katello/root_repository.rb

@@ -5,6 +5,9 @@ module Katello
     serialize :docker_tags_whitelist
 
     include Ext::LabelFromName
+    include Encryptable
+
+    encrypts :upstream_password
 
     IGNORABLE_CONTENT_UNIT_TYPES = %w(rpm drpm srpm distribution erratum).freeze
     CHECKSUM_TYPES = %w(sha1 sha256).freeze
@@ -97,7 +100,7 @@ module Katello
     scope :with_no_proxy, -> { where(:http_proxy_policy => RootRepository::NO_DEFAULT_HTTP_PROXY) }
     scope :with_selected_proxy, ->(http_proxy_id) {
       where(:http_proxy_policy => RootRepository::USE_SELECTED_HTTP_PROXY).
-      where("http_proxy_id = ?", http_proxy_id)
+      where(:http_proxy_id => http_proxy_id)
     }
     delegate :redhat?, :provider, :organization, to: :product
 

data/app/models/setting/content.rb

@@ -145,7 +145,9 @@ class Setting::Content < Setting
                   "as long as the registering DMI UUID is not used by another host."),
                true, N_('Host Profile Assume')),
       self.set('host_tasks_workers_pool_size', N_("Amount of workers in the pool to handle the execution of host-related tasks. When set to 0, the default queue will be used instead. Restart of the dynflowd/foreman-tasks service is required."),
-               5, N_('Host Tasks Workers Pool Size'))
+               5, N_('Host Tasks Workers Pool Size')),
+      self.set('applicability_batch_size', N_("Number of host applicability calculations to process per task."),
+               50, N_('Applicability Batch Size'))
     ]
   end
 

data/app/services/katello/applicable_host_queue.rb

@@ -0,0 +1,21 @@
+module Katello
+  class ApplicableHostQueue
+    def self.batch_size
+      ::Setting::Content.find_by(name: "applicability_batch_size").value
+    end
+
+    def self.queue_depth
+      ::Katello::HostQueueElement.all.size
+    end
+
+    def self.push_host(host_id)
+      HostQueueElement.create!({ host_id: host_id })
+    end
+
+    def self.pop_hosts(amount = self.batch_size)
+      queue = HostQueueElement.group(:host_id).select("MIN(created_at) as created_at, host_id").limit(amount)
+      HostQueueElement.where(host_id: queue.map(&:host_id)).delete_all
+      queue
+    end
+  end
+end

data/app/services/katello/candlepin/message_handler.rb

@@ -81,9 +81,8 @@ module Katello
       end
 
       def delete_pool
-        if pool
-          Rails.logger.info "deleting pool #{pool.id} from Katello"
-          pool.destroy!
+        if Katello::Pool.where(:cp_id => pool_id).destroy_all.any?
+          Rails.logger.info "deleted pool #{pool_id} from Katello"
         end
       end
     end

data/app/services/katello/pulp/pulp_content_unit.rb

@@ -34,6 +34,10 @@ module Katello
         nil
       end
 
+      def self.supports_id_fetch?
+        true
+      end
+
       def self.content_type
         self::CONTENT_TYPE
       end

data/app/services/katello/pulp3/api/content_guard.rb

@@ -0,0 +1,45 @@
+require 'pulp_certguard_client'
+
+module Katello
+  module Pulp3
+    module Api
+      class ContentGuard < Core
+        def self.client_module
+          PulpCertguardClient
+        end
+
+        def self.api_exception_class
+          PulpCertguardClient::ApiError
+        end
+
+        def api_client
+          PulpCertguardClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpCertguardClient::Configuration))
+        end
+
+        def rhsm_api_client
+          PulpCertguardClient::ContentguardsRhsmApi.new(api_client)
+        end
+
+        def create(name = "RHSMCertGuard", ca_certificate = Cert::Certs.ca_cert)
+          data = PulpCertguardClient::CertguardRHSMCertGuard.new(name: name, ca_certificate: ca_certificate)
+          rhsm_api_client.create(data)
+        rescue self.class.api_exception_class => e
+          raise e unless list&.results&.first
+        end
+
+        def list(options = {})
+          rhsm_api_client.list options
+        end
+
+        def partial_update(href, ca_certificate = Cert::Certs.ca_cert)
+          data = PulpCertguardClient::CertguardRHSMCertGuard.new(ca_certificate: ca_certificate)
+          rhsm_api_client.partial_update(href, data)
+        end
+
+        def delete(href)
+          rhsm_api_client.delete(href) if href
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/api/yum.rb

@@ -28,6 +28,10 @@ module Katello
           PulpRpmClient::RpmRepositorySyncURL
         end
 
+        def self.rpm_package_group_class
+          PulpRpmClient::RpmPackageGroup
+        end
+
         def api_client
           PulpRpmClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpRpmClient::Configuration))
         end
@@ -51,6 +55,26 @@ module Katello
         def distributions_api
           PulpRpmClient::DistributionsRpmApi.new(api_client)
         end
+
+        def copy_api
+          PulpRpmClient::RpmCopyApi.new(api_client)
+        end
+
+        def content_package_groups_api
+          PulpRpmClient::ContentPackagegroupsApi.new(api_client)
+        end
+
+        def content_package_environments_api
+          PulpRpmClient::ContentPackageenvironmentsApi.new(api_client)
+        end
+
+        def content_repo_metadata_files_api
+          PulpRpmClient::ContentRepoMetadataFilesApi.new(api_client)
+        end
+
+        def content_distribution_trees_api
+          PulpRpmClient::ContentDistributionTreesApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/erratum.rb

@@ -15,6 +15,10 @@ module Katello
         "pulp_href"
       end
 
+      def self.supports_id_fetch?
+        false
+      end
+
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::Yum.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -19,6 +19,10 @@ module Katello
         nil
       end
 
+      def self.supports_id_fetch?
+        true
+      end
+
       def update_model
         fail NotImplementedError
       end

data/app/services/katello/pulp3/repository.rb

@@ -48,6 +48,10 @@ module Katello
         nil
       end
 
+      def skip_types
+        nil
+      end
+
       def content_service
         Katello::Pulp3::Content
       end
@@ -143,7 +147,7 @@ module Katello
 
       def distribution_needs_update?
         if distribution_reference
-          expected = distribution_options(relative_path).except(:name).compact
+          expected = secure_distribution_options(relative_path).except(:name).compact
           actual = get_distribution.to_hash
           expected != actual.slice(*expected.keys)
         elsif repo.environment
@@ -180,13 +184,20 @@ module Katello
         api.repositories_api.list(options).results
       end
 
+      def read
+        api.repositories_api.read(repository_reference.try(:repository_href))
+      end
+
       def delete(href = repository_reference.try(:repository_href))
         repository_reference.try(:destroy)
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
       def sync
-        repository_sync_url_data = api.class.repository_sync_url_class.new(remote: repo.remote_href, mirror: repo.root.mirror_on_sync)
+        sync_url_params = {remote: repo.remote_href, mirror: repo.root.mirror_on_sync}
+        skip_type_param = skip_types
+        sync_url_params[:skip_types] = skip_type_param if skip_type_param
+        repository_sync_url_data = api.class.repository_sync_url_class.new(sync_url_params)
         [api.repositories_api.sync(repository_reference.repository_href, repository_sync_url_data)]
       end
 
@@ -209,7 +220,7 @@ module Katello
       end
 
       def create_distribution(path)
-        distribution_data = api.class.distribution_class.new(distribution_options(path))
+        distribution_data = api.class.distribution_class.new(secure_distribution_options(path))
         api.distributions_api.create(distribution_data)
       end
 
@@ -219,7 +230,8 @@ module Katello
 
       def update_distribution
         if distribution_reference
-          options = distribution_options(relative_path).except(:name)
+          options = secure_distribution_options(relative_path).except(:name)
+          distribution_reference.update(:content_guard_href => options[:content_guard])
           api.distributions_api.partial_update(distribution_reference.href, options)
         end
       end
@@ -246,9 +258,10 @@ module Katello
 
       def save_distribution_references(hrefs)
         hrefs.each do |href|
-          path = api.get_distribution(href)&.base_path
+          pulp3_distribution_data = api.get_distribution(href)
+          path, content_guard_href = pulp3_distribution_data&.base_path, pulp3_distribution_data&.content_guard
           unless distribution_reference
-            DistributionReference.create!(path: path, href: href, repository_id: repo.id)
+            DistributionReference.create!(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href)
           end
         end
       end
@@ -284,6 +297,16 @@ module Katello
         remote_options.merge!(ssl_remote_options)
       end
 
+      def secure_distribution_options(path)
+        secured_distribution_options = {}
+        if root.unprotected
+          secured_distribution_options[:content_guard] = nil
+        else
+          secured_distribution_options[:content_guard] = ::Katello::Pulp3::ContentGuard.first.pulp_href
+        end
+        secured_distribution_options.merge!(distribution_options(path))
+      end
+
       def ssl_remote_options
         if root.redhat? && Katello::Resources::CDN::CdnResource.redhat_cdn?(root.url)
           {

data/app/services/katello/pulp3/repository/yum.rb

@@ -4,6 +4,9 @@ module Katello
   module Pulp3
     class Repository
       class Yum < ::Katello::Pulp3::Repository
+        include Katello::Util::Errata
+        include Katello::Util::PulpcoreContentFilters
+
         def remote_options
           if root.url.blank?
             common_remote_options.merge(url: nil, policy: root.download_policy)
@@ -12,6 +15,14 @@ module Katello
           end
         end
 
+        def skip_types
+          skip_types = []
+          if root.ignorable_content.try(:include?, "srpm")
+            skip_types << "srpm"
+          end
+          skip_types
+        end
+
         def distribution_options(path)
           {
             base_path: path,
@@ -59,9 +70,93 @@ module Katello
           "/pulp/repos/#{repo.relative_path}/".sub('//', '/')
         end
 
-        def copy_content_for_source
-          # TODO
-          fail NotImplementedError
+        def copy_units(source_repository_version_href, content_unit_hrefs, dependency_solving)
+          tasks = []
+
+          if content_unit_hrefs.any?
+            data = PulpRpmClient::Copy.new
+            data.config = [
+              source_repo_version: source_repository_version_href,
+              dest_repo: repository_reference.repository_href,
+              dest_base_version: 0,
+              content: content_unit_hrefs.sort
+            ]
+            data.dependency_solving = dependency_solving
+            tasks << api.copy_api.copy_content(data)
+          else
+            data = PulpRpmClient::RepositoryAddRemoveContent.new(
+              remove_content_units: ['*'])
+            tasks << api.repositories_api.modify(repository_reference.repository_href, data)
+          end
+
+          tasks
+        end
+
+        def packageenvironments(options = {})
+          api.content_package_environments_api.list(options)
+        end
+
+        def metadatafiles(options = {})
+          api.content_repo_metadata_files_api.list(options)
+        end
+
+        def distributiontrees(options = {})
+          api.content_distribution_trees_api.list(options)
+        end
+
+        def copy_content_for_source(source_repository, options = {})
+          filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class|
+            filter_class.where(:id => options[:filter_ids])
+          end
+
+          filters.flatten!.compact!
+
+          whitelist_ids = []
+          blacklist_ids = []
+          filters.each do |filter|
+            if filter.inclusion
+              whitelist_ids += filter.content_unit_pulp_ids(source_repository)
+            else
+              blacklist_ids += filter.content_unit_pulp_ids(source_repository)
+            end
+          end
+
+          if whitelist_ids.empty? && filters.select { |filter| filter.inclusion }.empty?
+            whitelist_ids = source_repository.rpms.where(:modular => false).pluck(:pulp_id).sort
+          end
+
+          content_unit_hrefs = whitelist_ids - blacklist_ids
+
+          if content_unit_hrefs.any?
+            content_unit_hrefs += additional_content_hrefs(source_repository, content_unit_hrefs)
+          end
+          content_unit_hrefs += source_repository.srpms.pluck(:pulp_id)
+
+          dependency_solving = options[:solve_dependencies] || false
+          copy_units(source_repository.version_href, content_unit_hrefs.uniq, dependency_solving)
+        end
+
+        def additional_content_hrefs(source_repository, content_unit_hrefs)
+          repo_service = source_repository.backend_service(SmartProxy.pulp_master)
+          options = { :repository_version => source_repository.version_href }
+
+          errata_to_include = filter_errata_by_pulp_href(source_repository.errata, content_unit_hrefs)
+          content_unit_hrefs += errata_to_include.collect { |erratum| erratum.repository_errata.pluck(:erratum_pulp3_href) }.flatten
+
+          package_groups_to_include = filter_package_groups_by_pulp_href(source_repository.package_groups, content_unit_hrefs)
+          content_unit_hrefs += package_groups_to_include.pluck(:pulp_id)
+
+          package_environment_hrefs_to_include = filter_package_environments_by_pulp_hrefs(
+            repo_service.packageenvironments(options).results, package_groups_to_include.pluck(:pulp_id))
+          content_unit_hrefs += package_environment_hrefs_to_include
+
+          metadata_file_hrefs_to_include = filter_metadatafiles_by_pulp_hrefs(
+            repo_service.metadatafiles(options).results, content_unit_hrefs)
+          content_unit_hrefs += metadata_file_hrefs_to_include
+
+          distribution_tree_hrefs_to_include = filter_distribution_trees_by_pulp_hrefs(
+            repo_service.distributiontrees(options).results, content_unit_hrefs)
+          content_unit_hrefs + distribution_tree_hrefs_to_include
         end
 
         def regenerate_applicability

data/app/services/katello/pulp3/rpm.rb

@@ -119,11 +119,12 @@ module Katello
           each { |field| custom_json[field] = backend_data[field] }
         custom_json['release_sortable'] = Util::Package.sortable_version(backend_data['release'])
         custom_json['version_sortable'] = Util::Package.sortable_version(backend_data['version'])
-        custom_json['nvra'] = model.build_nvra
         custom_json['filename'] = backend_data['location_href']
         custom_json['checksum'] = backend_data['pkgId']
         custom_json['sourcerpm'] = backend_data['rpm_sourcerpm']
-        model.update!(custom_json)
+        model.assign_attributes(custom_json)
+        model.nvra = model.build_nvra
+        model.save!
       end
     end
   end

data/app/services/katello/ui_notifications/subscriptions/manifest_expired_warning.rb

@@ -7,14 +7,11 @@ module Katello
         CDN_PATH = '/content/dist/rhel/server/7/listing'.freeze
 
         def self.deliver!(orgs = ::Organization.all)
+          return if Setting[:content_disconnected]
+
           orgs.each do |org|
-            next unless redhat_connected?(org)
-            content = org.contents.find_by(:label => CONTENT_LABEL)
-            product = content&.products&.find { |p| p.key }
-            if content && product && product.pools.any?
-              if got_403? { product.cdn_resource.get(CDN_PATH) }
-                new(org).deliver!
-              end
+            if cdn_inaccessible?(org) || upstream_inaccessible?(org)
+              new(org).deliver!
             end
           end
         rescue StandardError => e
@@ -47,6 +44,14 @@ module Katello
           @blueprint ||= NotificationBlueprint.find_by(name: 'manifest_expired_warning')
         end
 
+        def self.cdn_inaccessible?(org)
+          content = org.contents.find_by(:label => CONTENT_LABEL)
+          product = content&.products&.find { |p| p.key }
+          if content && product && product.pools.any?
+            return got_403? { product.cdn_resource.get(CDN_PATH) }
+          end
+        end
+
         def self.got_403?
           yield
           false
@@ -54,8 +59,15 @@ module Katello
           true
         end
 
+        def self.upstream_inaccessible?(org)
+          return unless org.manifest_imported?
+
+          checker = Katello::UpstreamConnectionChecker.new(org)
+          !checker.can_connect?
+        end
+
         def self.redhat_connected?(org)
-          org.redhat_provider.repository_url.include?(CDN_HOSTNAME) && !Setting[:content_disconnected]
+          org.redhat_provider.repository_url.include?(CDN_HOSTNAME)
         end
       end
     end