katello 3.12.3 → 3.13.0.rc1

data/app/models/katello/srpm.rb

@@ -9,6 +9,10 @@ module Katello
 
     before_save lambda { |rpm| rpm.summary = rpm.summary.truncate(255) unless rpm.summary.blank? }
 
+    def self.default_sort
+      order(:name).order(:epoch).order(:version_sortable).order(:release_sortable)
+    end
+
     def self.repository_association_class
       RepositorySrpm
     end

data/app/models/katello/subscription_facet_purpose_addon.rb

@@ -0,0 +1,6 @@
+module Katello
+  class SubscriptionFacetPurposeAddon < Katello::Model
+    belongs_to :subscription_facet, inverse_of: :subscription_facet_purpose_addons, class_name: 'Katello::Host::SubscriptionFacet'
+    belongs_to :purpose_addon, inverse_of: :subscription_facet_purpose_addons, class_name: 'Katello::PurposeAddon'
+  end
+end

data/app/models/katello/yum_metadata_file.rb

@@ -5,12 +5,12 @@ module Katello
     belongs_to :repository, :inverse_of => :yum_metadata_files, :class_name => "Katello::Repository"
     CONTENT_TYPE = "yum_repo_metadata_file".freeze
 
-    def self.import_for_repository(repository, _force = true)
+    def self.import_for_repository(repository)
       ::Katello::YumMetadataFile.where(:repository_id => repository).destroy_all
-      super(repository, true)
+      super(repository)
     end
 
-    def self.manage_repository_association
+    def self.many_repository_associations
       false
     end
 

data/app/models/setting/content.rb

@@ -2,6 +2,8 @@ class Setting::Content < Setting
   #rubocop:disable Metrics/MethodLength
   #rubocop:disable Metrics/AbcSize
 
+  after_save :add_organizations_and_locations_if_global_http_proxy
+
   def self.hashify_parameters(parameters)
     Hash[parameters.map { |p| [p, p] }]
   end
@@ -10,14 +12,26 @@ class Setting::Content < Setting
     return unless super
 
     BLANK_ATTRS.concat %w(register_hostname_fact default_location_subscribed_hosts
-                          default_location_puppet_content)
+                          default_location_puppet_content content_default_http_proxy)
 
     download_policies = proc { hashify_parameters(::Runcible::Models::YumImporter::DOWNLOAD_POLICIES) }
     proxy_download_policies = proc { hashify_parameters(::SmartProxy::DOWNLOAD_POLICIES) }
     dependency_solving_options = proc { hashify_parameters(['conservative', 'greedy']) }
+    http_proxy_select = [{
+      name: _("HTTP Proxies"),
+      class: 'HttpProxy',
+      scope: 'all',
+      value_method: 'name',
+      text_method: 'name_and_url'
+    }]
 
     self.transaction do
       [
+        self.set('content_default_http_proxy', N_("Default HTTP Proxy for syncing content"),
+                        nil, N_('Default http proxy'),
+                        nil,
+                        collection: proc { http_proxy_select }, include_blank: N_("no global default")
+                ),
         self.set('katello_default_provision', N_("Default provisioning template for Operating Systems created from synced content"),
                  'Kickstart default', N_('Default synced OS provisioning template'),
                  nil, :collection => proc { katello_template_setting_values("provision") }
@@ -50,9 +64,7 @@ class Setting::Content < Setting
         self.set('katello_default_ptable', N_("Default partitioning table for new Operating Systems created from synced content"),
                  'Kickstart default', N_('Default synced OS partition table'),
                  nil, :collection => proc { Hash[Template.all.where(:type => "Ptable").map { |tmp| [tmp[:name], tmp[:name]] }] }
-                ),
-        self.set('katello_default_kexec', N_("Default kexec template for new Operating Systems created from synced content"),
-                 'Discovery Red Hat kexec', N_('Default synced OS kexec template'),
+                ), self.set('katello_default_kexec', N_("Default kexec template for new Operating Systems created from synced content"), 'Discovery Red Hat kexec', N_('Default synced OS kexec template'),
                  nil, :collection => proc { katello_template_setting_values("kexec") }
                 ),
         self.set('katello_default_atomic_provision', N_("Default provisioning template for new Atomic Operating Systems created from synced content"),
@@ -75,8 +87,10 @@ class Setting::Content < Setting
                  true, N_('Check services before actions')),
         self.set('foreman_proxy_content_auto_sync', N_("Whether or not to auto sync the Smart Proxies after a Content View promotion."),
                  true, N_('Sync Smart Proxies after Content View promotion')),
-        self.set('default_download_policy', N_("Default download policy for repositories (either 'immediate', 'on_demand', or 'background')"), "on_demand",
-                 N_('Default Repository download policy'), nil, :collection => download_policies),
+        self.set('default_download_policy', N_("Default download policy for custom repositories (either 'immediate', 'on_demand', or 'background')"), "immediate",
+                 N_('Default Custom Repository download policy'), nil, :collection => download_policies),
+        self.set('default_redhat_download_policy', N_("Default download policy for enabled Red Hat repositories (either 'immediate', 'on_demand', or 'background')"), "on_demand",
+                         N_('Default Red Hat Repository download policy'), nil, :collection => download_policies),
         self.set('default_proxy_download_policy', N_("Default download policy for Smart Proxy syncs (either 'inherit', immediate', 'on_demand', or 'background')"), "on_demand",
                  N_('Default Smart Proxy download policy'), nil, :collection => proxy_download_policies),
         self.set('pulp_docker_registry_port', N_("The port used by Pulp Crane to provide Docker Registries"),
@@ -115,7 +129,7 @@ class Setting::Content < Setting
                  false, N_('Content View Dependency Solving Default')),
         self.set('dependency_solving_algorithm',
                  N_("How the logic of solving dependencies in a Content View is managed. Conservative will only add " \
-                 "packages to solve the dependencies if the packaged needed doesn't exist. Greedy will pull in the " \
+                 "packages to solve the dependencies if the package needed doesn't exist. Greedy will pull in the " \
                  "latest package to solve a dependency even if it already does exist in the repository."),
                  'conservative', N_('Content View Dependency Solving Algorithm'), nil,
                  :collection => dependency_solving_options)
@@ -128,6 +142,17 @@ class Setting::Content < Setting
     templates = ProvisioningTemplate.where(:template_kind => TemplateKind.where(:name => name))
     templates.each_with_object({}) { |tmpl, hash| hash[tmpl.name] = tmpl.name }
   end
+
+  def add_organizations_and_locations_if_global_http_proxy
+    if name == 'content_default_http_proxy' && (::HttpProxy.table_exists? rescue(false))
+      proxy = HttpProxy.where(name: value).first
+
+      if proxy
+        proxy.update_attribute(:organizations, Organization.unscoped.all)
+        proxy.update_attribute(:locations, Location.unscoped.all)
+      end
+    end
+  end
 end
 
 # If the database is not migrated yet, the system will not be able to load

data/app/overrides/add_about_page.rb

@@ -1,10 +1,3 @@
-# Add System Status to About page
-Deface::Override.new(:virtual_path => "about/index",
-                     :name => "add_system_status_to_about",
-                     :insert_bottom => '#about',
-                     :partial => 'overrides/about/system_status'
-                    )
-
 # Add Installed Packages to About page
 Deface::Override.new(:virtual_path => "about/index",
                      :name => "add_installed_packages_to_about",

data/app/services/cert/certs.rb

@@ -19,7 +19,7 @@ module Cert
     def self.verify_ueber_cert(organization)
       ueber_cert = OpenSSL::X509::Certificate.new(self.ueber_cert(organization)[:cert])
       cert_store = OpenSSL::X509::Store.new
-      cert_store.add_file Setting[:ssl_ca_file]
+      cert_store.add_file SETTINGS[:katello][:candlepin][:ca_cert_file]
       organization.regenerate_ueber_cert unless cert_store.verify ueber_cert
     end
   end

data/app/services/katello/candlepin/consumer.rb

@@ -152,6 +152,10 @@ module Katello
           'Fedora'
         elsif name =~ /sles/ || name =~ /suse.*enterprise.*/
           'SLES'
+        elsif name =~ /debian/
+          'Debian'
+        elsif name =~ /ubuntu/
+          'Ubuntu'
         end
       end
     end

data/app/services/katello/event_queue.rb

@@ -1,15 +1,29 @@
 module Katello
   class EventQueue
+    MAX_RETRY_AGE = 6.hours
+
     @event_types = {}
 
+    def self.create_instance(event)
+      event_class = ::Katello::EventQueue.event_class(event.event_type)
+
+      event_class.new(event.object_id) do |instance|
+        instance.metadata = event.metadata if event.metadata
+      end
+    end
+
+    def self.runnable_events
+      Katello::Event.where(process_after: nil).or(Katello::Event.where(process_after: Date.new..Time.zone.now))
+    end
+
     def self.clear_events(event_type, object_id, on_or_earlier_than)
       Katello::Event.where(:in_progress => true, :object_id => object_id, :event_type => event_type).where('created_at <= ?', on_or_earlier_than).delete_all
     end
 
     def self.next_event
-      first = Katello::Event.where(:in_progress => false).order(:created_at => 'asc').first
+      first = runnable_events.where(:in_progress => false).order(:created_at => 'asc').first
       return if first.nil?
-      last = ::Katello::Event.where(:in_progress => false, :object_id => first.object_id,
+      last = runnable_events.where(:in_progress => false, :object_id => first.object_id,
                                     :event_type => first.event_type).order(:created_at => 'desc').first
       mark_in_progress(first)
       last
@@ -25,7 +39,23 @@ module Katello
     end
 
     def self.push_event(event_type, id)
-      Event.create!(:event_type => event_type, :object_id => id)
+      attrs = {
+        event_type: event_type,
+        object_id: id
+      }
+
+      yield(attrs) if block_given?
+
+      Event.create!(attrs)
+    end
+
+    def self.reschedule_event(event)
+      return :expired if event.created_at <= MAX_RETRY_AGE.ago
+
+      retry_seconds = event_class(event.event_type).try(:retry_seconds)
+      if retry_seconds
+        Katello::Event.update(event.id, in_progress: false, process_after: Time.zone.now + retry_seconds)
+      end
     end
 
     def self.register_event(event_type, klass)

data/app/services/katello/pulp/content_counts_calculator.rb

@@ -5,8 +5,11 @@ module Katello
         @repos = repos
       end
 
+      # rubocop:disable Metrics/MethodLength
       def calculate
         counts = {
+          :apt_repositories => 0,
+          :deb_packages => 0,
           :yum_repositories => 0,
           :packages => 0,
           :package_groups => 0,
@@ -19,6 +22,9 @@ module Katello
 
         @repos.each do |repo|
           case
+          when repo_type?(repo, 'deb')
+            counts[:apt_repositories] += 1
+            counts[:deb_packages] += get_unit_count(repo, 'deb')
           when repo_type?(repo, 'rpm')
             counts[:yum_repositories] += 1
             counts[:packages] += get_unit_count(repo, 'rpm')

data/app/services/katello/pulp/docker_manifest.rb

@@ -5,8 +5,7 @@ module Katello
 
       def update_model(model)
         model.update_attributes(:schema_version => backend_data[:schema_version],
-                          :digest => backend_data[:digest],
-                          :downloaded => backend_data[:downloaded]
+                          :digest => backend_data[:digest]
          )
       end
     end

data/app/services/katello/pulp/docker_manifest_list.rb

@@ -6,7 +6,6 @@ module Katello
       def update_model(model)
         model.update_attributes(:schema_version => backend_data[:schema_version],
                           :digest => backend_data[:digest],
-                          :downloaded => backend_data[:downloaded],
                           :docker_manifests => ::Katello::DockerManifest.where(:digest => backend_data[:manifests].pluck(:digest))
                          )
       end

data/app/services/katello/pulp/file_unit.rb

@@ -4,7 +4,7 @@ module Katello
       include LazyAccessor
 
       PULP_SELECT_FIELDS = %w(name checksum).freeze
-      PULP_INDEXED_FIELDS = %w(name checksum _id).freeze
+      PULP_INDEXED_FIELDS = %w(name checksum).freeze
       CONTENT_TYPE = "iso".freeze
 
       lazy_accessor :pulp_facts, :initializer => :backend_data

data/app/services/katello/pulp/pulp_content_unit.rb

@@ -22,10 +22,53 @@ module Katello
         unit_handler.find_by_unit_id(uuid)
       end
 
+      def self.model_class
+        Katello::RepositoryTypeManager.model_class(self)
+      end
+
+      def self.unit_identifier
+        "_id"
+      end
+
       def self.unit_handler
         Katello.pulp_server.extensions.send(self.name.demodulize.underscore)
       end
 
+      def self.pulp_units_batch_all(unit_ids = nil, page_size = SETTINGS[:katello][:pulp][:bulk_load_size])
+        fields = self.const_get(:PULP_INDEXED_FIELDS) if self.constants.include?(:PULP_INDEXED_FIELDS)
+        criteria = {:limit => page_size, :skip => 0}
+        criteria[:fields] = fields if fields
+        criteria[:filters] = {'_id' => {'$in' => unit_ids}} if unit_ids
+
+        pulp_units_batch(criteria, page_size) do
+          Katello.pulp_server.resources.unit.search(self::CONTENT_TYPE, criteria, :include_repos => true)
+        end
+      end
+
+      def self.pulp_units_batch_for_repo(repository, options = {})
+        page_size = options.fetch(:page_size, SETTINGS[:katello][:pulp][:bulk_load_size])
+
+        fields = self.const_get(:PULP_INDEXED_FIELDS) if self.constants.include?(:PULP_INDEXED_FIELDS)
+        criteria = {:type_ids => [const_get(:CONTENT_TYPE)], :limit => page_size, :skip => 0}
+        criteria[:fields] = {:unit => fields} if fields
+
+        pulp_units_batch(criteria, page_size) do
+          Katello.pulp_server.resources.repository.unit_search(repository.pulp_id, criteria).pluck(:metadata)
+        end
+      end
+
+      def self.pulp_units_batch(criteria, page_size = SETTINGS[:katello][:pulp][:bulk_load_size], &block)
+        response = {}
+        Enumerator.new do |yielder|
+          loop do
+            break if (response.blank? && criteria[:skip] != 0)
+            response = block.call
+            yielder.yield response
+            criteria[:skip] += page_size
+          end
+        end
+      end
+
       def self.fetch_all
         count = 0
         results = []

data/app/services/katello/pulp/repository.rb

@@ -90,8 +90,11 @@ module Katello
       end
 
       def create
-        smart_proxy.pulp_api.extensions.repository.create_with_importer_and_distributors(repo.pulp_id, generate_importer,
-                                                                              generate_distributors, display_name: root.name)
+        smart_proxy.pulp_api.extensions.repository.create_with_importer_and_distributors(
+          repo.pulp_id,
+          generate_importer,
+          generate_distributors,
+          display_name: root.name)
       end
 
       def delete
@@ -120,12 +123,12 @@ module Katello
 
       def master_importer_connection_options
         options = {
-          proxy_host: self.proxy_host_importer_value,
           basic_auth_username: root.upstream_username,
           basic_auth_password: root.upstream_password,
           ssl_validation: root.verify_ssl_on_sync?
         }
-        options.merge(master_importer_ssl_options)
+        options.merge!(proxy_options)
+        options.merge!(master_importer_ssl_options)
       end
 
       def master_importer_ssl_options
@@ -151,7 +154,8 @@ module Katello
         {
           ssl_client_cert: ueber_cert[:cert],
           ssl_client_key: ueber_cert[:key],
-          ssl_ca_cert: ::Cert::Certs.ca_cert
+          ssl_ca_cert: ::Cert::Certs.ca_cert,
+          ssl_validation: true
         }
       end
 
@@ -246,6 +250,29 @@ module Katello
           content_type.pulp2_service_class
         end
       end
+
+      def proxy_options
+        if repo.root.http_proxy_policy == RootRepository::NO_DEFAULT_HTTP_PROXY
+          return { proxy_host: '' }
+        end
+
+        proxy = repo.root.http_proxy
+        if repo.root.http_proxy_policy == RootRepository::GLOBAL_DEFAULT_HTTP_PROXY
+          proxy = HttpProxy.default_global_content_proxy
+        end
+
+        if proxy
+          uri = URI(proxy.url)
+          proxy_options = {
+            proxy_host: uri.scheme + '://' + uri.host,
+            proxy_port: uri.port,
+            proxy_username: proxy.username,
+            proxy_password: proxy.password
+          }
+          return proxy_options
+        end
+        { proxy_host: '' }
+      end
     end
   end
 end

data/app/services/katello/pulp/repository/file.rb

@@ -21,7 +21,12 @@ module Katello
         end
 
         def generate_distributors
-          [Runcible::Models::IsoDistributor.new(repo.relative_path, repo.unprotected, true, auto_publish: true)]
+          yum_dist_id = repo.pulp_id
+          options = {
+            id: yum_dist_id,
+            auto_publish: true
+          }
+          [Runcible::Models::IsoDistributor.new(repo.relative_path, repo.unprotected, true, options)]
         end
 
         def partial_repo_path

data/app/services/katello/pulp/repository/yum.rb

@@ -89,7 +89,7 @@ module Katello
 
           # always copy modular rpms
           modular_includes = {filters: {unit: { 'filename' => { '$in' => repo.rpms.modular.pluck(:filename) } }}}
-          smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id, modular_includes)
+          tasks << smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id, modular_includes)
 
           [:srpm, :errata, :package_group, :yum_repo_metadata_file, :distribution, :module, :module_default].each do |type|
             tasks << smart_proxy.pulp_api.extensions.send(type).copy(repo.pulp_id, destination_repo.pulp_id)
@@ -125,24 +125,24 @@ module Katello
         end
 
         def purge_empty_contents
-          [purge_empty_errata, purge_empty_package_groups]
+          [purge_partial_errata, purge_empty_package_groups]
         end
 
         def should_purge_empty_contents?
           true
         end
 
-        private
-
-        def purge_empty_errata
+        def purge_partial_errata
           task = nil
-          repo.empty_errata! do |errata_to_delete|
+          repo.remove_partial_errata! do |errata_to_delete|
             task = repo.unassociate_by_filter(::Katello::ContentViewErratumFilter::CONTENT_TYPE,
                                                 "id" => { "$in" => errata_to_delete.map(&:errata_id) })
           end
           task
         end
 
+        private
+
         def purge_empty_package_groups
           rpm_names = repo.rpms.pluck(:name).uniq
 

data/app/services/katello/pulp/rpm.rb

@@ -4,7 +4,7 @@ module Katello
       include LazyAccessor
 
       PULP_SELECT_FIELDS = %w(name epoch version release arch checksumtype checksum).freeze
-      PULP_INDEXED_FIELDS = %w(name version release arch epoch summary sourcerpm checksum filename _id is_modular).freeze
+      PULP_INDEXED_FIELDS = %w(name version release arch epoch summary sourcerpm checksum filename is_modular).freeze
       CONTENT_TYPE = "rpm".freeze
 
       lazy_accessor :description, :license, :buildhost, :vendor, :relativepath, :children, :checksumtype,