katello 3.7.1.1 → 3.8.0.rc1

data/app/models/katello/glue/candlepin/repository.rb

@@ -34,7 +34,7 @@ module Katello
       def yum_gpg_key_url
         # if the repo has a gpg key return a url to access it
         if (gpg_key && gpg_key.content.present?)
-          host = Facter.value(:fqdn) || SETTINGS[:fqdn]
+          host = SETTINGS[:fqdn]
           gpg_key_content_api_repository_url(self, :host => host + "/katello", :protocol => 'https')
         end
       end

data/app/models/katello/glue/pulp/repos.rb

@@ -211,6 +211,7 @@ module Katello
                        :arch => repo_param[:arch],
                        :name => repo_param[:name],
                        :label => repo_param[:label],
+                       :description => repo_param[:description],
                        :url => repo_param[:url],
                        :gpg_key => repo_param[:gpg_key],
                        :ssl_ca_cert => repo_param[:ssl_ca_cert],

data/app/models/katello/host/content_facet.rb

@@ -83,14 +83,13 @@ module Katello
       end
 
       def update_applicability_counts
-        self.assign_attributes(
+        self.update_attributes!(
             :installable_security_errata_count => self.installable_errata.security.count,
             :installable_bugfix_errata_count => self.installable_errata.bugfix.count,
             :installable_enhancement_errata_count => self.installable_errata.enhancement.count,
             :applicable_rpm_count => self.content_facet_applicable_rpms.count,
             :upgradable_rpm_count => self.installable_rpms.count
         )
-        self.save!(:validate => false)
       end
 
       def import_rpm_applicability(partial)

data/app/models/katello/kt_environment.rb

@@ -221,6 +221,12 @@ module Katello
       self.repositories.readable.where(:content_type => Katello::Repository::PUPPET_TYPE)
     end
 
+    def update_container_repositories(registry_unauthenticated_pull)
+      self.repositories.readable.where(:content_type => Katello::Repository::DOCKER_TYPE).each do |repository|
+        repository.update_attributes!(registry_unauthenticated_pull: registry_unauthenticated_pull)
+      end
+    end
+
     def as_json(_options = {})
       to_ret = self.attributes
       to_ret['prior'] = self.prior && self.prior.name

data/app/models/katello/product_content.rb

@@ -12,6 +12,7 @@ module Katello
 
     scope :displayable, -> {
       joins(:content).where.not("#{content_table_name}.content_type IN (?)", Katello::Repository.undisplayable_types)
+      .order("LOWER(#{content_table_name}.name) ASC")
     }
 
     scope :redhat, -> {
@@ -31,10 +32,6 @@ module Katello
       joins(:content).where("#{self.content_table_name}.cp_content_id" => Katello::Repository.in_organization(organization).select(:content_id))
     end
 
-    def self.with_valid_subscription(organization)
-      where(:product_id => Katello::PoolProduct.where(:pool_id => organization.pools).select(:product_id))
-    end
-
     # used by Katello::Api::V2::RepositorySetsController#index
     def repositories
       product.repositories.in_default_view.has_url.where(:content_id => content.cp_content_id)

data/app/models/katello/repository.rb

@@ -175,6 +175,8 @@ module Katello
     scoped_search :on => :distribution_uuid, :complete_value => true
     scoped_search :on => :ignore_global_proxy, :complete_value => true
     scoped_search :on => :redhat, :complete_value => { :true => true, :false => false }, :ext_method => :search_by_redhat
+    scoped_search :on => :container_repository_name, :complete_value => true
+    scoped_search :on => :description, :only_explicit => true
 
     def organization
       if self.environment

data/app/models/katello/rpm.rb

@@ -16,7 +16,7 @@ module Katello
     scoped_search :on => :release, :complete_value => true, :ext_method => :scoped_search_release
     scoped_search :on => :arch, :complete_value => true
     scoped_search :on => :epoch, :complete_value => true
-    scoped_search :on => :evr, :ext_method => :scoped_search_evr, :only_explicit => true
+    scoped_search :on => :evr, :ext_method => :scoped_search_evr
     scoped_search :on => :filename, :complete_value => true
     scoped_search :on => :sourcerpm, :complete_value => true
     scoped_search :on => :checksum
@@ -222,28 +222,132 @@ module Katello
     end
 
     def self.installable_for_hosts(hosts = nil)
-      hosts = ::Host.where(:id => hosts) if hosts && hosts.is_a?(Array)
-
-      query = Katello::Rpm.joins(:content_facet_applicable_rpms).
+      query = Katello::Rpm.joins(:content_facet_applicable_rpms).joins(:repository_rpms).
         joins("INNER JOIN #{Katello::ContentFacetRepository.table_name} on \
         #{Katello::ContentFacetRepository.table_name}.content_facet_id = #{Katello::ContentFacetApplicableRpm.table_name}.content_facet_id").
         joins("INNER JOIN #{Katello::RepositoryRpm.table_name} AS host_repo_rpm ON \
-          host_repo_rpm.rpm_id = #{Katello::Rpm.table_name}.id AND \
-          #{Katello::ContentFacetRepository.table_name}.repository_id = host_repo_rpm.repository_id")
+          host_repo_rpm.rpm_id = #{Katello::Rpm.table_name}.id").
+        where("#{Katello::ContentFacetRepository.table_name}.repository_id = host_repo_rpm.repository_id")
 
-      if hosts
-        query = query.where("#{Katello::ContentFacetRepository.table_name}.content_facet_id" => hosts.joins(:content_facet)
-                                .select("#{Katello::Host::ContentFacet.table_name}.id"))
-      else
-        query = query.joins(:content_facet_applicable_rpms)
-      end
-
-      query
+      query = query.joins(:content_facets).where("#{Katello::Host::ContentFacet.table_name}.host_id" => hosts.map(&:id)) if hosts
+      query.distinct
     end
 
     def self.applicable_to_hosts(hosts)
       self.joins(:content_facets).
         where("#{Katello::Host::ContentFacet.table_name}.host_id" => hosts).distinct
     end
+
+    def self.latest(relation)
+      # There are several different ways to implement this:
+
+      # 1) Use a LEFT OUTER JOIN to join the query with itself on name=name,
+      #    arch=arch, evr<evr, then select unmatched rows.
+      #    In PostgreSQL 9.2 populated with real data, EXPLAIN shows
+      #    cost=339901.07..381834.02 for the SQL generated by this when a
+      #    package search is performed with a content_view_version_id filter.
+      #    Unfortunately, this requires writing raw SQL for the JOIN and using
+      #    to_sql, which may be somewhat brittle.  For example, be aware of
+      #    quirks like https://github.com/rails/rails/issues/18379
+      return relation.joins(
+        "LEFT OUTER JOIN (#{relation.to_sql}) AS katello_rpms2 ON " \
+        'katello_rpms.name = katello_rpms2.name AND katello_rpms.arch = katello_rpms2.arch AND ' \
+        '(CAST(katello_rpms.epoch AS INT) < CAST(katello_rpms2.epoch AS INT) OR ' \
+        ' (CAST(katello_rpms.epoch AS INT) = CAST(katello_rpms2.epoch AS INT) AND ' \
+        '  (katello_rpms.version_sortable COLLATE "C" < katello_rpms2.version_sortable COLLATE "C" OR ' \
+        '   (katello_rpms.version_sortable = katello_rpms2.version_sortable AND ' \
+        '    katello_rpms.release_sortable COLLATE "C" < katello_rpms2.release_sortable COLLATE "C"))))'
+      ).where('katello_rpms2.release_sortable IS NULL')
+
+      # 2) Use GROUP BY with MAX() and several INNER JOINs to identify the
+      #    latest evr, then INNER JOIN again to get complete rows.
+      #    Like #1, this must also be written using raw SQL and to_sql.
+      #    cost=677566.03..704889.89
+      #max_epochs = relation.
+      #  group(:name, :arch).
+      #  select(:name, :arch, 'MAX(CAST(katello_rpms.epoch AS INT)) AS max_epoch')
+      #max_versions = relation.
+      #  joins(
+      #    "INNER JOIN (#{max_epochs.to_sql}) katello_rpms2 ON " \
+      #    'katello_rpms.name = katello_rpms2.name AND katello_rpms.arch = katello_rpms2.arch AND ' \
+      #    'CAST(katello_rpms.epoch AS INT) = katello_rpms2.max_epoch'
+      #  ).
+      #  group(:name, :arch, :epoch).
+      #  select(:name, :arch, :epoch, 'MAX(katello_rpms.version_sortable COLLATE "C") AS max_version')
+      #max_releases = relation.
+      #  joins(
+      #    "INNER JOIN (#{max_versions.to_sql}) katello_rpms2 ON " \
+      #    'katello_rpms.name = katello_rpms2.name AND katello_rpms.arch = katello_rpms2.arch AND ' \
+      #    'CAST(katello_rpms.epoch AS INT) = CAST(katello_rpms2.epoch AS INT) AND ' \
+      #    'katello_rpms.version_sortable = katello_rpms2.max_version'
+      #  ).
+      #  group(:name, :arch, :epoch, :version_sortable).
+      #  select(:name, :arch, :epoch, :version_sortable, 'MAX(katello_rpms.release_sortable COLLATE "C") AS max_release')
+      #return relation.
+      #  joins(
+      #    "INNER JOIN (#{max_releases.to_sql}) katello_rpms2 ON " \
+      #    'katello_rpms.name = katello_rpms2.name AND katello_rpms.arch = katello_rpms2.arch AND ' \
+      #    'CAST(katello_rpms.epoch AS INT) = CAST(katello_rpms2.epoch AS INT) AND ' \
+      #    'katello_rpms.version_sortable = katello_rpms2.version_sortable AND ' \
+      #    'katello_rpms.release_sortable = katello_rpms2.max_release'
+      #  )
+
+      # 3) Like #1, but use a Common Table Expression instead of a
+      #    LEFT OUTER JOIN to join the query with itself.
+      #    This seems to be faster than #1 in some cases and slower in others
+      #    depending on optimizer behavior.
+      #    For the same test case as used for #1, cost=404240.71..405796.40
+      #    However, this is not supported in all versions of all databases
+      #    (e.g. it only works in MySQL >=8.0).
+      #    This can be written using either raw SQL or Arel.
+      #
+      # 3a) Raw SQL
+      #cte_sql = relation.select('katello_rpms.*, CAST(katello_rpms.epoch AS INT) AS epoch_sortable').to_sql
+      #return Rpm.where('katello_rpms.id IN (' \
+      #  "WITH katello_rpms AS (#{cte_sql}) " \
+      #  'SELECT katello_rpms.id from katello_rpms LEFT OUTER JOIN katello_rpms AS katello_rpms2 ON ' \
+      #  'katello_rpms.name = katello_rpms2.name AND katello_rpms.arch = katello_rpms2.arch AND ' \
+      #  '(katello_rpms.epoch_sortable < katello_rpms2.epoch_sortable OR ' \
+      #  ' (katello_rpms.epoch_sortable = katello_rpms2.epoch_sortable AND ' \
+      #  '  (katello_rpms.version_sortable COLLATE "C" < katello_rpms2.version_sortable COLLATE "C" OR ' \
+      #  '   (katello_rpms.version_sortable = katello_rpms2.version_sortable AND ' \
+      #  '    katello_rpms.release_sortable COLLATE "C" < katello_rpms2.release_sortable COLLATE "C")))) ' \
+      #  'WHERE katello_rpms2.release_sortable IS NULL' \
+      #')')
+      #
+      # 3b) Arel
+      #     Unlike #1, #2, and #3a, this does not require raw SQL or to_sql.
+      #     Unfortunately this doesn't work reliably in Rails 4.2 because
+      #     relation.unscoped.where(relation.arel_table[:id].in(relation.arel))
+      #     looses the prepared statement parameter values.  See:
+      #     https://github.com/rails/rails/issues/13686
+      ## Convert current query to Arel for use in a CTE
+      #orig_query = relation.arel
+      ## Add another column to the original query: CAST(epoch AS INT) AS epoch_sortable
+      #orig_table = relation.arel_table
+      #orig_query = orig_query.project(Arel::Nodes::NamedFunction.new('CAST', [orig_table[:epoch].as('INT')], 'epoch_sortable'))
+      ## Create two names that will both refer to the existing query in the CTE
+      #cte1 = Arel::Table.new(:katello_rpms)
+      #cte2 = Arel::Table.new(:katello_rpms2)
+      ## Create a new Arel query and use a CTE to JOIN the original query with
+      ## itself and select the latest evr for each name/arch
+      #new_query = cte1.project(cte1[:id]).with(Arel::Nodes::As.new(cte1, orig_query)).
+      #  join(Arel::Nodes::As.new(cte1, cte2), Arel::Nodes::OuterJoin).on(
+      #    cte1[:name].eq(cte2[:name]).and(cte1[:arch].eq(cte2[:arch])).and(
+      #      cte1[:epoch_sortable].lt(cte2[:epoch_sortable]).or(
+      #        cte1[:epoch_sortable].eq(cte2[:epoch_sortable]).and(
+      #          Arel::Nodes::SqlLiteral.new('katello_rpms.version_sortable COLLATE "C"').lt(Arel::Nodes::SqlLiteral.new('katello_rpms2.version_sortable COLLATE "C"')).or(
+      #            cte1[:version_sortable].eq(cte2[:version_sortable]).and(
+      #              Arel::Nodes::SqlLiteral.new('katello_rpms.release_sortable COLLATE "C"').lt(Arel::Nodes::SqlLiteral.new('katello_rpms2.release_sortable COLLATE "C"'))
+      #            )
+      #          )
+      #        )
+      #      )
+      #    )
+      #  ).
+      #  where(cte2[:release_sortable].eq(nil))
+      ## Wrap the new Arel query in a new ActiveRecord::Relation
+      #return relation.unscoped.where(orig_table[:id].in(new_query))
+    end
   end
 end

data/app/services/katello/puppet_class_importer_extensions.rb

@@ -3,32 +3,29 @@ module Katello
     module PuppetClassImporterExtensions
       extend ActiveSupport::Concern
 
-      included do
-        def update_environment
-          change_types = %w(new obsolete updated)
-          changed = self.changes
+      def update_environment
+        change_types = %w(new obsolete updated)
+        changed = self.changes
 
-          change_types.each do |kind|
-            changed[kind].slice!(@environment) unless changed[kind].empty?
-          end
+        change_types.each do |kind|
+          changed[kind].slice!(@environment) unless changed[kind].empty?
+        end
 
-          #prevent the puppet environment from being deleted, by removing special '_destroy_' String
-          if changed['obsolete'][@environment]
-            changed['obsolete'][@environment] =
-              changed['obsolete'][@environment].select { |klass| klass != '_destroy_' }
-          end
+        #prevent the puppet environment from being deleted, by removing special '_destroy_' String
+        if changed['obsolete'][@environment]
+          changed['obsolete'][@environment] = changed['obsolete'][@environment].select { |klass| klass != '_destroy_' }
+        end
 
-          # PuppetClassImporter expects [kind][env] to be in json format
-          change_types.each do |kind|
-            unless (envs = changed[kind]).empty?
-              envs.keys.sort.each do |env|
-                changed[kind][env] = changed[kind][env].to_json
-              end
+        # PuppetClassImporter expects [kind][env] to be in json format
+        change_types.each do |kind|
+          unless (envs = changed[kind]).empty?
+            envs.keys.sort.each do |env|
+              changed[kind][env] = changed[kind][env].to_json
             end
           end
-
-          self.obsolete_and_new(changed)
         end
+
+        self.obsolete_and_new(changed)
       end
     end
   end

data/app/services/katello/ui_notifications/pulp/proxy_disk_space.rb

@@ -5,9 +5,7 @@ module Katello
         class << self
           def deliver!
             SmartProxy.unscoped.with_content.each do |proxy|
-              status = proxy.statuses[:pulp] || proxy.statuses[:pulpnode]
-              next unless (percentage = status&.storage&.dig('pulp_dir', 'percent'))
-
+              percentage = proxy.statuses[:pulp].storage['pulp_dir']['percent']
               if percentage[0..2].to_i < 90 && notification_already_exists?(proxy)
                 blueprint.notifications.where(subject: proxy).destroy_all
               elsif update_notifications(proxy).empty? && percentage[0..2].to_i > 90

data/app/views/katello/api/v2/common/copy.json.rabl

@@ -0,0 +1,3 @@
+object @resource
+
+extends "katello/api/v2/%s/show" % controller_name

data/app/views/katello/api/v2/environments/show.json.rabl

@@ -4,7 +4,7 @@ extends 'katello/api/v2/common/identifier'
 extends 'katello/api/v2/common/org_reference'
 
 attributes :library
-attributes :registry_name_pattern
+attributes :registry_name_pattern, :registry_unauthenticated_pull
 
 node :prior do |env|
   if env.prior

data/app/views/katello/api/v2/repositories/base.json.rabl

@@ -6,6 +6,7 @@ attributes :content_type, :url, :relative_path, :arch, :content_id
 attributes :pulp_id => :backend_identifier
 attributes :cp_label => :content_label
 attributes :major, :minor
+attributes :container_repository_name
 
 child :product do |_product|
   attributes :id, :cp_id, :name

data/app/views/katello/api/v2/repositories/show.json.rabl

@@ -6,7 +6,8 @@ extends 'katello/api/v2/common/timestamps'
 attributes :content_type
 attributes :docker_upstream_name
 attributes :mirror_on_sync, :verify_ssl_on_sync
-attributes :unprotected, :full_path, :checksum_type, :container_repository_name
+attributes :unprotected, :full_path, :checksum_type
+attributes :container_repository_name
 attributes :download_policy
 attributes :url,
            :relative_path
@@ -25,6 +26,7 @@ attributes :ostree_upstream_sync_policy, :ostree_upstream_sync_depth, :compute_o
 attributes :deb_releases, :deb_components, :deb_architectures
 attributes :ignore_global_proxy
 attributes :ignorable_content
+attributes :description
 
 if @resource.is_a?(Katello::Repository)
   if @resource.distribution_version || @resource.distribution_arch || @resource.distribution_family || @resource.distribution_variant
@@ -71,5 +73,5 @@ if @object && @object.library_instance_id.nil?
 end
 
 child :environment => :environment do |_repo|
-  attribute :id
+  attribute :id, :registry_unauthenticated_pull
 end

data/app/views/overrides/activation_keys/_host_environment_select.html.erb

@@ -35,11 +35,12 @@ end %>
 <% cs_select_name =  using_hostgroups_page? ? 'hostgroup[content_source_id]' : 'host[content_facet_attributes][content_source_id]' %>
 <% cs_select_attr = using_hostgroups_page? ? 'content_source' : 'content_facet.content_source' %>
 
+<% proxies = accessible_content_proxies(@host || @hostgroup) %>
 <%= field(f, cs_select_attr, {:label => _("Content Source")}) do
   if using_hostgroups_page?
-    select_tag cs_select_id, content_source_options(@hostgroup, :selected_host_group => @hostgroup.hostgroup, :include_blank => blank_or_inherit_with_id(f, :content_source)), :data => {"spinner_path" => spinner_path},
+    select_tag cs_select_id, options_from_collection_for_select(proxies, :id, :name, @hostgroup.content_source_id), :data => {"spinner_path" => spinner_path},
                :class => 'form-control',  :name => cs_select_name, include_blank: true
   else
-    select_tag cs_select_id, content_source_options(@host, :selected_host_group => @host.hostgroup, :include_blank => blank_or_inherit_with_id(f, :content_source)), :data => {"spinner_path" => spinner_path}, :class => 'form-control',  :name => cs_select_name, include_blank: true
+    select_tag cs_select_id, options_from_collection_for_select(proxies, :id, :name, (@hostgroup || @host).content_source_id), :data => {"spinner_path" => spinner_path}, :class => 'form-control',  :name => cs_select_name, include_blank: true
   end
 end %>

data/config/katello.yaml

@@ -0,0 +1,89 @@
+:katello:
+  #if this block is not define, all types are enabled
+  #types are defined in lib/katello/repository_types/*.rb
+  :content_types:
+    :yum: true
+    :file: true
+    :deb: true
+    :puppet: true
+    :docker: true
+    :ostree: true
+
+  :use_cp: true # set to true/false if you want to override default
+  :use_pulp: true # set to true/false if you want to override default
+
+  :rest_client_timeout: 30
+  :gpg_strict_validation: false
+
+  :puppet_repo_root: '/etc/puppet/environments/'
+
+  :post_sync_url: "http://localhost:3000/katello/api/v2/repositories/sync_complete?token=katello"
+
+  :redhat_repository_url: https://cdn.redhat.com
+
+  :consumer_cert_rpm: 'katello-ca-consumer-latest.noarch.rpm'
+  :consumer_cert_sh:  'katello-rhsm-consumer'
+
+  # Setup your candlepin environment here
+  :candlepin:
+    # refers to the url of the candlepin
+    # example https://localhost:8443/candlepin
+    :url: https://192.168.42.116:31329/candlepin
+
+    # Specify your oauth key and secret used to authenticate between Katello and Candlepin
+    :oauth_key: katello
+    :oauth_secret: katello
+
+    # refers to the candlepin-ca.crt location that is needed
+    # to connect to candlepin over https.This is not necessary
+    # if the candlepin server is running on the same machine as katello
+    # but required if the server is on a different machine.
+    # In the different machine case the katello host needs to have a
+    # copy of /etc/candlepin/certs/candlepin-ca.crt copied from
+    # the candlepin host and the location of the copied file needs to be
+    # specified here..
+    :ca_cert_file:
+    # :bulk_load_size: 1000
+  # Setup your pulp environment here
+  :pulp:
+    # refers to the url of the pulp
+    # example https://localhost/pulp/api
+    :url: https://192.168.42.116:30133/pulp/api/v2/
+
+    :sync_threads: 4
+    :bulk_load_size: 100
+    # refers to the apache certificate
+    # (typically /etc/pki/tls/certs/localhost.crt) location that is needed
+    # to connect to pulp over https.
+    :ca_cert_file:
+    :default_login: admin
+    :sync_KBlimit:
+    :upload_chunk_size: 1048575 # upload size in bytes to pulp. see SSLRenegBufferSize in apache
+    :skip_checksum_validation: false
+
+  :qpid:
+    :url: amqp:ssl:katello-devel.example.com:5671
+    :subscriptions_queue_address: katello_event_queue
+
+  :cdn_proxy:
+  #    :host: localhost
+  #    :port: 3128
+  #    :user: login
+  #    :password: password
+
+# Logging configuration can be changed by uncommenting the loggers
+# section and the logger configuration desired.
+#
+#  :loggers:
+#    :glue:
+#      :enabled: true
+#    :pulp_rest:
+#      :enabled: true
+#    :cp_rest:
+#      :enabled: true
+#    :cp_proxy:
+#      :enabled: true
+#    :action:
+#      :enabled: true
+#    :manifest_import_logger:
+#      :enabled: true