katello 3.9.1 → 3.10.0.rc1

data/app/models/katello/host_available_module_stream.rb

@@ -0,0 +1,47 @@
+module Katello
+  class HostAvailableModuleStream < Katello::Model
+    belongs_to :host, :inverse_of => :host_available_module_streams, :class_name => '::Host::Managed'
+    belongs_to :available_module_stream, :inverse_of => :host_available_module_streams, :class_name => 'Katello::AvailableModuleStream'
+    serialize :installed_profiles
+
+    scope :installed, -> { enabled.where.not(installed_profiles: []) }
+    scope :enabled, -> { where(status: ENABLED) }
+    scope :disabled, -> { where(status: DISABLED) }
+    scope :unknown, -> { where(status: UNKNOWN) }
+
+    ENABLED = "enabled".freeze
+    DISABLED = "disabled".freeze
+    UNKNOWN = "unknown".freeze
+    INSTALLED = "installed".freeze
+    UPGRADABLE = "upgradable".freeze
+
+    STATUS = [ENABLED, DISABLED, UNKNOWN].freeze
+
+    API_STATES = {
+      ENABLED => :enabled,
+      DISABLED => :disabled,
+      UNKNOWN => :unknown,
+      INSTALLED => :installed,
+      UPGRADABLE => :upgradable
+    }.with_indifferent_access
+
+    scoped_search :on => :name, :relation => :available_module_stream, :complete_value => true
+    scoped_search :on => :stream, :relation => :available_module_stream, :complete_value => false
+    scoped_search :on => :status, :complete_value => STATUS
+
+    def upgradable?
+      return false if status != ENABLED
+
+      ApplicableContentHelper.new(ModuleStream).installable_for_hosts([host_id]).
+                                         where(ModuleStream.table_name => {:name => available_module_stream.name,
+                                                                           :stream => available_module_stream.stream}).exists?
+    end
+
+    def self.upgradable(host)
+      upgradable_module_name_streams = ApplicableContentHelper.new(ModuleStream).installable_for_hosts([host]).select(:name, :stream)
+
+      enabled.joins(:available_module_stream).where(:host_id => host).
+              where("(#{AvailableModuleStream.table_name}.name, #{AvailableModuleStream.table_name}.stream) in (#{upgradable_module_name_streams.to_sql})")
+    end
+  end
+end

data/app/models/katello/module_stream.rb

@@ -7,6 +7,12 @@ module Katello
     has_many :repositories, through: :repository_module_streams, class_name: "Katello::Repository"
     has_many :profiles, class_name: "Katello::ModuleProfile", dependent: :destroy, inverse_of: :module_stream
     has_many :artifacts, class_name: "Katello::ModuleStreamArtifact", dependent: :destroy, inverse_of: :module_stream
+    has_many :module_stream_errata_packages, class_name: "Katello::ModuleStreamErratumPackage", dependent: :destroy, inverse_of: :module_stream
+    has_many :erratum_packages, class_name: "Katello::ErratumPackage", :through => :module_stream_errata_packages
+
+    has_many :content_facet_applicable_module_streams, :class_name => "Katello::ContentFacetApplicableModuleStream",
+             :dependent => :destroy, :inverse_of => :module_stream
+    has_many :content_facets, :through => :content_facet_applicable_module_streams, :class_name => "Katello::Host::ContentFacet"
 
     scoped_search on: :name, complete_value: true
     scoped_search on: :uuid, complete_value: true
@@ -36,6 +42,14 @@ module Katello
       RepositoryModuleStream
     end
 
+    def self.content_facet_association_class
+      ContentFacetApplicableModuleStream
+    end
+
+    def self.installable_for_hosts(hosts = nil)
+      ApplicableContentHelper.new(ModuleStream).installable_for_hosts(hosts)
+    end
+
     def update_from_json(json)
       shared_attributes = json.keys & self.class.column_names
       shared_json = json.select { |key, _v| shared_attributes.include?(key) }
@@ -83,6 +97,10 @@ module Katello
       items.join(":")
     end
 
+    def module_spec_hash
+      {:name => name, :stream => stream, :version => version, :context => context, :arch => arch, :id => id}.compact
+    end
+
     def self.parse_module_spec(module_spec)
       # NAME:STREAM:VERSION:CONTEXT:ARCH/PROFILE
       spec = module_spec.split("/").first

data/app/models/katello/module_stream_erratum_package.rb

@@ -0,0 +1,6 @@
+module Katello
+  class ModuleStreamErratumPackage < Katello::Model
+    belongs_to :module_stream, class_name: "Katello::ModuleStream", inverse_of: :module_stream_errata_packages
+    belongs_to :erratum_package, class_name: "Katello::ErratumPackage", inverse_of: :module_stream_errata_packages
+  end
+end

data/app/models/katello/product.rb

@@ -127,8 +127,6 @@ module Katello
       !self.provider.redhat_provider? || self.repositories.present?
     end
 
-    delegate :organization, to: :provider
-
     delegate :library, to: :organization
 
     def plan_name

data/app/models/katello/product_content.rb

@@ -22,6 +22,7 @@ module Katello
     scoped_search :on => :content_type, :relation => :content
     scoped_search :on => :label, :relation => :content
     scoped_search :on => :name, :relation => :product, :rename => :product_name
+    scoped_search :on => :id, :relation => :product, :rename => :product_id, :only_explicit => true
 
     def self.content_table_name
       Katello::Content.table_name

data/app/models/katello/repository.rb

@@ -96,7 +96,10 @@ module Katello
 
     validates_with Validators::ContainerImageNameValidator, :attributes => :container_repository_name, :allow_blank => false, :if => :docker?
     validates :pulp_id, :presence => true, :uniqueness => true, :if => proc { |r| r.name.present? }
-    validates :container_repository_name, :uniqueness => true, :if => :docker?
+    validates :container_repository_name, :if => :docker?, :uniqueness => {message: ->(object, _data) do
+      _("for repository '%{name}' is not unique and cannot be created in '%{env}'. Its Container Repository Name (%{container_name}) conflicts with an existing repository.  Consider changing the Lifecycle Environment's Registry Name Pattern to something more specific.") %
+          {name: object.name, container_name: object.container_repository_name, :env => object.environment.name}
+    end}
 
     before_validation :set_pulp_id
     before_validation :set_container_repository_name, :if => :docker?
@@ -134,12 +137,14 @@ module Katello
     scoped_search :on => :redhat, :complete_value => { :true => true, :false => false }, :ext_method => :search_by_redhat
     scoped_search :on => :container_repository_name, :complete_value => true
     scoped_search :on => :description, :relation => :root, :only_explicit => true
+    scoped_search :on => :name, :relation => :product, :rename => :product_name
+    scoped_search :on => :id, :relation => :product, :rename => :product_id, :only_explicit => true
 
     delegate :product, :redhat?, :custom?, :to => :root
     delegate :yum?, :docker?, :puppet?, :deb?, :file?, :ostree?, :to => :root
     delegate :name, :label, :docker_upstream_name, :url, :to => :root
 
-    delegate :name, :created_at, :updated_at, :major, :minor, :gpg_key_id, :content_id, :arch, :label, :url, :unprotected,
+    delegate :name, :created_at, :updated_at, :major, :minor, :gpg_key_id, :gpg_key, :content_id, :arch, :label, :url, :unprotected,
              :content_type, :product_id, :checksum_type, :docker_upstream_name, :mirror_on_sync, :"mirror_on_sync?",
              :download_policy, :verify_ssl_on_sync, :"verify_ssl_on_sync?", :upstream_username, :upstream_password,
              :ostree_upstream_sync_policy, :ostree_upstream_sync_depth, :deb_releases, :deb_components, :deb_architectures,
@@ -150,6 +155,10 @@ module Katello
       joins(:root).where("#{RootRepository.table_name}.content_type" => content_type)
     end
 
+    def backend_service(smart_proxy)
+      @service ||= Katello::Pulp::Repository.instance_for_type(self, smart_proxy)
+    end
+
     def organization
       if self.environment
         self.environment.organization
@@ -278,6 +287,10 @@ module Katello
       all_instances
     end
 
+    def to_hash(content_source = nil)
+      {id: id, name: label, url: full_path(content_source)}
+    end
+
     #is the repo cloned in the specified environment
     def cloned_in?(env)
       !get_clone(env).nil?
@@ -481,10 +494,6 @@ module Katello
       self.ostree_branches.map(&:name)
     end
 
-    def ostree_capsule_sync_depth
-      -1
-    end
-
     def units_for_removal(ids)
       table_name = removable_unit_association.table_name
       is_integer = Integer(ids.first) rescue false #assume all ids are either integers or not

data/app/models/katello/root_repository.rb

@@ -235,7 +235,7 @@ module Katello
       changeable_attributes = %w(url unprotected checksum_type docker_upstream_name download_policy mirror_on_sync verify_ssl_on_sync
                                  upstream_username upstream_password ostree_upstream_sync_policy ostree_upstream_sync_depth ignore_global_proxy ignorable_content)
       changeable_attributes += %w(name container_repository_name) if docker?
-      changeable_attributes += %w(deb_releases deb_components deb_architectures) if deb?
+      changeable_attributes += %w(deb_releases deb_components deb_architectures gpg_key_id) if deb?
       changeable_attributes.any? { |key| previous_changes.key?(key) }
     end
 

data/app/models/katello/rpm.rb

@@ -31,6 +31,10 @@ module Katello
       RepositoryRpm
     end
 
+    def self.content_facet_association_class
+      ContentFacetApplicableRpm
+    end
+
     def self.scoped_search_version(_key, operator, value)
       self.scoped_search_sortable('version', operator, value)
     end
@@ -222,23 +226,7 @@ module Katello
     end
 
     def self.installable_for_hosts(hosts = nil)
-      hosts = ::Host.where(:id => hosts) if hosts && hosts.is_a?(Array)
-
-      query = Katello::Rpm.joins(:content_facet_applicable_rpms).
-        joins("INNER JOIN #{Katello::ContentFacetRepository.table_name} on \
-        #{Katello::ContentFacetRepository.table_name}.content_facet_id = #{Katello::ContentFacetApplicableRpm.table_name}.content_facet_id").
-        joins("INNER JOIN #{Katello::RepositoryRpm.table_name} AS host_repo_rpm ON \
-          host_repo_rpm.rpm_id = #{Katello::Rpm.table_name}.id AND \
-          #{Katello::ContentFacetRepository.table_name}.repository_id = host_repo_rpm.repository_id")
-
-      if hosts
-        query = query.where("#{Katello::ContentFacetRepository.table_name}.content_facet_id" => hosts.joins(:content_facet)
-                                .select("#{Katello::Host::ContentFacet.table_name}.id"))
-      else
-        query = query.joins(:content_facet_applicable_rpms)
-      end
-
-      query
+      ApplicableContentHelper.new(Rpm).installable_for_hosts(hosts)
     end
 
     def self.applicable_to_hosts(hosts)

data/app/services/katello/applicable_content_helper.rb

@@ -0,0 +1,111 @@
+module Katello
+  class ApplicableContentHelper
+    attr_accessor :content_facet, :content_unit_class
+
+    def initialize(content_unit_class, content_facet = nil)
+      self.content_facet = content_facet
+      self.content_unit_class = content_unit_class
+    end
+
+    def import(partial)
+      to_add, to_remove = applicable_differences(partial)
+      content_facet_association_class.where(:content_facet_id => content_facet.id).delete_all unless partial
+      ActiveRecord::Base.transaction do
+        insert(to_add) unless to_add.blank?
+        remove(to_remove) unless to_remove.blank?
+      end
+    end
+
+    def installable(env = nil, content_view = nil)
+      repos = if env && content_view
+                Katello::Repository.in_environment(env).in_content_views([content_view])
+              else
+                content_facet.bound_repositories.pluck(:id)
+              end
+      content_facet.send(applicable_units).in_repositories(repos)
+    end
+
+    def installable_for_hosts(hosts = nil)
+      # Main goal of this query
+      # 1) Get me the applicable content units for these set of hosts
+      # 2) Now further prune this list. Only include units from repos that have been "enabled" on those hosts.
+      #    In other words, prune the list to only include the units in the "bound" repositories signified by
+      #    the inner join between ContentFacetRepository and Repository<Unit>
+
+      facet_repos = Katello::ContentFacetRepository.joins(:content_facet => :host).select(:repository_id)
+      facet_content_units = content_facet_association_class.joins(:content_facet => :host).select(content_unit_association_id)
+
+      if hosts
+        hosts = ::Host.where(id: hosts) if hosts.is_a?(Array)
+        facet_repos = facet_repos.merge(hosts).reorder(nil)
+        facet_content_units = facet_content_units.merge(hosts).reorder(nil)
+      end
+
+      content_unit_class.joins(repository_association_units).
+                         where(repository_association_class.table_name => {:repository_id => facet_repos,
+                                                                           content_unit_association_id => facet_content_units}).distinct
+    end
+
+    private
+
+    def content_units
+      content_unit_class.name.demodulize.pluralize.underscore.to_sym
+    end
+
+    def applicable_units
+      "applicable_#{content_units}".to_sym
+    end
+
+    def content_unit_association_id
+      "#{content_unit_class.name.demodulize.underscore}_id".to_sym
+    end
+
+    def content_type
+      content_unit_class.const_get(:CONTENT_TYPE)
+    end
+
+    def content_facet_association_class
+      # Example: ContentFacetErratum
+      self.content_unit_class.content_facet_association_class
+    end
+
+    def content_facet_association_units
+      content_facet_association_class.name.demodulize.pluralize.underscore.to_sym
+    end
+
+    def repository_association_class
+      content_unit_class.repository_association_class
+    end
+
+    def repository_association_units
+      repository_association_class.name.demodulize.pluralize.underscore.to_sym
+    end
+
+    def applicable_differences(partial)
+      content_uuids = ::Katello::Pulp::Consumer.new(content_facet.uuid).applicable_ids(content_type)
+      if partial
+        consumer_uuids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.uuid")
+        to_remove = consumer_uuids - content_uuids
+        to_add = content_uuids - consumer_uuids
+      else
+        to_add = content_uuids
+        to_remove = nil
+      end
+      [to_add, to_remove]
+    end
+
+    def insert(uuids)
+      applicable_ids = content_unit_class.where(:uuid => uuids).pluck(:id)
+      unless applicable_ids.empty?
+        inserts = applicable_ids.map { |applicable_id| "(#{applicable_id.to_i}, #{content_facet.id.to_i})" }
+        sql = "INSERT INTO #{content_facet_association_class.table_name} (#{content_unit_association_id}, content_facet_id) VALUES #{inserts.join(', ')}"
+        ActiveRecord::Base.connection.execute(sql)
+      end
+    end
+
+    def remove(uuids)
+      applicable_ids = content_unit_class.where(:uuid => uuids).pluck(:id)
+      content_facet_association_class.where(:content_facet_id => content_facet.id, content_unit_association_id => applicable_ids).delete_all
+    end
+  end
+end

data/app/services/katello/managed_content_medium_provider.rb

@@ -14,6 +14,13 @@ module Katello
       URI.parse(url)
     end
 
+    # If there is an 'AppStream' variant, we need to make it
+    # available to Anaconda
+    def additional_media
+      appstream = entity.operatingsystem.variant_repo(entity, 'AppStream')
+      super + (appstream ? [appstream] : [])
+    end
+
     def unique_id
       @unique_id ||= begin
         "#{kickstart_repo.name.parameterize}-#{kickstart_repo.id}"

data/app/services/katello/pulp/consumer.rb

@@ -20,16 +20,22 @@ module Katello
         Katello.pulp_server.extensions.consumer.upload_profile(self.uuid, 'rpm', profile)
       end
 
-      def applicable_errata_ids
-        response = Katello.pulp_server.extensions.consumer.applicable_errata([self.uuid])
-        return [] if response.empty?
-        response[0]['applicability']['erratum'] || []
+      def upload_module_stream_profile(profile)
+        Katello.pulp_server.extensions.consumer.upload_profile(self.uuid, 'modulemd', profile)
       end
 
-      def applicable_rpm_ids
-        response = Katello.pulp_server.extensions.consumer.applicable_rpms([self.uuid])
+      def applicable_ids(content_unit_type)
+        consumer_method = case content_unit_type
+                          when ::Katello::Erratum::CONTENT_TYPE
+                            :applicable_errata
+                          when ::Katello::ModuleStream::CONTENT_TYPE
+                            :applicable_module_streams
+                          else
+                            :applicable_rpms
+                          end
+        response = Katello.pulp_server.extensions.consumer.send(consumer_method, [self.uuid])
         return [] if response.empty?
-        response[0]['applicability']['rpm'] || []
+        response[0]['applicability'][content_unit_type] || []
       end
 
       def bind_yum_repositories(ids)

data/app/services/katello/pulp/repository.rb

@@ -1,21 +1,174 @@
 module Katello
   module Pulp
     class Repository < ::Actions::Pulp::Abstract
-      attr_accessor :repo
+      attr_accessor :repo, :input, :pulp_api
+      attr_accessor :smart_proxy
+      delegate :root, to: :repo
 
-      def initialize(repo, smart_proxy = nil)
+      def initialize(repo, smart_proxy)
         @repo = repo
         @smart_proxy = smart_proxy
       end
 
+      def backend_data(force = false)
+        return smart_proxy.pulp_api.extensions.repository.retrieve_with_details(repo.pulp_id) if (repo.pulp_id && force)
+        @backend_data ||= smart_proxy.pulp_api.extensions.repository.retrieve_with_details(repo.pulp_id) if repo.pulp_id
+      rescue RestClient::ResourceNotFound
+        nil
+      end
+
+      def self.instance_for_type(repo, smart_proxy)
+        Katello::RepositoryTypeManager.repository_types[repo.root.content_type].service_class.new(repo, smart_proxy)
+      end
+
+      def partial_repo_path
+        fail NotImplementedError
+      end
+
+      def importer_class
+        fail NotImplementedError
+      end
+
+      def master_importer_configuration
+        fail NotImplementedError
+      end
+
+      def mirror_importer_configuration
+        fail NotImplementedError
+      end
+
+      def generate_mirror_importer
+        fail NotImplementedError
+      end
+
+      def generate_master_importer
+        fail NotImplementedError
+      end
+
+      def generate_distributors
+        fail NotImplementedError
+      end
+
       def sync(overrides = {})
         sync_options = {}
         sync_options[:max_speed] = SETTINGS.dig(:katello, :pulp, :sync_KBlimit)
         sync_options[:num_threads] = SETTINGS.dig(:katello, :pulp, :sync_threads)
-        sync_options[:feed] = overrides[:source_url] if overrides[:source_url]
+        sync_options[:feed] = overrides[:feed] if overrides[:feed]
         sync_options[:validate] = !SETTINGS.dig(:katello, :pulp, :skip_checksum_validation)
         sync_options.merge!(overrides[:options]) if overrides[:options]
-        [::Katello::CapsuleContent.new(@smart_proxy).pulp_server.resources.repository.sync(@repo.pulp_id, override_config: sync_options.compact!)]
+        [smart_proxy.pulp_api.resources.repository.sync(@repo.pulp_id, override_config: sync_options.compact!)]
+      end
+
+      def create
+        smart_proxy.pulp_api.extensions.repository.create_with_importer_and_distributors(repo.pulp_id, generate_importer,
+                                                                              generate_distributors, display_name: root.name)
+      end
+
+      def external_url(force_https = false)
+        uri = URI.parse(::SmartProxy.pulp_master.pulp_url)
+        uri.scheme = (root.unprotected && !force_https) ? 'http' : 'https'
+        uri.path = partial_repo_path
+        uri.to_s
+      end
+
+      def generate_importer
+        if smart_proxy.pulp_mirror?
+          generate_mirror_importer
+        elsif repo.in_default_view?
+          generate_master_importer
+        else #content view repositories don't need any importer configuration
+          importer_class.new
+        end
+      end
+
+      def master_importer_connection_options
+        options = {
+          proxy_host: self.proxy_host_importer_value,
+          basic_auth_username: root.upstream_username,
+          basic_auth_password: root.upstream_password,
+          ssl_validation: root.verify_ssl_on_sync?
+        }
+        options.merge(master_importer_ssl_options)
+      end
+
+      def master_importer_ssl_options
+        if root.redhat? && Katello::Resources::CDN::CdnResource.redhat_cdn?(root.url)
+          {
+            ssl_client_cert: root.product.certificate,
+            ssl_client_key: root.product.key,
+            ssl_ca_cert: Katello::Repository.feed_ca_cert(root.url)
+          }
+        elsif root.custom?
+          {
+            ssl_client_cert: root.ssl_client_cert&.content,
+            ssl_client_key: root.ssl_client_key&.content,
+            ssl_ca_cert: root.ssl_ca_cert&.content
+          }
+        else
+          {}
+        end
+      end
+
+      def mirror_importer_connection_options
+        ueber_cert = ::Cert::Certs.ueber_cert(root.organization)
+        {
+          ssl_client_cert: ueber_cert[:cert],
+          ssl_client_key: ueber_cert[:key],
+          ssl_ca_cert: ::Cert::Certs.ca_cert
+        }
+      end
+
+      def proxy_host_importer_value
+        root.ignore_global_proxy ? "" : nil
+      end
+
+      def refresh
+        update_or_associate_importer
+        update_or_associate_distributors
+        remove_unnecessary_distributors
+      end
+
+      def update_or_associate_importer
+        existing_importers = backend_data["importers"]
+        importer = generate_importer
+        found = existing_importers.find { |i| i['importer_type_id'] == importer.id }
+
+        if found
+          ssl_ca_cert = importer.config.delete('ssl_ca_cert')
+          ssl_client_cert = importer.config.delete('ssl_client_cert')
+          ssl_client_key = importer.config.delete('ssl_client_key')
+          importer.config['basic_auth_username'] = nil if importer.config['basic_auth_username'].blank?
+          importer.config['basic_auth_password'] = nil if importer.config['basic_auth_password'].blank?
+          # Update ssl options by themselves workaround for https://pulp.plan.io/issues/2727
+          smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], :ssl_client_cert => ssl_client_cert,
+                                                    :ssl_client_key => ssl_client_key, :ssl_ca_cert => ssl_ca_cert)
+          smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], importer.config)
+        else
+          smart_proxy.pulp_api.resources.repository.associate_importer(repo.pulp_id, repo.importers.first['importer_type_id'], importer.config)
+        end
+      end
+
+      def update_or_associate_distributors
+        existing_distributors = backend_data["distributors"]
+        generate_distributors.each do |distributor|
+          found = existing_distributors.find { |i| i['distributor_type_id'] == distributor.type_id }
+          if found
+            smart_proxy.pulp_api.resources.repository.
+                update_distributor(repo.pulp_id, found['id'], distributor.config)
+          else
+            smart_proxy.pulp_api.resources.repository.
+                associate_distributor(repo.pulp_id, distributor.type_id, distributor.config, :distributor_id => distributor.id)
+          end
+        end
+      end
+
+      def remove_unnecessary_distributors
+        existing_distributors = backend_data["distributors"]
+        generated_distributors = generate_distributors
+        existing_distributors.each do |distributor|
+          found = generated_distributors.find { |dist| dist.type_id == distributor['distributor_type_id'] }
+          smart_proxy.pulp_api.resources.repository.delete_distributor(repo.pulp_id, distributor['id']) unless found
+        end
       end
     end
   end