katello 3.18.0.rc2.1 → 3.18.2.1

data/app/models/katello/host/subscription_facet.rb

@@ -282,6 +282,10 @@ module Katello
         name.gsub('_', '-').chomp('.').downcase
       end
 
+      def unsubscribed_hypervisor?
+        self.hypervisor && !self.candlepin_consumer.entitlements?
+      end
+
       def candlepin_consumer
         @candlepin_consumer ||= Katello::Candlepin::Consumer.new(self.uuid, self.host.organization.label)
       end

data/app/models/katello/ping.rb

@@ -23,25 +23,18 @@ module Katello
         services
       end
 
-      # Calls "status" services in all backend engines.
       def ping(services: nil, capsule_id: nil)
-        services ||= self.services(capsule_id)
-        result = {}
-        services.each { |service| result[service] = {} }
+        ping_services_for_capsule(services, capsule_id)
+      end
 
-        ping_pulp3_without_auth(result[:pulp3], capsule_id) if result.include?(:pulp3)
-        ping_pulp_without_auth(result[:pulp], capsule_id) if result.include?(:pulp)
-        ping_candlepin_without_auth(result[:candlepin]) if result.include?(:candlepin)
+      def ping!(services: nil, capsule_id: nil)
+        result = ping_services_for_capsule(services, capsule_id)
 
-        ping_pulp_with_auth(result[:pulp_auth], result[:pulp][:status]) if result.include?(:pulp_auth)
-        ping_candlepin_with_auth(result[:candlepin_auth]) if result.include?(:candlepin_auth)
-        ping_foreman_tasks(result[:foreman_tasks]) if result.include?(:foreman_tasks)
-        ping_katello_events(result[:katello_events]) if result.include?(:katello_events)
-        ping_candlepin_events(result[:candlepin_events]) if result.include?(:candlepin_events)
+        if result[:status] != OK_RETURN_CODE
+          failed_names = failed_services(result).keys
+          fail("The following services have not been started or are reporting errors: #{failed_names.join(', ')}")
+        end
 
-        # set overall status result code
-        result = {:services => result}
-        result[:status] = result[:services].each_value.any? { |v| v[:status] == FAIL_RETURN_CODE } ? FAIL_RETURN_CODE : OK_RETURN_CODE
         result
       end
 
@@ -221,6 +214,33 @@ module Katello
 
       private
 
+      def failed_services(result)
+        result[:services].reject do |_name, details|
+          details[:status] != OK_RETURN_CODE
+        end
+      end
+
+      def ping_services_for_capsule(services, capsule_id)
+        services ||= self.services(capsule_id)
+        result = {}
+        services.each { |service| result[service] = {} }
+
+        ping_pulp3_without_auth(result[:pulp3], capsule_id) if result.include?(:pulp3)
+        ping_pulp_without_auth(result[:pulp], capsule_id) if result.include?(:pulp)
+        ping_candlepin_without_auth(result[:candlepin]) if result.include?(:candlepin)
+
+        ping_pulp_with_auth(result[:pulp_auth], result[:pulp][:status]) if result.include?(:pulp_auth)
+        ping_candlepin_with_auth(result[:candlepin_auth]) if result.include?(:candlepin_auth)
+        ping_foreman_tasks(result[:foreman_tasks]) if result.include?(:foreman_tasks)
+        ping_katello_events(result[:katello_events]) if result.include?(:katello_events)
+        ping_candlepin_events(result[:candlepin_events]) if result.include?(:candlepin_events)
+
+        # set overall status result code
+        result = {:services => result}
+        result[:status] = result[:services].each_value.any? { |v| v[:status] == FAIL_RETURN_CODE } ? FAIL_RETURN_CODE : OK_RETURN_CODE
+        result
+      end
+
       def fetch_proxy(capsule_id)
         capsule_id ? SmartProxy.unscoped.find(capsule_id) : SmartProxy.pulp_primary
       end

data/app/models/katello/repository.rb

@@ -122,6 +122,8 @@ module Katello
 
     scope :has_url, -> { joins(:root).where.not("#{RootRepository.table_name}.url" => nil) }
     scope :on_demand, -> { joins(:root).where("#{RootRepository.table_name}.download_policy" => ::Runcible::Models::YumImporter::DOWNLOAD_ON_DEMAND) }
+    scope :immediate, -> { joins(:root).where("#{RootRepository.table_name}.download_policy" => ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE) }
+    scope :non_immediate, -> { joins(:root).where.not("#{RootRepository.table_name}.download_policy" => ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE) }
     scope :in_default_view, -> { joins(:content_view_version => :content_view).where("#{Katello::ContentView.table_name}.default" => true) }
     scope :in_non_default_view, -> { joins(:content_view_version => :content_view).where("#{Katello::ContentView.table_name}.default" => false) }
     scope :deb_type, -> { with_type(DEB_TYPE) }
@@ -154,6 +156,7 @@ module Katello
     scoped_search :on => :redhat, :complete_value => { :true => true, :false => false }, :ext_method => :search_by_redhat
     scoped_search :on => :container_repository_name, :complete_value => true
     scoped_search :on => :description, :relation => :root, :only_explicit => true
+    scoped_search :on => :download_policy, :relation => :root, :only_explicit => true
     scoped_search :on => :name, :relation => :product, :rename => :product_name
     scoped_search :on => :id, :relation => :product, :rename => :product_id, :only_explicit => true
     scoped_search :on => :label, :relation => :root, :complete_value => true, :only_explicit => true
@@ -287,6 +290,10 @@ module Katello
       root.download_policy == Runcible::Models::YumImporter::DOWNLOAD_ON_DEMAND
     end
 
+    def immediate?
+      root.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE
+    end
+
     def yum_gpg_key_url
       # if the repo has a gpg key return a url to access it
       if self.root.gpg_key.try(:content).present?

data/app/models/katello/subscription_status.rb

@@ -43,9 +43,10 @@ module Katello
 
     def to_status(options = {})
       return UNKNOWN unless host.subscription_facet.try(:uuid)
-      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.hypervisor && !host.subscription_facet.candlepin_consumer.entitlements?
+      return DISABLED if host.organization.simple_content_access?
+      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.unsubscribed_hypervisor?
       status_override ||= options.fetch(:status_override, nil)
-      status = status_override || Katello::Candlepin::Consumer.new(host.subscription_facet.uuid, host.organization.label).entitlement_status
+      status = status_override || host.subscription_facet.candlepin_consumer.entitlement_status
 
       case status
       when Katello::Candlepin::Consumer::ENTITLEMENTS_DISABLED

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -60,17 +60,7 @@ module Katello
       end
 
       def fetch_rpm_content_ids
-        # Query for applicable RPM ids
-        # -> Include all non-modular rpms or rpms that exist within installed module streams
-        enabled_module_stream_ids = ::Katello::ModuleStream.
-          joins("inner join katello_available_module_streams on
-            katello_module_streams.name = katello_available_module_streams.name and
-            katello_module_streams.stream = katello_available_module_streams.stream").
-          joins("inner join katello_host_available_module_streams on
-            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
-          where("katello_host_available_module_streams.host_id = :content_facet_id and
-            katello_host_available_module_streams.status = 'enabled'",
-            :content_facet_id => self.content_facet.host.id).select(:id)
+        enabled_module_stream_ids = fetch_enabled_module_stream_ids
 
         ::Katello::Rpm.
           joins("INNER JOIN katello_repository_rpms ON
@@ -78,7 +68,8 @@ module Katello
           joins("INNER JOIN katello_installed_packages ON
             katello_rpms.name = katello_installed_packages.name AND
             katello_rpms.arch = katello_installed_packages.arch AND
-            katello_rpms.evr > katello_installed_packages.evr").
+            katello_rpms.evr > katello_installed_packages.evr AND
+            katello_installed_packages.id in (#{newest_distinct_installed_packages_query})").
           joins("LEFT JOIN katello_module_stream_rpms ON
             katello_rpms.id = katello_module_stream_rpms.rpm_id").
           joins("INNER JOIN katello_host_installed_packages ON
@@ -87,9 +78,47 @@ module Katello
             :bound_library_repos => self.bound_library_instance_repos).
           where("katello_host_installed_packages.host_id = :content_facet_id",
             :content_facet_id => self.content_facet.host.id).
-          where("katello_module_stream_rpms.module_stream_id is null or
-            katello_module_stream_rpms.module_stream_id in (:enabled_module_streams)",
-            :enabled_module_streams => enabled_module_stream_ids).pluck(:id).uniq
+          where("(katello_module_stream_rpms.module_stream_id IS NULL AND
+            katello_installed_packages.id NOT IN (:locked_modular_installed_packages)) OR
+            (katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)
+            AND katello_installed_packages.id IN (:locked_modular_installed_packages))",
+            :enabled_module_streams => enabled_module_stream_ids,
+            :locked_modular_installed_packages => locked_modular_installed_packages(enabled_module_stream_ids)).pluck(:id).uniq
+      end
+
+      def fetch_enabled_module_stream_ids
+        # Query for applicable RPM ids
+        # -> Include all non-modular rpms or rpms that exist within installed module streams
+        ::Katello::ModuleStream.
+          joins("inner join katello_available_module_streams on
+            katello_module_streams.name = katello_available_module_streams.name and
+            katello_module_streams.stream = katello_available_module_streams.stream").
+          joins("inner join katello_host_available_module_streams on
+            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
+          where("katello_host_available_module_streams.host_id = :content_facet_id and
+            katello_host_available_module_streams.status = 'enabled'",
+            :content_facet_id => self.content_facet.host.id).select(:id)
+      end
+
+      # Installed packages that are locked for the host due to enabled module stream membership
+      def locked_modular_installed_packages(enabled_module_streams)
+        rpms_in_enabled_module_streams = ::Katello::Rpm.
+          joins("INNER JOIN katello_module_stream_rpms ON katello_rpms.id = katello_module_stream_rpms.rpm_id").
+          where("katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)",
+                :enabled_module_streams => enabled_module_streams).select(:nvra, :epoch)
+
+        ::Katello::InstalledPackage.where(nvra: rpms_in_enabled_module_streams.map(&:nvra),
+                                          epoch: rpms_in_enabled_module_streams.map(&:epoch)).select(:id)
+      end
+
+      def newest_distinct_installed_packages_query
+        "SELECT DISTINCT ON (katello_installed_packages.name) katello_installed_packages.id " \
+          "FROM katello_installed_packages INNER JOIN " \
+          "katello_host_installed_packages ON " \
+          "katello_installed_packages.id = " \
+          "katello_host_installed_packages.installed_package_id " \
+          "WHERE katello_host_installed_packages.host_id = " \
+          "#{content_facet.host.id} ORDER BY katello_installed_packages.name, katello_installed_packages.evr DESC"
       end
 
       def applicable_differences

data/app/services/katello/pulp3/api/docker.rb

@@ -32,6 +32,10 @@ module Katello
           PulpContainerClient::RecursiveManage
         end
 
+        def self.tag_image_class
+          PulpContainerClient::TagImage
+        end
+
         def api_client
           PulpContainerClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpContainerClient::Configuration))
         end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -4,13 +4,37 @@ module Katello
       class Export
         include ImportExportCommon
 
-        def initialize(smart_proxy:, content_view_version: nil, destination_server: nil, from_content_view_version: nil)
+        def initialize(smart_proxy:,
+                        content_view_version: nil,
+                        destination_server: nil,
+                        from_content_view_version: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
           @from_content_view_version = from_content_view_version
         end
 
+        def repository_hrefs
+          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
+        end
+
+        def version_hrefs
+          repositories.pluck(:version_href).compact
+        end
+
+        def repositories(fetch_all: false)
+          repos = if @content_view_version.default?
+                    @content_view_version.repositories.yum_type
+                  else
+                    @content_view_version.archived_repos.yum_type
+                  end
+          if fetch_all
+            repos
+          else
+            repos.immediate
+          end
+        end
+
         def generate_exporter_path
           export_path = "#{@content_view_version.content_view}/#{@content_view_version.version}/#{@destination_server}/#{date_dir}".gsub(/\s/, '_')
           "#{@content_view_version.organization.label}/#{export_path}"
@@ -66,12 +90,29 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
+        def validate!(fail_on_missing_content: true, validate_incremental: true)
+          validate_repositories_immediate! if fail_on_missing_content
+          validate_incremental_export! if validate_incremental && !@from_content_view_version.blank?
+        end
+
+        def validate_repositories_immediate!
+          non_immediate_repos = repositories(fetch_all: true).non_immediate
+          if non_immediate_repos.any?
+            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories without the 'immediate' download policy."\
+                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
+                   " and export the generated version. \n %{repos}" %
+                   { content_view: @content_view_version.content_view.name,
+                     current: @content_view_version.version,
+                     repos: self.class.generate_product_repo_strings(repositories: non_immediate_repos)})
+          end
+        end
+
         def validate_incremental_export!
-          return if @from_content_view_version.blank?
           from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
 
-          from_exporter_repos = generate_repo_mapping(from_exporter.repositories)
-          to_exporter_repos = generate_repo_mapping(repositories)
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
+          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
 
           invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
             from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
@@ -99,7 +140,8 @@ module Katello
           ret = { organization: @content_view_version.organization.name,
                   repository_mapping: {},
                   content_view: @content_view_version.content_view.name,
-                  content_view_version: @content_view_version.slice(:major, :minor)
+                  content_view_version: @content_view_version.slice(:major, :minor),
+                  incremental: @from_content_view_version.present?
           }
 
           unless @from_content_view_version.blank?
@@ -120,6 +162,22 @@ module Katello
           end
           ret
         end
+
+        def self.find_library_export_view(create_by_default: false,
+                                          destination_server:,
+                                          organization:)
+          name = "Export-Library"
+          name += "-#{destination_server}" unless destination_server.blank?
+          select_method = create_by_default ? :first_or_create : :first
+          ::Katello::ContentView.where(name: name, organization: organization).send(select_method)
+        end
+
+        def self.generate_product_repo_strings(repositories:)
+          repositories.map do |repo|
+            _("Product: '%{product}', Repository: '%{repository}'" % { product: repo.product.name,
+                                                                       repository: repo.name})
+          end
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -46,6 +46,46 @@ module Katello
         def self.check!(content_view:, metadata:, path:)
           ImportValidator.new(content_view: content_view, metadata: metadata, path: path).check!
         end
+
+        def self.reset_content_view_repositories_from_metadata!(content_view:, metadata:)
+          # Given metadata from the dump and a content view
+          # this method
+          # 1) Fetches ids of the library repos whose product name, repo name amd redhat?
+          # =>  match values provided in the metadata's repository mapping
+          # 2) Removes all the repositories associated to this content view
+          # 3) Adds the repositories matched from the dump
+          # The main intent of this method is to assume that the user intends for the
+          # content view to exaclty look like what is specified in metadata
+
+          repos_in_library = Katello::Repository.
+                    in_default_view.
+                    yum_type.
+                    joins(:product => :provider, :content_view_version => :content_view).
+                    joins(:root).
+                    where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                    pluck("#{::Katello::Repository.table_name}.id",
+                          "#{::Katello::RootRepository.table_name}.name",
+                          "#{::Katello::Product.table_name}.name",
+                          "#{::Katello::Provider.table_name}.provider_type"
+                          )
+          repos_in_library_map = {}
+          # repos_in_library_map is going to look like {['repo1', 'product1', false] => 100, ['repo1', 'product1', true] => 200 }
+          repos_in_library.each do |id, repo, product, provider_type|
+            repos_in_library_map[[repo, product, provider_type == Katello::Provider::REDHAT]] = id
+          end
+
+          repo_ids = metadata[:repository_mapping].values.map do |repo|
+            repos_in_library_map[[repo[:repository], repo[:product], repo[:redhat]]]
+          end
+          content_view.update!(repository_ids: repo_ids)
+        end
+
+        def self.find_or_create_library_import_view(organization)
+          name = ::Katello::ContentView::IMPORT_LIBRARY
+          ::Katello::ContentView.where(name: name,
+                                       organization: organization,
+                                       import_only: true).first_or_create
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -19,22 +19,6 @@ module Katello
           repo_api.read(version_href_to_repository_href(version_href))
         end
 
-        def repository_hrefs
-          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
-        end
-
-        def version_hrefs
-          repositories.pluck(:version_href).compact
-        end
-
-        def repositories
-          if @content_view_version.default?
-            @content_view_version.repositories.yum_type
-          else
-            @content_view_version.archived_repos.yum_type
-          end
-        end
-
         def version_href_to_repository_href(version_href)
           version_href.split("/")[0..-3].join("/") + "/"
         end

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -2,7 +2,6 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class ImportValidator
-        BASEDIR = '/var/lib/pulp'.freeze
         attr_accessor :metadata, :path, :content_view
         def initialize(content_view:, path:, metadata:)
           self.content_view = content_view
@@ -11,10 +10,11 @@ module Katello
         end
 
         def check!
-          check_permissions!
-          ensure_importing_cvv_does_not_exist!
-          ensure_from_cvv_exists!
-          ensure_repositories_metadata_and_content_view_match!
+          unless content_view.default?
+            ensure_importing_cvv_does_not_exist!
+            ensure_from_cvv_exists!
+          end
+          ensure_repositories_metadata_are_in_the_library!
         end
 
         def ensure_importing_cvv_does_not_exist!
@@ -43,55 +43,32 @@ module Katello
           end
         end
 
-        def ensure_repositories_metadata_and_content_view_match!
-          product_repos_in_content_view = content_view.repositories.yum_type.map { |repo| [repo.product.name, repo.name, repo.redhat?] }
+        def ensure_repositories_metadata_are_in_the_library!
+          repos_in_library = Katello::Repository.
+                              in_default_view.
+                              yum_type.
+                              joins(:product => :provider, :content_view_version => :content_view).
+                              joins(:root).
+                              where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                              pluck("#{::Katello::Product.table_name}.name",
+                                    "#{::Katello::RootRepository.table_name}.name",
+                                    "#{::Katello::Provider.table_name}.provider_type"
+                                    )
+
+          # repos_in_library look like [["prod1", "repo1", "Anonymous"], ["prod2", "repo2", "Red Hat"]]
+          product_repos_in_library = repos_in_library.map { |product, repo, provider| [product, repo, provider == ::Katello::Provider::REDHAT] }
           product_repos_in_metadata = metadata[:repository_mapping].values.map { |repo| [repo[:product], repo[:repository], repo[:redhat]] }
-
-          product_repos_in_content_view.sort!
-          product_repos_in_metadata.sort!
-          # product_repos_in_content_view & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
-
-          if product_repos_in_content_view != product_repos_in_metadata
-            repos_in_content_view = generate_product_repo_i18n_string(product_repos_in_content_view)
-            repos_in_import = generate_product_repo_i18n_string(product_repos_in_metadata)
-
-            fail _("Repositories in the importing content view do not match the repositories provided in the import metadata.\n "\
-                    "Repositories in Content View '%{content_view}': %{repos_in_content_view}\n "\
-                    "Repositories in the Import Metadata: %{repos_in_import}" % { content_view: content_view.name,
-                                                                                  repos_in_content_view: repos_in_content_view.join(""),
-                                                                                  repos_in_import: repos_in_import.join("")}
+          # product_repos_in_library & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos_not_in_library = product_repos_in_metadata - product_repos_in_library
+          unless product_repos_not_in_library.blank?
+            repos_in_import = generate_product_repo_i18n_string(product_repos_not_in_library)
+            fail _("The following repositories provided in the import metadata are either not available in the Library or are of incorrect Respository Type. "\
+                    "Please add or enable the repositories before importing\n "\
+                    "%{repos}" % { content_view: content_view.name, repos: repos_in_import.join("")}
                   )
           end
         end
 
-        def check_permissions!
-          fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
-          fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
-          fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
-
-          Dir.glob("#{path}/*").each do |file|
-            fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
-          end
-          toc_path = "#{path}/#{metadata[:toc]}"
-          fail _("The TOC file specified in the metadata does not exist. %s " % toc_path) unless File.exist?(toc_path)
-        end
-
-        def pulp_user_accessible?(path)
-          pulp_info = fetch_pulp_user_info
-          return false if pulp_info.blank?
-
-          stat = File.stat(path)
-          stat.gid.to_s == pulp_info.gid ||
-            stat.uid.to_s == pulp_info.uid ||
-            stat.mode.to_s(8)[-1].to_i >= 4
-        end
-
-        def fetch_pulp_user_info
-          pulp_user = nil
-          Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
-          pulp_user
-        end
-
         def generate_product_repo_i18n_string(product_repos)
           # product_repos look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
           product_repos.map do |product, repo, redhat|