katello 3.18.0.rc1 → 4.0.0.rc1

data/app/models/katello/subscription_status.rb

@@ -43,9 +43,10 @@ module Katello
 
     def to_status(options = {})
       return UNKNOWN unless host.subscription_facet.try(:uuid)
-      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.hypervisor && !host.subscription_facet.candlepin_consumer.entitlements?
+      return DISABLED if host.organization.simple_content_access?
+      status_override = 'unsubscribed_hypervisor' if host.subscription_facet.unsubscribed_hypervisor?
       status_override ||= options.fetch(:status_override, nil)
-      status = status_override || Katello::Candlepin::Consumer.new(host.subscription_facet.uuid, host.organization.label).entitlement_status
+      status = status_override || host.subscription_facet.candlepin_consumer.entitlement_status
 
       case status
       when Katello::Candlepin::Consumer::ENTITLEMENTS_DISABLED

data/app/presenters/katello/host_subscription_presenter.rb

@@ -1,11 +1,10 @@
 module Katello
   class HostSubscriptionPresenter < SimpleDelegator
-    attr_accessor :quantity_consumed
+    attr_reader :quantity_consumed
 
-    def initialize(entitlement)
-      @subscription = Katello::Pool.find_by(:cp_id => entitlement['pool']['id'])
+    def initialize(pool:, entitlement:)
       @quantity_consumed = entitlement.try(:[], :quantity)
-      super(@subscription)
+      super(pool)
     end
   end
 end

data/app/presenters/katello/host_subscriptions_presenter.rb

@@ -0,0 +1,24 @@
+module Katello
+  class HostSubscriptionsPresenter
+    attr_reader :subscriptions
+
+    def initialize(host)
+      pools = host.subscription_facet&.pools || []
+      @pools = pools.group_by(&:cp_id)
+
+      entitlements = host.subscription_facet.candlepin_consumer.entitlements if @pools.any?
+      entitlements ||= []
+
+      @subscriptions = entitlements.map do |e|
+        HostSubscriptionPresenter.new(pool: pool_for_entitlement(e), entitlement: e)
+      end
+    end
+
+    private
+
+    def pool_for_entitlement(entitlement)
+      pool_cp_id = entitlement['pool']['id']
+      @pools[pool_cp_id]&.first
+    end
+  end
+end

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -60,17 +60,7 @@ module Katello
       end
 
       def fetch_rpm_content_ids
-        # Query for applicable RPM ids
-        # -> Include all non-modular rpms or rpms that exist within installed module streams
-        enabled_module_stream_ids = ::Katello::ModuleStream.
-          joins("inner join katello_available_module_streams on
-            katello_module_streams.name = katello_available_module_streams.name and
-            katello_module_streams.stream = katello_available_module_streams.stream").
-          joins("inner join katello_host_available_module_streams on
-            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
-          where("katello_host_available_module_streams.host_id = :content_facet_id and
-            katello_host_available_module_streams.status = 'enabled'",
-            :content_facet_id => self.content_facet.host.id).select(:id)
+        enabled_module_stream_ids = fetch_enabled_module_stream_ids
 
         ::Katello::Rpm.
           joins("INNER JOIN katello_repository_rpms ON
@@ -78,7 +68,8 @@ module Katello
           joins("INNER JOIN katello_installed_packages ON
             katello_rpms.name = katello_installed_packages.name AND
             katello_rpms.arch = katello_installed_packages.arch AND
-            katello_rpms.evr > katello_installed_packages.evr").
+            katello_rpms.evr > katello_installed_packages.evr AND
+            katello_installed_packages.id in (#{newest_distinct_installed_packages_query})").
           joins("LEFT JOIN katello_module_stream_rpms ON
             katello_rpms.id = katello_module_stream_rpms.rpm_id").
           joins("INNER JOIN katello_host_installed_packages ON
@@ -87,9 +78,47 @@ module Katello
             :bound_library_repos => self.bound_library_instance_repos).
           where("katello_host_installed_packages.host_id = :content_facet_id",
             :content_facet_id => self.content_facet.host.id).
-          where("katello_module_stream_rpms.module_stream_id is null or
-            katello_module_stream_rpms.module_stream_id in (:enabled_module_streams)",
-            :enabled_module_streams => enabled_module_stream_ids).pluck(:id).uniq
+          where("(katello_module_stream_rpms.module_stream_id IS NULL AND
+            katello_installed_packages.id NOT IN (:locked_modular_installed_packages)) OR
+            (katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)
+            AND katello_installed_packages.id IN (:locked_modular_installed_packages))",
+            :enabled_module_streams => enabled_module_stream_ids,
+            :locked_modular_installed_packages => locked_modular_installed_packages(enabled_module_stream_ids)).pluck(:id).uniq
+      end
+
+      def fetch_enabled_module_stream_ids
+        # Query for applicable RPM ids
+        # -> Include all non-modular rpms or rpms that exist within installed module streams
+        ::Katello::ModuleStream.
+          joins("inner join katello_available_module_streams on
+            katello_module_streams.name = katello_available_module_streams.name and
+            katello_module_streams.stream = katello_available_module_streams.stream").
+          joins("inner join katello_host_available_module_streams on
+            katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
+          where("katello_host_available_module_streams.host_id = :content_facet_id and
+            katello_host_available_module_streams.status = 'enabled'",
+            :content_facet_id => self.content_facet.host.id).select(:id)
+      end
+
+      # Installed packages that are locked for the host due to enabled module stream membership
+      def locked_modular_installed_packages(enabled_module_streams)
+        rpms_in_enabled_module_streams = ::Katello::Rpm.
+          joins("INNER JOIN katello_module_stream_rpms ON katello_rpms.id = katello_module_stream_rpms.rpm_id").
+          where("katello_module_stream_rpms.module_stream_id IN (:enabled_module_streams)",
+                :enabled_module_streams => enabled_module_streams).select(:nvra, :epoch)
+
+        ::Katello::InstalledPackage.where(nvra: rpms_in_enabled_module_streams.map(&:nvra),
+                                          epoch: rpms_in_enabled_module_streams.map(&:epoch)).select(:id)
+      end
+
+      def newest_distinct_installed_packages_query
+        "SELECT DISTINCT ON (katello_installed_packages.name) katello_installed_packages.id " \
+          "FROM katello_installed_packages INNER JOIN " \
+          "katello_host_installed_packages ON " \
+          "katello_installed_packages.id = " \
+          "katello_host_installed_packages.installed_package_id " \
+          "WHERE katello_host_installed_packages.host_id = " \
+          "#{content_facet.host.id} ORDER BY katello_installed_packages.name, katello_installed_packages.evr DESC"
       end
 
       def applicable_differences

data/app/services/katello/candlepin_event_listener.rb

@@ -1,7 +1,5 @@
 module Katello
   class CandlepinEventListener
-    STATUS_CACHE_KEY = 'candlepin_events_status'.freeze
-
     Event = Struct.new(:subject, :content)
 
     cattr_accessor :client_factory
@@ -14,22 +12,20 @@ module Katello
     end
 
     def self.running?
-      @running == true && @client&.running?
+      @client&.running? || false
     end
 
     def self.close
-      return unless running?
-
-      logger.info("Closing candlepin event listener")
-      @client&.close
+      if @client&.close
+        logger.info("Closed candlepin event listener")
+      end
       reset
     end
 
     def self.reset
       @processed_count = 0
       @failed_count = 0
-      @running = false
-      Rails.cache.delete(STATUS_CACHE_KEY)
+      @client = nil
     end
 
     def self.run
@@ -37,18 +33,14 @@ module Katello
       @client.subscribe do |message|
         handle_message(message)
       end
-
-      @running = true
     end
 
-    def self.status(refresh: true)
-      Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
-        {
-          processed_count: @processed_count,
-          failed_count: @failed_count,
-          running: running?
-        }
-      end
+    def self.status
+      {
+        processed_count: @processed_count,
+        failed_count: @failed_count,
+        running: running?
+      }
     end
 
     def self.handle_message(message)

data/app/services/katello/event_monitor/poller_thread.rb

@@ -2,7 +2,6 @@ module Katello
   module EventMonitor
     class PollerThread
       SLEEP_INTERVAL = 3
-      STATUS_CACHE_KEY = 'katello_events_status'.freeze
 
       cattr_accessor :instance
 
@@ -15,7 +14,6 @@ module Katello
           self.instance.close
           self.instance = nil
         end
-        reset_status
       end
 
       def self.run
@@ -24,14 +22,8 @@ module Katello
         instance.poll_for_events
       end
 
-      def self.status(refresh: true)
-        Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
-          instance&.status
-        end
-      end
-
-      def self.reset_status
-        Rails.cache.delete(STATUS_CACHE_KEY)
+      def self.status
+        instance&.status
       end
 
       def initialize(logger = nil)
@@ -85,7 +77,6 @@ module Katello
       end
 
       def poll_for_events
-        @thread&.kill
         @thread = Thread.new do
           @logger.info("Polling Katello Event Queue")
           loop do

data/app/services/katello/pulp/smart_proxy_repository.rb

@@ -42,21 +42,6 @@ module Katello
         puppet_repos.where(:pulp_id => pulp_repos.map { |pulp_repo| pulp_repo['id'] })
       end
 
-      def current_repositories_data(environment = nil, content_view = nil)
-        @pulp_repositories ||= smart_proxy.pulp_repositories
-
-        repos = Katello::Repository
-        repos = repos.in_environment(environment) if environment
-        repos = repos.in_content_views([content_view]) if content_view
-        puppet_envs = Katello::ContentViewPuppetEnvironment
-        puppet_envs = puppet_envs.in_environment(environment) if environment
-        puppet_envs = puppet_envs.in_content_view(content_view) if content_view
-
-        repo_ids = repos.pluck(:pulp_id) + puppet_envs.pluck(:pulp_id)
-
-        @pulp_repositories.select { |r| repo_ids.include?(r['id']) }
-      end
-
       def orphaned_repos
         @smart_proxy.pulp_repositories.map { |x| x["id"] } - repos_available_to_capsule.map { |x| x.pulp_id }
       end

data/app/services/katello/pulp3/api/docker.rb

@@ -32,6 +32,10 @@ module Katello
           PulpContainerClient::RecursiveManage
         end
 
+        def self.tag_image_class
+          PulpContainerClient::TagImage
+        end
+
         def api_client
           PulpContainerClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpContainerClient::Configuration))
         end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -4,10 +4,35 @@ module Katello
       class Export
         include ImportExportCommon
 
-        def initialize(smart_proxy:, content_view_version: nil, destination_server: nil)
+        def initialize(smart_proxy:,
+                        content_view_version: nil,
+                        destination_server: nil,
+                        from_content_view_version: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
+          @from_content_view_version = from_content_view_version
+        end
+
+        def repository_hrefs
+          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
+        end
+
+        def version_hrefs
+          repositories.pluck(:version_href).compact
+        end
+
+        def repositories(fetch_all: false)
+          repos = if @content_view_version.default?
+                    @content_view_version.repositories.yum_type
+                  else
+                    @content_view_version.archived_repos.yum_type
+                  end
+          if fetch_all
+            repos
+          else
+            repos.immediate
+          end
         end
 
         def generate_exporter_path
@@ -25,9 +50,32 @@ module Katello
                                   repositories: repository_hrefs)
         end
 
-        def create_export(exporter_href, chunk_size = nil)
+        def create_export(exporter_href, chunk_size: nil)
           options = { versions: version_hrefs }
           options[:chunk_size] = "#{chunk_size}MB" if chunk_size
+          if @from_content_view_version
+            from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
+            start_versions = from_exporter.version_hrefs
+
+            # current_cvv - cvv_from , i.e. repos in current cvv that are not in from
+            # implying something got added to the current cvv
+            # make sure you set the start_versions as 0
+            added_repo_hrefs = repository_hrefs - from_exporter.repository_hrefs
+            added_repo_hrefs.each do |added_repo_href|
+              start_versions << zero_version_href(added_repo_href)
+            end
+
+            # cvv_from - current_cvv , i.e. repos in from cvv that are not in current
+            # implying something got removed the current cvv
+            # make sure the start_versions doesn't contain those
+            deleted_repo_hrefs = from_exporter.repository_hrefs - repository_hrefs
+            start_versions.select! do |href|
+              !deleted_repo_hrefs.include?(version_href_to_repository_href(href))
+            end
+
+            options[:start_versions] = start_versions
+            options[:full] = 'false'
+          end
           [api.export_api.create(exporter_href, options)]
         end
 
@@ -42,15 +90,67 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
+        def validate!(fail_on_missing_content: true, validate_incremental: true)
+          validate_repositories_immediate! if fail_on_missing_content
+          validate_incremental_export! if validate_incremental && !@from_content_view_version.blank?
+        end
+
+        def validate_repositories_immediate!
+          non_immediate_repos = repositories(fetch_all: true).non_immediate
+          if non_immediate_repos.any?
+            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories without the 'immediate' download policy."\
+                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
+                   " and export the generated version. \n %{repos}" %
+                   { content_view: @content_view_version.content_view.name,
+                     current: @content_view_version.version,
+                     repos: self.class.generate_product_repo_strings(repositories: non_immediate_repos)})
+          end
+        end
+
+        def validate_incremental_export!
+          from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
+
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
+          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
+
+          invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
+            from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
+          end
+
+          if invalid_repos_exist
+            fail _("The exported Content View Version '%{content_view} %{current}' cannot be incrementally updated from version '%{from}'."\
+                   " Please do a full export." % { content_view: @content_view_version.content_view.name,
+                                                   current: @content_view_version.version,
+                                                   from: @from_content_view_version.version})
+          end
+        end
+
+        def generate_repo_mapping(repositories)
+          # return a repo mapping with key being the  library_instance_id and value being the repostiory_href
+          # used by validate_incremental_export
+          repo_map = {}
+          repositories.each do |repo|
+            repo_map[repo.library_instance_id] = version_href_to_repository_href(repo.version_href)
+          end
+          repo_map
+        end
+
         def generate_metadata
           ret = { organization: @content_view_version.organization.name,
                   repository_mapping: {},
                   content_view: @content_view_version.content_view.name,
-                  content_view_version: {
-                    major: @content_view_version.major,
-                    minor: @content_view_version.minor
-                  }
+                  content_view_version: @content_view_version.slice(:major, :minor),
+                  incremental: @from_content_view_version.present?
           }
+
+          unless @from_content_view_version.blank?
+            ret[:from_content_view_version] = {
+              major: @from_content_view_version.major,
+              minor: @from_content_view_version.minor
+            }
+          end
+
           repositories.each do |repo|
             next if repo.version_href.blank?
             pulp3_repo = fetch_repository_info(repo.version_href).name
@@ -62,6 +162,22 @@ module Katello
           end
           ret
         end
+
+        def self.find_library_export_view(create_by_default: false,
+                                          destination_server:,
+                                          organization:)
+          name = "Export-Library"
+          name += "-#{destination_server}" unless destination_server.blank?
+          select_method = create_by_default ? :first_or_create : :first
+          ::Katello::ContentView.where(name: name, organization: organization).send(select_method)
+        end
+
+        def self.generate_product_repo_strings(repositories:)
+          repositories.map do |repo|
+            _("Product: '%{product}', Repository: '%{repository}'" % { product: repo.product.name,
+                                                                       repository: repo.name})
+          end
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -3,7 +3,6 @@ module Katello
     module ContentViewVersion
       class Import
         include ImportExportCommon
-        BASEDIR = '/var/lib/pulp'.freeze
 
         def initialize(smart_proxy:, content_view_version: nil, path: nil, metadata: nil)
           @smart_proxy = smart_proxy
@@ -44,32 +43,48 @@ module Katello
           api.importer_api.delete(importer_href)
         end
 
-        class << self
-          def check_permissions!(path)
-            fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
-            fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
-            fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
-
-            Dir.glob("#{path}/*").each do |file|
-              fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
-            end
-          end
+        def self.check!(content_view:, metadata:, path:)
+          ImportValidator.new(content_view: content_view, metadata: metadata, path: path).check!
+        end
 
-          def pulp_user_accessible?(path)
-            pulp_info = fetch_pulp_user_info
-            return false if pulp_info.blank?
+        def self.reset_content_view_repositories_from_metadata!(content_view:, metadata:)
+          # Given metadata from the dump and a content view
+          # this method
+          # 1) Fetches ids of the library repos whose product name, repo name amd redhat?
+          # =>  match values provided in the metadata's repository mapping
+          # 2) Removes all the repositories associated to this content view
+          # 3) Adds the repositories matched from the dump
+          # The main intent of this method is to assume that the user intends for the
+          # content view to exaclty look like what is specified in metadata
 
-            stat = File.stat(path)
-            stat.gid.to_s == pulp_info.gid ||
-              stat.uid.to_s == pulp_info.uid ||
-              stat.mode.to_s(8)[-1].to_i >= 4
+          repos_in_library = Katello::Repository.
+                    in_default_view.
+                    yum_type.
+                    joins(:product => :provider, :content_view_version => :content_view).
+                    joins(:root).
+                    where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                    pluck("#{::Katello::Repository.table_name}.id",
+                          "#{::Katello::RootRepository.table_name}.name",
+                          "#{::Katello::Product.table_name}.name",
+                          "#{::Katello::Provider.table_name}.provider_type"
+                          )
+          repos_in_library_map = {}
+          # repos_in_library_map is going to look like {['repo1', 'product1', false] => 100, ['repo1', 'product1', true] => 200 }
+          repos_in_library.each do |id, repo, product, provider_type|
+            repos_in_library_map[[repo, product, provider_type == Katello::Provider::REDHAT]] = id
           end
 
-          def fetch_pulp_user_info
-            pulp_user = nil
-            Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
-            pulp_user
+          repo_ids = metadata[:repository_mapping].values.map do |repo|
+            repos_in_library_map[[repo[:repository], repo[:product], repo[:redhat]]]
           end
+          content_view.update!(repository_ids: repo_ids)
+        end
+
+        def self.find_or_create_library_import_view(organization)
+          name = ::Katello::ContentView::IMPORT_LIBRARY
+          ::Katello::ContentView.where(name: name,
+                                       organization: organization,
+                                       import_only: true).first_or_create
         end
       end
     end