katello 4.7.3 → 4.8.0.rc1

data/app/models/katello/product.rb

@@ -33,6 +33,7 @@ module Katello
 
     validates_lengths_from_database :except => [:label]
     validates :provider_id, :presence => true
+    validate :ensure_provider_type_matches_id
     validates_with Validators::KatelloNameFormatValidator, :attributes => :name
     validates_with Validators::KatelloLabelFormatValidator, :attributes => :label
     validates_with Validators::ProductUniqueAttributeValidator, :attributes => :name
@@ -236,7 +237,7 @@ module Katello
     end
 
     def self.unused_product_id
-      id = SecureRandom.random_number(999_999_999_999)
+      id = (SecureRandom.random_number * (10**12 - 10**11) + 10**11).to_i # guarantee that we get 12 digits
       if ::Katello::Product.find_by(:cp_id => id)
         unused_product_id
       else
@@ -253,5 +254,14 @@ module Katello
     class Jail < ::Safemode::Jail
       allow :name, :label
     end
+
+    private
+
+    def ensure_provider_type_matches_id
+      return if errors[:provider_id].any?
+      if provider.redhat_provider? && ::Katello::Glue::Candlepin::Product.custom_product_id?(cp_id)
+        errors.add(:base, _("Cannot associate a Red Hat provider with a custom product"))
+      end
+    end
   end
 end

data/app/models/katello/repository.rb

@@ -12,7 +12,6 @@ module Katello
 
     include ForemanTasks::Concerns::ActionSubject
     include Glue::Candlepin::Repository
-    include Glue::Pulp::Repo
 
     include Glue
     include Authorization::Repository
@@ -185,10 +184,13 @@ module Katello
     delegate :content_id, to: :root, allow_nil: true
     delegate :repository_type, to: :root
 
+    def self.exportable_types(format: ::Katello::Pulp3::ContentViewVersion::Export::IMPORTABLE)
+      return [YUM_TYPE, FILE_TYPE] if format == ::Katello::Pulp3::ContentViewVersion::Export::SYNCABLE
+      EXPORTABLE_TYPES
+    end
+
     def self.exportable(format: ::Katello::Pulp3::ContentViewVersion::Export::IMPORTABLE)
-      types = EXPORTABLE_TYPES
-      types = [YUM_TYPE, FILE_TYPE] if format == ::Katello::Pulp3::ContentViewVersion::Export::SYNCABLE
-      with_type(types)
+      with_type(exportable_types(format: format))
     end
 
     def self.with_type(content_type)
@@ -203,12 +205,10 @@ module Katello
       name
     end
 
-    def backend_service(smart_proxy, force_pulp3 = false)
-      if force_pulp3 || smart_proxy.pulp3_support?(self)
-        @service ||= Katello::Pulp3::Repository.instance_for_type(self, smart_proxy)
-      else
-        @service ||= Katello::Pulp::Repository.instance_for_type(self, smart_proxy)
-      end
+    def backend_service(smart_proxy)
+      fail('Pulp 3 not supported') unless smart_proxy.pulp3_support?(self)
+
+      @service ||= Katello::Pulp3::Repository.instance_for_type(self, smart_proxy)
     end
 
     def backend_content_service(smart_proxy)
@@ -581,7 +581,7 @@ module Katello
 
       if self.yum?
         if self.library_instance?
-          checksum_type = root.checksum_type || pulp_scratchpad_checksum_type
+          checksum_type = root.checksum_type
         else
           checksum_type = self.saved_checksum_type
         end

data/app/models/katello/root_repository.rb

@@ -228,7 +228,7 @@ module Katello
       elsif ignorable_content.any? { |item| !IGNORABLE_CONTENT_UNIT_TYPES.include?(item) }
         errors.add(:ignorable_content, N_("Invalid value specified for ignorable content. Permissible values %s") % IGNORABLE_CONTENT_UNIT_TYPES.join(","))
       elsif self.mirroring_policy == MIRRORING_POLICY_COMPLETE
-        errors.add(:ignorable_content, N_("Ignore SRPMs can not be set in combination with 'Complete Mirroring' mirroring policy."))
+        errors.add(:ignorable_content, N_("Ignore %s can not be set in combination with 'Complete Mirroring' mirroring policy.") % IGNORABLE_CONTENT_UNIT_TYPES.join(","))
       end
     end
 
@@ -417,9 +417,10 @@ module Katello
     end
 
     def http_proxy
-      if http_proxy_policy == NO_DEFAULT_HTTP_PROXY
+      case http_proxy_policy
+      when NO_DEFAULT_HTTP_PROXY
         return nil
-      elsif http_proxy_policy == GLOBAL_DEFAULT_HTTP_PROXY
+      when GLOBAL_DEFAULT_HTTP_PROXY
         return HttpProxy.default_global_content_proxy
       end
       super
@@ -427,7 +428,7 @@ module Katello
 
     def format_arches
       if content_type == ::Katello::Repository::DEB_TYPE
-        self.deb_architectures
+        self.deb_architectures&.gsub(" ", ",")
       else
         self.arch == "noarch" ? nil : self.arch
       end

data/app/models/katello/task_status.rb

@@ -97,14 +97,6 @@ module Katello
       self
     end
 
-    def merge_pulp_task!(pulp_task)
-      PulpTaskStatus.dump_state(pulp_task, self)
-    end
-
-    def refresh_pulp
-      PulpTaskStatus.refresh(self)
-    end
-
     def human_readable_message
       task_template = TaskStatus::TYPES[self.task_type]
       return '' if task_template.nil?
@@ -276,16 +268,6 @@ module Katello
       end
     end
 
-    def self.refresh(ids)
-      unless ids.blank?
-        uuids = TaskStatus.where(:id => ids).pluck(:uuid)
-        ret = Katello.pulp_server.resources.task.poll_all(uuids)
-        ret.each do |pulp_task|
-          PulpTaskStatus.dump_state(pulp_task, TaskStatus.find_by(:uuid => pulp_task[:task_id]))
-        end
-      end
-    end
-
     protected
 
     def setup_task_type

data/app/services/katello/candlepin/consumer.rb

@@ -16,8 +16,8 @@ module Katello
       lazy_accessor :entitlements, :initializer => lambda { |_s| Resources::Candlepin::Consumer.entitlements(uuid) }
       lazy_accessor :consumer_attributes, :initializer => lambda { |_s| Resources::Candlepin::Consumer.get(uuid) }
       lazy_accessor :installed_products, :initializer => lambda { |_s| consumer_attributes['installedProducts'] }
-      lazy_accessor :available_pools, :initializer => lambda { |_s| Resources::Candlepin::Consumer.available_pools(owner_label, uuid, false) }
-      lazy_accessor :all_available_pools, :initializer => lambda { |_s| Resources::Candlepin::Consumer.available_pools(owner_label, uuid, true) }
+      lazy_accessor :available_pools, :initializer => lambda { |_s| Resources::Candlepin::Consumer.available_pools(owner_label, uuid, listall: false) }
+      lazy_accessor :all_available_pools, :initializer => lambda { |_s| Resources::Candlepin::Consumer.available_pools(owner_label, uuid, listall: true) }
       lazy_accessor :content_overrides, :initializer => (lambda do |_s|
                                                            Resources::Candlepin::Consumer.content_overrides(uuid).map do |override|
                                                              ::Katello::ContentOverride.from_entitlement_hash(override)

data/app/services/katello/host/package_profile_uploader.rb

@@ -29,7 +29,7 @@ module Katello
           Rails.logger.warn("Host with ID %s has no content facet; continuing" % host_id)
         else
           begin
-            simple_packages = profile.map { |item| ::Katello::Pulp::SimplePackage.new(item) }
+            simple_packages = profile.map { |item| ::Katello::SimplePackage.new(item) }
             host.import_package_profile(simple_packages)
           rescue ActiveRecord::InvalidForeignKey # this happens if the host gets deleted in between the "find_by" and "import_package_profile"
             Rails.logger.warn("Host installed package list with ID %s was not able to be written to the DB (host likely is deleted); continuing" % host_id)

data/app/services/katello/product_content_finder.rb

@@ -2,9 +2,8 @@ module Katello
   class ProductContentFinder
     attr_accessor :match_environment, :match_subscription, :consumable
 
-    #consumable must implement:
-    #  content_view
-    #  lifecycle_environment
+    # consumable must implement:
+    #  content_view_environments
     #  organization
     #  products
     def initialize(params = {})
@@ -16,16 +15,20 @@ module Katello
 
     def product_content
       if match_environment
-        environment = consumable.lifecycle_environment
-        view = consumable.content_view
-        return ProductContent.none unless environment && view
-        version = ContentViewVersion.in_environment(environment).where(:content_view_id => view).first
+        if consumable.is_a?(::Katello::ActivationKey) # TODO: remove this when AKs are refactored for multi CV
+          environment = consumable.lifecycle_environment
+          view = consumable.content_view
+          return ProductContent.none unless environment && view
+          versions = ContentViewVersion.in_environment(environment).where(:content_view_id => view).first
+        else # consumable is a SubscriptionFacet
+          versions = consumable.content_view_environments.select(:content_view_version_id).map(&:content_view_version_id)
+        end
       end
 
       considered_products = match_subscription ? consumable.products : consumable.organization.products.enabled.uniq
 
       roots = Katello::RootRepository.where(:product_id => considered_products).subscribable
-      roots = roots.in_content_view_version(version) if version
+      roots = roots.in_content_view_version(versions) if versions.present?
 
       consumable.organization.enabled_product_content_for(roots)
     end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -51,6 +51,10 @@ module Katello
           end
         end
 
+        def incremental?
+          from_content_view_version.present?
+        end
+
         def generate_exporter_path
           return base_path if base_path
           export_path = "#{content_view_version.content_view.label}/#{content_view_version.version}/"
@@ -114,72 +118,11 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
-        def validate_chunk_size(size)
-          return if size.blank?
-
-          unless size.is_a?(Numeric) && size > 0 && size < 1e6
-            fail _("Specify an export chunk size less than 1_000_000 GB")
-          end
-        end
-
         def validate!(fail_on_missing_content: true, validate_incremental: true, chunk_size: nil)
-          validate_repositories_immediate! if fail_on_missing_content
-          validate_incremental_export! if validate_incremental && !from_content_view_version.blank?
-          validate_chunk_size(chunk_size)
-          validate_export_types! if fail_on_missing_content
-        end
-
-        def validate_export_types!
-          repos = repositories(fetch_all: true).where.not(id: ::Katello::Repository.exportable(format: format))
-          if repos.any?
-            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
-                   " it contains repositories with un-exportable content types. \n %{repos}" %
-                   { content_view: content_view_version.content_view.name,
-                     current: content_view_version.version,
-                     repos: self.class.generate_product_repo_strings(repositories: repos)})
-
-          end
-        end
-
-        def validate_repositories_immediate!
-          non_immediate_repos = repositories(fetch_all: true).yum_type.non_immediate
-          if non_immediate_repos.any?
-            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
-                   " it contains repositories without the 'immediate' download policy."\
-                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
-                   " and export the generated version. \n %{repos}" %
-                   { content_view: content_view_version.content_view.name,
-                     current: content_view_version.version,
-                     repos: self.class.generate_product_repo_strings(repositories: non_immediate_repos)})
-          end
-        end
-
-        def validate_incremental_export!
-          from_exporter = Export.new(smart_proxy: smart_proxy, content_view_version: from_content_view_version)
-
-          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
-          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
-
-          invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
-            from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
-          end
-
-          if invalid_repos_exist
-            fail _("The exported Content View Version '%{content_view} %{current}' cannot be incrementally updated from version '%{from}'."\
-                   " Please do a full export." % { content_view: content_view_version.content_view.name,
-                                                   current: content_view_version.version,
-                                                   from: from_content_view_version.version})
-          end
-        end
-
-        def generate_repo_mapping(repositories)
-          # return a repo mapping with key being the  library_instance_id and value being the repostiory_href
-          # used by validate_incremental_export
-          repo_map = {}
-          repositories.each do |repo|
-            repo_map[repo.library_instance_id] = version_href_to_repository_href(repo.version_href)
-          end
-          repo_map
+          ExportValidator.new(export_service: self,
+                              fail_on_missing_content: fail_on_missing_content,
+                              validate_incremental: validate_incremental,
+                              chunk_size: chunk_size).validate!
         end
 
         def generate_metadata

data/app/services/katello/pulp3/content_view_version/export_validation_error.rb

@@ -0,0 +1,7 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ExportValidationError < HttpErrors::BadRequest; end
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/export_validator.rb

@@ -0,0 +1,105 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ExportValidator
+        delegate :content_view_version, :from_content_view_version, :format, :repositories,
+                 :smart_proxy, :version_href_to_repository_href, to: :@export_service
+
+        def initialize(export_service:, fail_on_missing_content: true, validate_incremental: true, chunk_size: nil)
+          @export_service = export_service
+          @fail_on_missing_content = fail_on_missing_content
+          @validate_incremental = validate_incremental
+          @chunk_size = chunk_size
+        end
+
+        def validate!
+          validate_repositories_immediate! if @fail_on_missing_content
+          validate_incremental_export! if @validate_incremental && !from_content_view_version.blank?
+          validate_chunk_size
+          validate_export_types! if @fail_on_missing_content
+        end
+
+        def validate_chunk_size
+          return if @chunk_size.blank?
+
+          unless @chunk_size.is_a?(Numeric) && @chunk_size > 0 && @chunk_size < 1e6
+            fail ExportValidationError, _("Specify an export chunk size less than 1_000_000 GB")
+          end
+        end
+
+        def validate_export_types!
+          repos = repositories(fetch_all: true).where.not(id: ::Katello::Repository.exportable(format: format))
+          if repos.any?
+            fail ExportValidationError,
+                 _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories with un-exportable content types. \n %{repos}" %
+                   { content_view: content_view_version.content_view.name,
+                     current: content_view_version.version,
+                     repos: Export.generate_product_repo_strings(repositories: repos)})
+
+          end
+        end
+
+        def validate_repositories_immediate!
+          non_immediate_repos = repositories(fetch_all: true).yum_type.non_immediate
+          if non_immediate_repos.any?
+            fail ExportValidationError,
+                 _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories without the 'immediate' download policy."\
+                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
+                   " and export the generated version. \n %{repos}" %
+                   { content_view: content_view_version.content_view.name,
+                     current: content_view_version.version,
+                     repos: Export.generate_product_repo_strings(repositories: non_immediate_repos)})
+          end
+        end
+
+        def validate_incremental_export_point_release!
+          # You are trying to export between an incrementally updated content view version and regular version
+          if from_content_view_version.incrementally_updated? != content_view_version.incrementally_updated?
+            fail ExportValidationError,
+                 _("Cannot incrementally export from a incrementally exported version and a regular version or vice-versa. "\
+                   " The exported Content View Version '%{content_view} %{current}' "\
+                   "cannot be incrementally exported from version '%{from}.'"\
+                               " Please do a full export." % { content_view: content_view_version.content_view.name,
+                                                               current: content_view_version.version,
+                                                               from: from_content_view_version.version})
+          end
+        end
+
+        def validate_incremental_export!
+          validate_incremental_export_point_release!
+
+          from_exporter = Export.new(smart_proxy: smart_proxy, content_view_version: from_content_view_version)
+
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
+          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
+
+          invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
+            from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
+          end
+
+          if invalid_repos_exist
+            fail ExportValidationError,
+                _("Cannot incrementally export from a filtered and a non-filtered content view version."\
+                   " The exported content view version '%{content_view} %{current}' "\
+                   " cannot be incrementally updated from version '%{from}.'. "\
+                   " Please do a full export." % { content_view: content_view_version.content_view.name,
+                                                   current: content_view_version.version,
+                                                   from: from_content_view_version.version})
+          end
+        end
+
+        def generate_repo_mapping(repositories)
+          # return a repo mapping with key being the  library_instance_id and value being the repostiory_href
+          # used by validate_incremental_export
+          repo_map = {}
+          repositories.each do |repo|
+            repo_map[repo.library_instance_id] = version_href_to_repository_href(repo.version_href)
+          end
+          repo_map
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/importable_products.rb

@@ -64,7 +64,7 @@ module Katello
 
         def gpg_key_id(metadata_product)
           if metadata_product.gpg_key
-            @organization.gpg_keys.where(name: metadata_product.gpg_key.name).pluck(:id).first
+            @organization.gpg_keys.where(name: metadata_product.gpg_key.name).pick(:id)
           end
         end
       end

data/app/services/katello/pulp3/content_view_version/syncable_format_export.rb

@@ -9,7 +9,13 @@ module Katello
         end
 
         def create_export(exporter_data, _options = {})
-          [api.yum_export_api.create(exporter_data[:pulp_href], publication: repository.publication_href)]
+          options = { publication: repository.publication_href }
+          if incremental?
+            from_exporter = Export.new(smart_proxy: smart_proxy, content_view_version: from_content_view_version)
+            from_repo = from_exporter.repositories.find_by(library_instance_id: repository.library_instance)
+            options[:start_repository_version] = from_repo.version_href unless from_repo.blank?
+          end
+          [api.yum_export_api.create(exporter_data[:pulp_href], options)]
         end
 
         def fetch_export(exporter_href)

data/app/services/katello/pulp3/erratum.rb

@@ -28,6 +28,7 @@ module Katello
       def self.generate_model_row(unit)
         keys = %w(title id severity issued_date type description reboot_suggested solution updated_date summary)
         custom_json = unit.slice(*keys)
+        custom_json.inject(HashWithIndifferentAccess.new({})) { |h, (k, v)| h.merge({ k => v.respond_to?(:strip) ? v.strip : v }) }
         custom_json['pulp_id'] = custom_json['id']
         custom_json["issued"] = custom_json.delete("issued_date")
         custom_json["updated"] = custom_json.delete("updated_date")

data/app/services/katello/pulp3/module_stream.rb

@@ -25,16 +25,25 @@ module Katello
       end
 
       def self.build_artifacts(katello_id, artifacts_json)
-        return [] if artifacts_json.nil?
-        artifacts = artifacts_json.map do |name|
-          {name: name,
-           module_stream_id: katello_id}
+        # TODO: Fully support artifact hashes
+        # => {"rpms"=>["swig"], "description"=>"Simplified Wrapper and Interface Generator (SWIG)"}
+        return [] if artifacts_json.empty?
+
+        if artifacts_json.is_a?(::Hash)
+          artifacts = artifacts_json['rpms'].map do |name|
+            {name: name, module_stream_id: katello_id}
+          end
+        else
+          # For compatibility with pulpcore 3.21 + pulp-rpm 3.18 and below.
+          artifacts = artifacts_json.map do |name|
+            {name: name, module_stream_id: katello_id}
+          end
         end
         add_timestamps(artifacts)
       end
 
       def self.build_profiles(katello_id, profiles_json)
-        return [] if profiles_json.nil?
+        return [] if profiles_json.empty?
         profiles = profiles_json.map do |profile, _rpms|
           {
             module_stream_id: katello_id,
@@ -45,14 +54,20 @@ module Katello
       end
 
       def self.build_profile_rpms(katello_id, profiles_json)
+        # TODO: Fully support artifact hashes
+        # => {"rpms"=>["swig"], "description"=>"Simplified Wrapper and Interface Generator (SWIG)"}
         return [] if profiles_json.nil?
-        profile_rpms = profiles_json.map do |profile, rpms|
+        profile_rpms = profiles_json.map do |profile, artifacts_json|
           profile_id = Katello::ModuleProfile.find_by(module_stream_id: katello_id, name: profile).id
-          rpms.map do |rpm|
-            {
-              module_profile_id: profile_id,
-              name: rpm
-            }
+          if artifacts_json.is_a?(::Hash)
+            artifacts_json['rpms'].map do |rpm|
+              {name: rpm, module_profile_id: profile_id}
+            end
+          else
+            # For compatibility with pulpcore 3.21 + pulp-rpm 3.18 and below.
+            artifacts_json.map do |rpm|
+              {name: rpm, module_profile_id: profile_id}
+            end
           end
         end
         add_timestamps(profile_rpms.flatten)

data/app/services/katello/pulp3/repository/yum.rb

@@ -440,7 +440,8 @@ module Katello
 
           errata_to_include = filter_errata_by_pulp_href(source_repository.errata, content_unit_hrefs,
                                                          source_repository.rpms.pluck(:filename) +
-                                                         source_repository.srpms.pluck(:filename)) -
+                                                         source_repository.srpms.pluck(:filename),
+                                                         source_repository.module_streams.map(&:module_spec)) -
                                                          all_excluded_errata
           content_unit_hrefs += errata_to_include.collect do |erratum|
             erratum.repository_errata.where(repository_id: source_repository.id).pluck(:erratum_pulp3_href)

data/app/services/katello/pulp3/smart_proxy_mirror_repository.rb

@@ -96,7 +96,8 @@ module Katello
 
       def delete_orphan_remotes
         tasks = []
-        repo_names = Katello::Repository.pluck(:pulp_id)
+        smart_proxy_helper = ::Katello::SmartProxyHelper.new(smart_proxy)
+        repo_names = smart_proxy_helper.combined_repos_available_to_capsule.map(&:pulp_id)
         acs_remotes = Katello::SmartProxyAlternateContentSource.pluck(:remote_href)
         pulp3_enabled_repo_types.each do |repo_type|
           api = repo_type.pulp3_api(smart_proxy)