katello 4.5.1 → 4.8.0.rc2

data/app/services/katello/pulp3/alternate_content_source.rb

@@ -5,10 +5,12 @@ module Katello
       include Katello::Pulp3::ServiceCommon
       attr_accessor :acs
       attr_accessor :smart_proxy
+      attr_accessor :repository
 
-      def initialize(acs, smart_proxy)
+      def initialize(acs, smart_proxy, repository = nil)
         @acs = acs
         @smart_proxy = smart_proxy
+        @repository = repository
       end
 
       def api
@@ -20,18 +22,40 @@ module Katello
       end
 
       def smart_proxy_acs
-        ::Katello::SmartProxyAlternateContentSource.find_by(alternate_content_source_id: acs.id, smart_proxy_id: smart_proxy.id)
+        if %w[custom rhui].include?(acs.alternate_content_source_type)
+          ::Katello::SmartProxyAlternateContentSource.find_by(alternate_content_source_id: acs.id, smart_proxy_id: smart_proxy.id)
+        else
+          ::Katello::SmartProxyAlternateContentSource.find_by(alternate_content_source_id: acs.id, smart_proxy_id: smart_proxy.id, repository_id: repository.id)
+        end
+      end
+
+      def simplified_acs_remote_options
+        options = repository.backend_service(smart_proxy).remote_options
+        options[:policy] = 'on_demand'
+        # Potential RFE: allow inheriting of default smart proxy repo's HTTP proxies for simplified ACSs
+        if acs.use_http_proxies
+          options[:proxy_url] = smart_proxy.http_proxy&.url
+          options[:proxy_username] = smart_proxy.http_proxy&.username
+          options[:proxy_password] = smart_proxy.http_proxy&.password
+        else
+          options[:proxy_url] = nil
+          options[:proxy_username] = nil
+          options[:proxy_password] = nil
+        end
+        options
       end
 
       def remote_options
+        return simplified_acs_remote_options if repository.present?
+
         remote_options = {
           tls_validation: acs.verify_ssl,
           name: generate_backend_object_name,
           url: acs.base_url,
           policy: 'on_demand',
-          proxy_url: acs.http_proxy&.url,
-          proxy_username: acs.http_proxy&.username,
-          proxy_password: acs.http_proxy&.password,
+          proxy_url: smart_proxy.http_proxy&.url,
+          proxy_username: smart_proxy.http_proxy&.username,
+          proxy_password: smart_proxy.http_proxy&.password,
           total_timeout: Setting[:sync_connect_timeout]
         }
         if acs.content_type == ::Katello::Repository::FILE_TYPE && acs.subpaths.empty? && !remote_options[:url].end_with?('/PULP_MANIFEST')
@@ -42,7 +66,7 @@ module Katello
       end
 
       def ssl_remote_options
-        if acs.custom?
+        if acs.custom? || acs.rhui?
           {
             client_cert: acs.ssl_client_cert&.content,
             client_key: acs.ssl_client_key&.content,
@@ -66,8 +90,11 @@ module Katello
         api.remotes_api.partial_update(smart_proxy_acs.remote_href, remote_options)
       end
 
-      def delete_remote(href = smart_proxy_acs.remote_href)
-        ignore_404_exception { remote_options[:url]&.start_with?('uln') ? api.remotes_uln_api.delete(href) : api.remotes_api.delete(href) } if href
+      # The old repo URL is needed to determine which remote API to use.
+      def delete_remote(options = {})
+        options[:href] ||= smart_proxy_acs.remote_href
+        options[:old_url] ||= remote_options[:url]
+        ignore_404_exception { options[:old_url]&.start_with?('uln') ? api.remotes_uln_api.delete(options[:href]) : api.remotes_api.delete(options[:href]) } if options[:href]
       end
 
       def create
@@ -103,8 +130,7 @@ module Katello
 
       def refresh
         href = smart_proxy_acs.alternate_content_source_href
-        # https://github.com/pulp/pulp_rpm/issues/2504
-        api.alternate_content_source_api.refresh(href, 'placeholder')
+        api.alternate_content_source_api.refresh(href)
       end
 
       private

data/app/services/katello/pulp3/ansible_collection.rb

@@ -30,21 +30,24 @@ module Katello
       end
 
       def self.insert_child_associations(units, pulp_id_to_id)
-        tag_names = units.map { |unit| unit['tags'].map { |tag| tag[:name] } }.flatten
-        tag_rows = tag_names.map { |name| {name: name } }
-        Katello::AnsibleTag.insert_all(tag_rows, unique_by: [:name]) if tag_rows.any?
-
+        insert_tags units
         collection_tag_rows = []
         units.each do |unit|
           katello_id = pulp_id_to_id[unit['pulp_href']]
           #delete old tags
-          unit_tags = unit['tags'].map { |tag| tag[:name] }
+          unit_tags = unit['tags']&.map { |tag| tag[:name] }
           Katello::AnsibleCollectionTag.where(:ansible_collection_id => katello_id).where.not(:ansible_tag_id => Katello::AnsibleTag.where(:name => unit_tags)).delete_all
-          collection_tag_rows += Katello::AnsibleTag.where(:name => unit_tags).pluck(:id).map { |tag_id| {ansible_collection_id: katello_id, ansible_tag_id: tag_id} }
+          collection_tag_rows += Katello::AnsibleTag.where(:name => unit_tags)&.pluck(:id)&.map { |tag_id| {ansible_collection_id: katello_id, ansible_tag_id: tag_id} }
         end
 
         collection_tag_rows.flatten!
-        Katello::AnsibleCollectionTag.insert_all(collection_tag_rows, unique_by: [:ansible_collection_id, :ansible_tag_id])
+        Katello::AnsibleCollectionTag.insert_all(collection_tag_rows, unique_by: [:ansible_collection_id, :ansible_tag_id]) unless collection_tag_rows.empty?
+      end
+
+      def self.insert_tags(units)
+        tag_names = units.map { |unit| unit['tags']&.map { |tag| tag[:name] } }&.flatten
+        tag_rows = tag_names&.compact&.map { |name| {name: name } }
+        Katello::AnsibleTag.insert_all(tag_rows, unique_by: [:name]) if tag_rows.any?
       end
     end
   end

data/app/services/katello/pulp3/api/apt.rb

@@ -11,6 +11,18 @@ module Katello
         def publications_verbatim_api
           PulpDebClient::PublicationsVerbatimApi.new(api_client)
         end
+
+        def self.copy_class
+          PulpDebClient::Copy
+        end
+
+        def self.add_remove_content_class
+          PulpDebClient::RepositoryAddRemoveContent
+        end
+
+        def copy_api
+          PulpDebClient::CopyApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/api/core.rb

@@ -74,8 +74,8 @@ module Katello
           nil
         end
 
-        def repository_version_class
-          client_module::RepositoryVersion
+        def repair_class
+          client_module::Repair
         end
 
         def cancel_task(task_href)

data/app/services/katello/pulp3/api/generic.rb

@@ -15,10 +15,6 @@ module Katello
         def self.repository_sync_url_class(repository_type)
           repository_type.repo_sync_url_class
         end
-
-        def self.add_remove_content_class
-          fail NotImplementedError
-        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -40,9 +40,9 @@ module Katello
 
         def repositories(fetch_all: false)
           repos = if content_view_version.default?
-                    content_view_version.repositories.exportable
+                    content_view_version.repositories.exportable(format: format)
                   else
-                    content_view_version.archived_repos.exportable
+                    content_view_version.archived_repos.exportable(format: format)
                   end
           if fetch_all
             repos
@@ -51,11 +51,15 @@ module Katello
           end
         end
 
+        def incremental?
+          from_content_view_version.present?
+        end
+
         def generate_exporter_path
           return base_path if base_path
-          export_path = "#{content_view_version.content_view}/#{content_view_version.version}/"
-          export_path += "#{destination_server}/" unless destination_server.blank?
-          export_path += "#{date_dir}".gsub(/\s/, '_')
+          export_path = "#{content_view_version.content_view.label}/#{content_view_version.version}/"
+          export_path += "#{destination_server}/".gsub(/\s/, '_') unless destination_server.blank?
+          export_path += "#{date_dir}"
           @base_path = "#{Setting['pulpcore_export_destination']}/#{content_view_version.organization.label}/#{export_path}"
         end
 
@@ -114,59 +118,11 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
-        def validate_chunk_size(size)
-          return if size.blank?
-
-          unless size.is_a?(Numeric) && size > 0 && size < 1e6
-            fail _("Specify an export chunk size less than 1_000_000 GB")
-          end
-        end
-
         def validate!(fail_on_missing_content: true, validate_incremental: true, chunk_size: nil)
-          validate_repositories_immediate! if fail_on_missing_content
-          validate_incremental_export! if validate_incremental && !from_content_view_version.blank?
-          validate_chunk_size(chunk_size)
-        end
-
-        def validate_repositories_immediate!
-          non_immediate_repos = repositories(fetch_all: true).non_immediate
-          if non_immediate_repos.any?
-            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
-                   " it contains repositories without the 'immediate' download policy."\
-                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
-                   " and export the generated version. \n %{repos}" %
-                   { content_view: content_view_version.content_view.name,
-                     current: content_view_version.version,
-                     repos: self.class.generate_product_repo_strings(repositories: non_immediate_repos)})
-          end
-        end
-
-        def validate_incremental_export!
-          from_exporter = Export.new(smart_proxy: smart_proxy, content_view_version: from_content_view_version)
-
-          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
-          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
-
-          invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
-            from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
-          end
-
-          if invalid_repos_exist
-            fail _("The exported Content View Version '%{content_view} %{current}' cannot be incrementally updated from version '%{from}'."\
-                   " Please do a full export." % { content_view: content_view_version.content_view.name,
-                                                   current: content_view_version.version,
-                                                   from: from_content_view_version.version})
-          end
-        end
-
-        def generate_repo_mapping(repositories)
-          # return a repo mapping with key being the  library_instance_id and value being the repostiory_href
-          # used by validate_incremental_export
-          repo_map = {}
-          repositories.each do |repo|
-            repo_map[repo.library_instance_id] = version_href_to_repository_href(repo.version_href)
-          end
-          repo_map
+          ExportValidator.new(export_service: self,
+                              fail_on_missing_content: fail_on_missing_content,
+                              validate_incremental: validate_incremental,
+                              chunk_size: chunk_size).validate!
         end
 
         def generate_metadata
@@ -191,21 +147,38 @@ module Katello
 
         def self.find_library_export_view(create_by_default: false,
                                           destination_server:,
-                                          organization:)
+                                          organization:,
+                                          format: IMPORTABLE)
+          if format == IMPORTABLE
+            generated_for = :library_export
+            name = ::Katello::ContentView::EXPORT_LIBRARY
+          else
+            generated_for = :library_export_syncable
+            name = "#{::Katello::ContentView::EXPORT_LIBRARY}-SYNCABLE"
+          end
+
           find_generated_export_view(create_by_default: create_by_default,
                                      destination_server: destination_server,
                                      organization: organization,
-                                     name: ::Katello::ContentView::EXPORT_LIBRARY,
-                                     generated_for: :library_export)
+                                     name: name,
+                                     generated_for: generated_for)
         end
 
         def self.find_repository_export_view(create_by_default: false,
-                                              repository:)
+                                              repository:,
+                                              format: IMPORTABLE)
+          if format == IMPORTABLE
+            generated_for = :repository_export
+            name = "Export-#{repository.label}-#{repository.library_instance_or_self.id}"
+          else
+            generated_for = :repository_export_syncable
+            name = "Export-SYNCABLE-#{repository.label}-#{repository.library_instance_or_self.id}"
+          end
           find_generated_export_view(create_by_default: create_by_default,
                                      destination_server: nil,
                                      organization: repository.organization,
-                                     name: "Export-#{repository.label}-#{repository.library_instance_or_self.id}",
-                                     generated_for: :repository_export)
+                                     name: name,
+                                     generated_for: generated_for)
         end
 
         def self.generate_product_repo_strings(repositories:)

data/app/services/katello/pulp3/content_view_version/export_validation_error.rb

@@ -0,0 +1,7 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ExportValidationError < StandardError; end
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/export_validator.rb

@@ -0,0 +1,121 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ExportValidator
+        delegate :content_view_version, :from_content_view_version, :format, :repositories,
+                 :smart_proxy, :version_href_to_repository_href, to: :@export_service
+
+        def initialize(export_service:, fail_on_missing_content: true, validate_incremental: true, chunk_size: nil)
+          @export_service = export_service
+          @fail_on_missing_content = fail_on_missing_content
+          @validate_incremental = validate_incremental
+          @chunk_size = chunk_size
+        end
+
+        def validate!
+          validate_has_repos!
+          validate_repositories_immediate! if @fail_on_missing_content
+          validate_incremental_export! if @validate_incremental && !from_content_view_version.blank?
+          validate_chunk_size
+          validate_export_types! if @fail_on_missing_content
+        end
+
+        def validate_has_repos!
+          repos = repositories(fetch_all: true).where(id: ::Katello::Repository.exportable(format: format))
+          if repos.empty?
+            fail ExportValidationError,
+                 _("NOTE: Content view version '%{content_view} %{current}'"\
+                   " does not have any exportable repositories. At least one repository with"\
+                   " any of the following types"\
+                   " is required to be able to export: '%{exportable_types}'." %
+                   { content_view: content_view_version.content_view.name,
+                     current: content_view_version.version,
+                     exportable_types: ::Katello::Repository.exportable_types(format: format).join("', '")
+                   })
+          end
+        end
+
+        def validate_chunk_size
+          return if @chunk_size.blank?
+
+          unless @chunk_size.is_a?(Numeric) && @chunk_size > 0 && @chunk_size < 1e6
+            fail ExportValidationError, _("Specify an export chunk size less than 1_000_000 GB")
+          end
+        end
+
+        def validate_export_types!
+          repos = repositories(fetch_all: true).where.not(id: ::Katello::Repository.exportable(format: format))
+          if repos.any?
+            fail ExportValidationError,
+                 _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories with un-exportable content types. \n %{repos}" %
+                   { content_view: content_view_version.content_view.name,
+                     current: content_view_version.version,
+                     repos: Export.generate_product_repo_strings(repositories: repos)})
+
+          end
+        end
+
+        def validate_repositories_immediate!
+          non_immediate_repos = repositories(fetch_all: true).yum_type.non_immediate
+          if non_immediate_repos.any?
+            fail ExportValidationError,
+                 _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories without the 'immediate' download policy."\
+                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
+                   " and export the generated version. \n %{repos}" %
+                   { content_view: content_view_version.content_view.name,
+                     current: content_view_version.version,
+                     repos: Export.generate_product_repo_strings(repositories: non_immediate_repos)})
+          end
+        end
+
+        def validate_incremental_export_point_release!
+          # You are trying to export between an incrementally updated content view version and regular version
+          if from_content_view_version.incrementally_updated? != content_view_version.incrementally_updated?
+            fail ExportValidationError,
+                 _("Cannot incrementally export from a incrementally exported version and a regular version or vice-versa. "\
+                   " The exported Content View Version '%{content_view} %{current}' "\
+                   "cannot be incrementally exported from version '%{from}.'"\
+                               " Please do a full export." % { content_view: content_view_version.content_view.name,
+                                                               current: content_view_version.version,
+                                                               from: from_content_view_version.version})
+          end
+        end
+
+        def validate_incremental_export!
+          validate_incremental_export_point_release!
+
+          from_exporter = Export.new(smart_proxy: smart_proxy, content_view_version: from_content_view_version)
+
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
+          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
+
+          invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
+            from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
+          end
+
+          if invalid_repos_exist
+            fail ExportValidationError,
+                _("Cannot incrementally export from a filtered and a non-filtered content view version."\
+                   " The exported content view version '%{content_view} %{current}' "\
+                   " cannot be incrementally updated from version '%{from}.'. "\
+                   " Please do a full export." % { content_view: content_view_version.content_view.name,
+                                                   current: content_view_version.version,
+                                                   from: from_content_view_version.version})
+          end
+        end
+
+        def generate_repo_mapping(repositories)
+          # return a repo mapping with key being the  library_instance_id and value being the repostiory_href
+          # used by validate_incremental_export
+          repo_map = {}
+          repositories.each do |repo|
+            repo_map[repo.library_instance_id] = version_href_to_repository_href(repo.version_href)
+          end
+          repo_map
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -156,6 +156,8 @@ module Katello
           fail _("Content View label not provided.") if metadata_map.content_view.label.blank?
 
           params = import_content_view_params
+          return if @metadata_map.syncable_format? && params[:generated_for] != :none
+
           cv = ::Katello::ContentView.find_by(label: params[:label],
                                               organization: organization)
           if cv.blank?

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -13,10 +13,14 @@ module Katello
         end
 
         def check!
-          fail _("Content view not provided in the metadata") if @metadata_map.content_view.blank?
+          if @metadata_map.content_view.blank? && !metadata_map.syncable_format?
+            fail _("Content view not provided in the metadata")
+          end
 
+          ensure_non_syncable_path_valid! unless @metadata_map.syncable_format?
           ensure_pulp_importable!
-          unless @content_view.default?
+          if @content_view && !@content_view.default?
+            ensure_non_composite!
             ensure_importing_cvv_does_not_exist!
             ensure_from_cvv_exists!
           end
@@ -25,7 +29,20 @@ module Katello
           ensure_redhat_products_metadata_are_in_the_library!
         end
 
+        def ensure_non_syncable_path_valid!
+          uri = URI(@path)
+          unless uri.scheme.blank? || uri.scheme == "file"
+            fail _("Invalid path provided. Content can be only imported from file system. ")
+          end
+        end
+
+        def ensure_non_composite!
+          return if @content_view.blank?
+          fail _("Content cannot be imported into a Composite Content View. ") if @content_view.composite?
+        end
+
         def ensure_pulp_importable!
+          return if @metadata_map.syncable_format?
           api = ::Katello::Pulp3::Api::Core.new(@smart_proxy).importer_check_api
           response = api.pulp_import_check_post(toc: "#{@path}/#{@metadata_map.toc}")
           unless response.toc.is_valid
@@ -80,8 +97,7 @@ module Katello
           if bad_repos.any?
             fail _("The following repositories provided in the import metadata have an incorrect content type or provider type. "\
                     "Make sure the export and import repositories are of the same type before importing\n "\
-                    "%{repos}" % { content_view: @content_view.name,
-                                   repos: generate_product_repo_i18n_string(bad_repos).join("")}
+                    "%{repos}" % { repos: generate_product_repo_i18n_string(bad_repos).join("")}
                   )
           end
         end
@@ -94,7 +110,7 @@ module Katello
           if missing.any?
             repos_in_import = generate_product_repo_i18n_string(missing)
             fail _("The organization's manifest does not contain the subscriptions required to enable the following repositories.\n "\
-                    "%{repos}" % { content_view: @content_view.name, repos: repos_in_import.join("")}
+                    "%{repos}" % { repos: repos_in_import.join("")}
                   )
           end
         end

data/app/services/katello/pulp3/content_view_version/importable_products.rb

@@ -37,12 +37,22 @@ module Katello
         def create_params(metadata_product)
           {
             gpg_key_id: gpg_key_id(metadata_product),
-            name: metadata_product.name,
+            name: find_unique_name(metadata_product),
             label: metadata_product.label,
             description: metadata_product.description
           }
         end
 
+        def find_unique_name(metadata_product)
+          name = metadata_product.name
+          i = 1
+          while @products_in_library.where(name: name).exists?
+            name = "#{metadata_product.name} #{i}"
+            i += 1
+          end
+          name
+        end
+
         def update_params(metadata_product)
           params = {
             gpg_key_id: gpg_key_id(metadata_product),
@@ -54,7 +64,7 @@ module Katello
 
         def gpg_key_id(metadata_product)
           if metadata_product.gpg_key
-            @organization.gpg_keys.where(name: metadata_product.gpg_key.name).pluck(:id).first
+            @organization.gpg_keys.where(name: metadata_product.gpg_key.name).pick(:id)
           end
         end
       end

data/app/services/katello/pulp3/content_view_version/importable_repositories.rb

@@ -4,11 +4,16 @@ module Katello
       class ImportableRepositories
         attr_reader :creatable, :updatable
 
-        def initialize(organization:, metadata_repositories:)
+        def initialize(organization:,
+                       metadata_repositories:,
+                       syncable_format: false,
+                       path: nil)
           @organization = organization
           @metadata_repositories = metadata_repositories
           @creatable = []
           @updatable = []
+          @syncable_format = syncable_format
+          @path = path
         end
 
         def generate!
@@ -41,15 +46,29 @@ module Katello
                 basearch: repo.arch,
                 releasever: repo.minor
               }
-              creatable << { product: product, content: product_content, substitutions: substitutions }
+              creatable << { product: product,
+                             content: product_content,
+                             substitutions: substitutions,
+                             override_url: fetch_feed_url(repo)
+                           }
             else
-              creatable << { repository: product.add_repo(create_repo_params(repo)) }
+              creatable << { repository: product.add_repo(create_repo_params(repo, product)) }
             end
           end
         end
 
         private
 
+        def find_unique_name(metadata_repo, product)
+          name = metadata_repo.name
+          i = 1
+          while product.root_repositories.where(name: name).exists?
+            name = "#{metadata_repo.name} #{i}"
+            i += 1
+          end
+          name
+        end
+
         def product_for_metadata_repo(repo)
           if repo.redhat && repo.product.cp_id
             @organization.products.includes(:root_repositories).find_by(cp_id: repo.product.cp_id)
@@ -74,9 +93,8 @@ module Katello
           end
         end
 
-        def create_repo_params(metadata_repo)
+        def create_repo_params(metadata_repo, product)
           keys = [
-            :name,
             :label,
             :description,
             :arch,
@@ -90,16 +108,26 @@ module Katello
             :mirroring_policy
           ]
 
-          params = {}
+          params = { name: find_unique_name(metadata_repo, product) }
           params[:gpg_key_id] = gpg_key_id(metadata_repo)
-
           keys.each do |key|
             params[key] = metadata_repo.send(key)
           end
-
+          url = fetch_feed_url(metadata_repo)
+          params[:url] = url if url
           params
         end
 
+        def fetch_feed_url(metadata_repo)
+          return unless @syncable_format
+          uri = URI(@path)
+          if uri.scheme.blank? || uri.scheme == "file"
+            "file://#{uri.path.chomp('/')}#{metadata_repo.content.url}"
+          else
+            "#{@path.chomp('/')}#{metadata_repo.content.url}"
+          end
+        end
+
         def update_repo_params(metadata_repo)
           keys = [
             :description,
@@ -112,8 +140,9 @@ module Katello
             :download_policy,
             :mirroring_policy
           ]
-
           params = {}
+          url = fetch_feed_url(metadata_repo)
+          params[:url] = url if url
           params[:gpg_key_id] = gpg_key_id(metadata_repo)
 
           keys.each do |key|