katello 4.4.1 → 4.5.0

data/app/services/katello/pulp3/alternate_content_source.rb

@@ -0,0 +1,123 @@
+require "pulpcore_client"
+module Katello
+  module Pulp3
+    class AlternateContentSource
+      include Katello::Pulp3::ServiceCommon
+      attr_accessor :acs
+      attr_accessor :smart_proxy
+
+      def initialize(acs, smart_proxy)
+        @acs = acs
+        @smart_proxy = smart_proxy
+      end
+
+      def api
+        @api ||= ::Katello::Pulp3::Repository.api(smart_proxy, @acs.content_type)
+      end
+
+      def generate_backend_object_name
+        "#{acs.label}-#{smart_proxy.url}-#{rand(9999)}"
+      end
+
+      def smart_proxy_acs
+        ::Katello::SmartProxyAlternateContentSource.find_by(alternate_content_source_id: acs.id, smart_proxy_id: smart_proxy.id)
+      end
+
+      def remote_options
+        remote_options = {
+          tls_validation: acs.verify_ssl,
+          name: generate_backend_object_name,
+          url: acs.base_url,
+          policy: 'on_demand',
+          proxy_url: acs.http_proxy&.url,
+          proxy_username: acs.http_proxy&.username,
+          proxy_password: acs.http_proxy&.password,
+          total_timeout: Setting[:sync_connect_timeout]
+        }
+        if acs.content_type == ::Katello::Repository::FILE_TYPE && acs.subpaths.empty? && !remote_options[:url].end_with?('/PULP_MANIFEST')
+          remote_options[:url] = acs.base_url + '/PULP_MANIFEST'
+        end
+        remote_options.merge!(username: acs&.upstream_username, password: acs&.upstream_password)
+        remote_options.merge!(ssl_remote_options)
+      end
+
+      def ssl_remote_options
+        if acs.custom?
+          {
+            client_cert: acs.ssl_client_cert&.content,
+            client_key: acs.ssl_client_key&.content,
+            ca_cert: acs.ssl_ca_cert&.content
+          }
+        end
+      end
+
+      def create_remote
+        if smart_proxy_acs&.remote_href.nil?
+          response = super
+          smart_proxy_acs.update!(remote_href: response.pulp_href)
+        end
+      end
+
+      def get_remote(href = smart_proxy_acs.remote_href)
+        acs.base_url&.start_with?('uln') ? api.remotes_uln_api.read(href) : api.remotes_api.read(href)
+      end
+
+      def update_remote
+        api.remotes_api.partial_update(smart_proxy_acs.remote_href, remote_options)
+      end
+
+      def delete_remote(href = smart_proxy_acs.remote_href)
+        ignore_404_exception { remote_options[:url]&.start_with?('uln') ? api.remotes_uln_api.delete(href) : api.remotes_api.delete(href) } if href
+      end
+
+      def create
+        if smart_proxy_acs&.alternate_content_source_href.nil?
+          paths = acs.subpaths.deep_dup
+          if acs.content_type == ::Katello::Repository::FILE_TYPE && acs.subpaths.present?
+            paths = insert_pulp_manifest!(paths)
+          end
+          response = api.alternate_content_source_api.create(name: generate_backend_object_name, paths: paths,
+                                                             remote: smart_proxy_acs.remote_href)
+          smart_proxy_acs.update!(alternate_content_source_href: response.pulp_href)
+          return response
+        end
+      end
+
+      def read(href = smart_proxy_acs.alternate_content_source_href)
+        api.alternate_content_source_api.read(href)
+      end
+
+      def update
+        href = smart_proxy_acs.alternate_content_source_href
+        paths = acs.subpaths.deep_dup
+        if acs.content_type == ::Katello::Repository::FILE_TYPE && acs.subpaths.present?
+          paths = insert_pulp_manifest!(paths)
+        end
+        api.alternate_content_source_api.update(href, name: generate_backend_object_name, paths: paths, remote: smart_proxy_acs.remote_href)
+      end
+
+      def delete_alternate_content_source
+        href = smart_proxy_acs.alternate_content_source_href
+        ignore_404_exception { api.alternate_content_source_api.delete(href) } if href
+      end
+
+      def refresh
+        href = smart_proxy_acs.alternate_content_source_href
+        # https://github.com/pulp/pulp_rpm/issues/2504
+        api.alternate_content_source_api.refresh(href, 'placeholder')
+      end
+
+      private
+
+      def insert_pulp_manifest!(subpaths)
+        subpaths.map! do |subpath|
+          if subpath.end_with?('/PULP_MANIFEST')
+            subpath
+          else
+            subpath + '/PULP_MANIFEST'
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/api/file.rb

@@ -7,6 +7,14 @@ module Katello
         def self.add_remove_content_class
           PulpFileClient::RepositoryAddRemoveContent
         end
+
+        def self.alternate_content_source_class
+          PulpFileClient::FileFileAlternateContentSource
+        end
+
+        def alternate_content_source_api
+          PulpFileClient::AcsFileApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/api/yum.rb

@@ -20,6 +20,14 @@ module Katello
           PulpRpmClient::RepositoryAddRemoveContent
         end
 
+        def self.alternate_content_source_class
+          PulpRpmClient::RpmRpmAlternateContentSource
+        end
+
+        def alternate_content_source_api
+          PulpRpmClient::AcsRpmApi.new(api_client)
+        end
+
         def remotes_uln_api
           PulpRpmClient::RemotesUlnApi.new(api_client)
         end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -46,7 +46,7 @@ module Katello
         end
 
         def create_exporter(export_base_dir: Setting['pulpcore_export_destination'])
-          api.exporter_api.create(name: generate_id,
+          api.exporter_api.create(name: generate_id(content_view_version),
                                   path: "#{export_base_dir}/#{generate_exporter_path}",
                                   repositories: repository_hrefs)
         end
@@ -150,13 +150,35 @@ module Katello
           MetadataGenerator.new(export_service: self).generate!
         end
 
+        def self.find_generated_export_view(create_by_default: false,
+                                            destination_server:,
+                                            organization:,
+                                            name:,
+                                            generated_for:)
+          name += "-#{destination_server}" unless destination_server.blank?
+          select_method = create_by_default ? :first_or_create : :first
+          ::Katello::ContentView.where(name: name,
+                                       organization: organization,
+                                       generated_for: generated_for).send(select_method)
+        end
+
         def self.find_library_export_view(create_by_default: false,
                                           destination_server:,
                                           organization:)
-          name = ::Katello::ContentView::EXPORT_LIBRARY
-          name += "-#{destination_server}" unless destination_server.blank?
-          select_method = create_by_default ? :first_or_create : :first
-          ::Katello::ContentView.where(name: name, organization: organization).send(select_method)
+          find_generated_export_view(create_by_default: create_by_default,
+                                     destination_server: destination_server,
+                                     organization: organization,
+                                     name: ::Katello::ContentView::EXPORT_LIBRARY,
+                                     generated_for: :library_export)
+        end
+
+        def self.find_repository_export_view(create_by_default: false,
+                                              repository:)
+          find_generated_export_view(create_by_default: create_by_default,
+                                     destination_server: nil,
+                                     organization: repository.organization,
+                                     name: "Export-#{repository.label}-#{repository.library_instance_or_self.id}",
+                                     generated_for: :repository_export)
         end
 
         def self.generate_product_repo_strings(repositories:)

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -4,33 +4,23 @@ module Katello
       class Import
         include ImportExportCommon
 
-        def initialize(smart_proxy:, content_view_version: nil, path: nil, metadata: nil)
+        attr_reader :content_view, :path, :metadata_map, :smart_proxy, :organization
+
+        def initialize(organization:, smart_proxy:, path:, metadata_map:)
+          @organization = organization
           @smart_proxy = smart_proxy
-          @content_view_version = content_view_version
           @path = path
-          @metadata = metadata
+          @metadata_map = metadata_map
+          @content_view = find_or_create_import_view
         end
 
-        def repository_mapping
-          mapping = {}
-          @metadata[:repositories].each do |key, value|
-            repo = @content_view_version.importable_repositories.joins(:root, :product).
-                        where("#{::Katello::Product.table_name}" => {:label => value[:product][:label]},
-                                                  "#{::Katello::RootRepository.table_name}" => {:label => value[:label]}).first
-            next unless repo&.version_href
-            repo_info = fetch_repository_info(repo.version_href)
-            mapping[key] = repo_info.name
-          end
-          mapping
-        end
-
-        def create_importer
-          api.importer_api.create(name: generate_id,
-                                  repo_mapping: repository_mapping)
+        def create_importer(content_view_version)
+          api.importer_api.create(name: generate_id(content_view_version),
+                                  repo_mapping: repository_mapping(content_view_version))
         end
 
         def create_import(importer_href)
-          [api.import_api.create(importer_href, toc: "#{@path}/#{@metadata[:toc]}")]
+          [api.import_api.create(importer_href, toc: "#{@path}/#{@metadata_map.toc}")]
         end
 
         def fetch_import(importer_href)
@@ -43,53 +33,32 @@ module Katello
           api.importer_api.delete(importer_href)
         end
 
-        def self.check!(content_view:, metadata:, path:, smart_proxy:)
-          ImportValidator.new(smart_proxy: smart_proxy,
-                               content_view: content_view,
-                               metadata: metadata,
-                               path: path).check!
-        end
-
-        def self.create_or_update_gpg!(organization:, params:)
-          return if params.blank?
-          gpg = organization.gpg_keys.find_by(:name => params[:name])
-          if gpg
-            gpg.update!(params.except(:name))
-          else
-            gpg = organization.gpg_keys.create!(params)
-          end
-          gpg
-        end
-
-        def self.metadata_map(metadata, product_only: false, custom_only: false, redhat_only: false)
-          # Create a map that looks like -> {[product, repo]: {name: 'Foo Repo', label:.....}}
-          # these values should be curated from the metadata.
-          metadata_map = {}
-          metadata[:repositories].values.each do |repo|
-            next if (custom_only && repo[:redhat]) || (redhat_only && !repo[:redhat])
-            if product_only
-              metadata_map[repo[:product][:label]] = repo[:product]
-            else
-              metadata_map[[repo[:product][:label], repo[:label]]] = repo
-            end
-          end
-          metadata_map
+        def check!
+          ImportValidator.new(import: self).check!
         end
 
-        def self.intersecting_repos_library_and_metadata(organization:, metadata:)
+        def intersecting_repos_library_and_metadata
           # Returns repositories in library that are part of the metadata
           # In other words if metadata had repos {label:foo, product: bar}
           # this would match it to the repo with the label foo and product bar
           # in the library.
-          queries = metadata_map(metadata).keys.map do |product_label, repo_label|
-            repositories_in_library(organization).
-                        where("#{Katello::Product.table_name}.label": product_label,
-                              "#{Katello::RootRepository.table_name}.label": repo_label)
+          queries = metadata_map.repositories.map do |repo|
+            if repo.redhat && repo.product.cp_id && repo.content&.id
+              library_repositories.where("#{Katello::Product.table_name}.cp_id": repo.product.cp_id,
+                                         "#{::Katello::RootRepository.table_name}" => {
+                                           content_id: repo.content.id,
+                                           arch: repo.arch,
+                                           major: repo.major,
+                                           minor: repo.minor })
+            else
+              library_repositories.where("#{Katello::Product.table_name}.label": repo.product.label,
+                                         "#{Katello::RootRepository.table_name}.label": repo.label)
+            end
           end
           queries.inject(&:or)
         end
 
-        def self.repositories_in_library(organization)
+        def library_repositories
           Katello::Repository.
                     in_default_view.
                     exportable.
@@ -98,7 +67,7 @@ module Katello
                     where("#{::Katello::ContentView.table_name}.organization_id": organization)
         end
 
-        def self.reset_content_view_repositories_from_metadata!(content_view:, metadata:)
+        def reset_content_view_repositories!
           # Given metadata from the dump and a content view
           # this method
           # 1) Fetches ids of the library repos whose product name, repo name amd redhat?
@@ -107,26 +76,90 @@ module Katello
           # 3) Adds the repositories matched from the dump
           # The main intent of this method is to assume that the user intends for the
           # content view to exaclty look like what is specified in metadata
-          repo_ids = intersecting_repos_library_and_metadata(organization: content_view.organization,
-                                                             metadata: metadata).
-                                                             pluck("#{Katello::Repository.table_name}.id")
+          repo_ids = intersecting_repos_library_and_metadata.pluck("#{Katello::Repository.table_name}.id")
           content_view.update!(repository_ids: repo_ids)
         end
 
-        def self.find_or_create_import_view(organization:, metadata:, library: false)
-          if library
-            metadata = { name: ::Katello::ContentView::IMPORT_LIBRARY,
-                         label: ::Katello::ContentView::IMPORT_LIBRARY,
-                         description: "Content View used for importing library"
-                        }
+        private
+
+        def repository_mapping(content_view_version)
+          mapping = {}
+          relation = content_view_version.importable_repositories.joins(:root, :product)
+
+          metadata_map.repositories.each do |metadata_repo|
+            if metadata_repo.redhat && metadata_repo.product.cp_id && metadata_repo.content&.id
+              repo = relation.where("#{::Katello::Product.table_name}" => {cp_id: metadata_repo.product.cp_id},
+                                    "#{::Katello::RootRepository.table_name}" => {
+                                      content_id: metadata_repo.content.id,
+                                      arch: metadata_repo.arch,
+                                      major: metadata_repo.major,
+                                      minor: metadata_repo.minor}).first
+            else
+              repo = relation.where("#{::Katello::Product.table_name}" => {label: metadata_repo.product.label},
+                                        "#{::Katello::RootRepository.table_name}" => {label: metadata_repo.label}).first
+            end
+
+            next unless repo&.version_href
+            repo_info = fetch_repository_info(repo.version_href)
+            mapping[metadata_repo.pulp_name] = repo_info.name
           end
+          mapping
+        end
+
+        def import_cv_name_from_export(name:, generated_for:)
+          if generated_for == :library_import
+            ::Katello::ContentView::IMPORT_LIBRARY
+          elsif generated_for == :repository_import
+            name.gsub(/^Export/, 'Import')
+          else
+            name
+          end
+        end
+
+        def import_content_view_params
+          generated_for = metadata_map.content_view.generated_for
+
+          if generated_for.blank?
+            generated_for = if metadata_map.content_view.label.start_with?(::Katello::ContentView::EXPORT_LIBRARY)
+                              "library_export"
+                            else
+                              "none"
+                            end
+          end
+
+          generated_for = generated_for.to_sym
+
+          if generated_for == :none
+            return {
+              name: metadata_map.content_view.name,
+              label: metadata_map.content_view.label,
+              description: metadata_map.content_view.description,
+              generated_for: generated_for
+            }
+          end
+
+          if generated_for == :library_export
+            generated_for = :library_import
+          elsif generated_for == :repository_export
+            generated_for = :repository_import
+          end
+
+          {
+            name: import_cv_name_from_export(name: metadata_map.content_view.name, generated_for: generated_for),
+            label: import_cv_name_from_export(name: metadata_map.content_view.label, generated_for: generated_for),
+            description: "Content View used for importing into library",
+            generated_for: generated_for
+          }
+        end
 
-          fail _("Content View label not provided.") if metadata[:label].blank?
+        def find_or_create_import_view
+          fail _("Content View label not provided.") if metadata_map.content_view.label.blank?
 
-          cv = ::Katello::ContentView.find_by(label: metadata[:label],
+          params = import_content_view_params
+          cv = ::Katello::ContentView.find_by(label: params[:label],
                                               organization: organization)
           if cv.blank?
-            ::Katello::ContentView.create!(metadata.merge(organization: organization, import_only: true))
+            ::Katello::ContentView.create!(params.merge(organization: organization, import_only: true))
           elsif !cv.import_only?
             msg = _("Unable to import in to Content View specified in the metadata - '%{name}'. "\
                      "The 'import_only' attribute for the content view is set to false. "\
@@ -136,7 +169,7 @@ module Katello
             command = "foreman-rake katello:set_content_view_import_only ID=#{cv.id}"
             fail msg + "\n" + command
           else
-            cv.update!(description: cv_metadata[:description]) if cv.description != metadata[:description]
+            cv.update!(description: params[:description]) if cv.description != params[:description]
             cv
           end
         end

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -2,12 +2,12 @@ module Katello
   module Pulp3
     module ContentViewVersion
       module ImportExportCommon
-        def generate_name
-          @content_view_version.name.gsub(/\s/, '_')
+        def generate_name(content_view_version)
+          content_view_version.name.gsub(/\s/, '_')
         end
 
-        def generate_id
-          "#{@content_view_version.organization.label}_#{generate_name}"
+        def generate_id(content_view_version)
+          "#{content_view_version.organization.label}_#{generate_name(content_view_version)}"
         end
 
         def api

data/app/services/katello/pulp3/content_view_version/import_gpg_keys.rb

@@ -2,28 +2,30 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class ImportGpgKeys
-        attr_accessor :organization, :metadata
-
-        def initialize(organization:, metadata:)
-          self.organization = organization
-          self.metadata = metadata
+        def initialize(organization:, metadata_gpg_keys:)
+          @organization = organization
+          @metadata_gpg_keys = metadata_gpg_keys
         end
 
         def create_or_update_gpg!(params)
-          return if params.blank?
-          gpg = organization.gpg_keys.find_by(:name => params[:name])
+          gpg = @organization.gpg_keys.find_by(:name => params[:name])
           if gpg
             gpg.update!(params.except(:name))
           else
-            gpg = organization.gpg_keys.create!(params)
+            gpg = @organization.gpg_keys.create!(params)
           end
           gpg
         end
 
         def import!
-          return if metadata[:gpg_keys].blank?
-          metadata[:gpg_keys].values.each do |gpg|
-            create_or_update_gpg!(gpg)
+          @metadata_gpg_keys.each do |gpg|
+            params = {
+              name: gpg.name,
+              content_type: ::Katello::ContentCredential::GPG_KEY_TYPE,
+              content: gpg.content
+            }
+
+            create_or_update_gpg!(params)
           end
         end
       end