katello 4.5.0.rc2 → 4.6.0.rc2

data/app/services/cert/certs.rb

@@ -12,28 +12,20 @@ module Cert
       File.read(SETTINGS[:katello][:candlepin][:ca_cert_file])
     end
 
-    def self.ssl_client_cert(use_admin_as_cn_cert: false)
-      @ssl_client_cert ||= OpenSSL::X509::Certificate.new(File.read(ssl_client_cert_filename(use_admin_as_cn_cert: use_admin_as_cn_cert)))
+    def self.ssl_client_cert
+      @ssl_client_cert ||= OpenSSL::X509::Certificate.new(File.read(ssl_client_cert_filename))
     end
 
-    def self.ssl_client_cert_filename(use_admin_as_cn_cert: false)
-      if use_admin_as_cn_cert
-        Setting[:pulp_client_cert]
-      else
-        Setting[:ssl_certificate]
-      end
+    def self.ssl_client_cert_filename
+      Setting[:ssl_certificate]
     end
 
-    def self.ssl_client_key(use_admin_as_cn_cert: false)
-      @ssl_client_key ||= OpenSSL::PKey::RSA.new(File.read(ssl_client_key_filename(use_admin_as_cn_cert: use_admin_as_cn_cert)))
+    def self.ssl_client_key
+      @ssl_client_key ||= OpenSSL::PKey::RSA.new(File.read(ssl_client_key_filename))
     end
 
-    def self.ssl_client_key_filename(use_admin_as_cn_cert: false)
-      if use_admin_as_cn_cert
-        Setting[:pulp_client_key]
-      else
-        Setting[:ssl_priv_key]
-      end
+    def self.ssl_client_key_filename
+      Setting[:ssl_priv_key]
     end
 
     def self.verify_ueber_cert(organization)

data/app/services/katello/organization_creator.rb

@@ -27,15 +27,16 @@ module Katello
       end
     end
 
-    def initialize(organization)
+    def initialize(organization, sca: true)
       @organization = organization
+      @content_access_mode = sca ? 'org_environment' : 'entitlement'
     end
 
     def seed!
       ActiveRecord::Base.transaction do
         @organization.setup_label_from_name
 
-        # existing validation errors are not resolvable here, so don't validatate
+        # existing validation errors are not resolvable here, so don't validate
         @organization.save(validate: false)
 
         create_library_environment
@@ -66,9 +67,8 @@ module Katello
 
     def create_backend_objects!
       Katello::Ping.ping!(services: [:candlepin])
-
       if needs_candlepin_organization?
-        ::Katello::Resources::Candlepin::Owner.create(@organization.label, @organization.name)
+        ::Katello::Resources::Candlepin::Owner.create(@organization.label, @organization.name, content_access_mode: @content_access_mode)
       end
 
       ::Katello::ContentViewManager.create_candlepin_environment(
@@ -101,12 +101,23 @@ module Katello
     end
 
     def create_library_view
-      @library_view = Katello::ContentView.where(
+      cv_wrong_label = Katello::ContentView.where(
         default: true,
         name: DEFAULT_CONTENT_VIEW_NAME,
-        label: DEFAULT_CONTENT_VIEW_LABEL,
         organization: @organization
-      ).first_or_create!
+      ).where.not(label: DEFAULT_CONTENT_VIEW_LABEL)&.first
+
+      if cv_wrong_label
+        cv_wrong_label.update_attribute(:label, DEFAULT_CONTENT_VIEW_LABEL)
+        @library_view = cv_wrong_label
+      else
+        @library_view = Katello::ContentView.where(
+          default: true,
+          name: DEFAULT_CONTENT_VIEW_NAME,
+          label: DEFAULT_CONTENT_VIEW_LABEL,
+          organization: @organization
+        ).first_or_create!
+      end
     end
 
     def create_library_cvv

data/app/services/katello/pulp/server.rb

@@ -17,8 +17,8 @@ module Katello
             :debug => true
           },
           :cert_auth => {
-            :ssl_client_cert => ::Cert::Certs.ssl_client_cert(use_admin_as_cn_pulp_cert: true),
-            :ssl_client_key => ::Cert::Certs.ssl_client_key(use_admin_as_cn_pulp_cert: true)
+            :ssl_client_cert => '/etc/pki/katello/certs/pulp-client.crt',
+            :ssl_client_key => '/etc/pki/katello/private/pulp-client.key'
           }
         }
 

data/app/services/katello/pulp3/alternate_content_source.rb

@@ -5,10 +5,12 @@ module Katello
       include Katello::Pulp3::ServiceCommon
       attr_accessor :acs
       attr_accessor :smart_proxy
+      attr_accessor :repository
 
-      def initialize(acs, smart_proxy)
+      def initialize(acs, smart_proxy, repository = nil)
         @acs = acs
         @smart_proxy = smart_proxy
+        @repository = repository
       end
 
       def api
@@ -20,10 +22,23 @@ module Katello
       end
 
       def smart_proxy_acs
-        ::Katello::SmartProxyAlternateContentSource.find_by(alternate_content_source_id: acs.id, smart_proxy_id: smart_proxy.id)
+        if acs.alternate_content_source_type == 'custom'
+          ::Katello::SmartProxyAlternateContentSource.find_by(alternate_content_source_id: acs.id, smart_proxy_id: smart_proxy.id)
+        else
+          ::Katello::SmartProxyAlternateContentSource.find_by(alternate_content_source_id: acs.id, smart_proxy_id: smart_proxy.id, repository_id: repository.id)
+        end
       end
 
       def remote_options
+        if repository.present?
+          options = repository.backend_service(smart_proxy).remote_options
+          options[:policy] = 'on_demand'
+          options[:proxy_url] = acs.http_proxy&.url
+          options[:proxy_username] = acs.http_proxy&.username
+          options[:proxy_password] = acs.http_proxy&.password
+          return options
+        end
+
         remote_options = {
           tls_validation: acs.verify_ssl,
           name: generate_backend_object_name,
@@ -66,8 +81,11 @@ module Katello
         api.remotes_api.partial_update(smart_proxy_acs.remote_href, remote_options)
       end
 
-      def delete_remote(href = smart_proxy_acs.remote_href)
-        ignore_404_exception { remote_options[:url]&.start_with?('uln') ? api.remotes_uln_api.delete(href) : api.remotes_api.delete(href) } if href
+      # The old repo URL is needed to determine which remote API to use.
+      def delete_remote(options = {})
+        options[:href] ||= smart_proxy_acs.remote_href
+        options[:old_url] ||= remote_options[:url]
+        ignore_404_exception { options[:old_url]&.start_with?('uln') ? api.remotes_uln_api.delete(options[:href]) : api.remotes_api.delete(options[:href]) } if options[:href]
       end
 
       def create

data/app/services/katello/pulp3/api/apt.rb

@@ -11,6 +11,18 @@ module Katello
         def publications_verbatim_api
           PulpDebClient::PublicationsVerbatimApi.new(api_client)
         end
+
+        def self.copy_class
+          PulpDebClient::Copy
+        end
+
+        def self.add_remove_content_class
+          PulpDebClient::RepositoryAddRemoveContent
+        end
+
+        def copy_api
+          PulpDebClient::CopyApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/api/core.rb

@@ -89,6 +89,10 @@ module Katello
           PulpcoreClient::RepositoriesReclaimSpaceApi.new(core_api_client)
         end
 
+        def yum_exporter_api
+          PulpcoreClient::ExportersFilesystemApi.new(core_api_client)
+        end
+
         def exporter_api
           PulpcoreClient::ExportersPulpApi.new(core_api_client)
         end
@@ -101,6 +105,10 @@ module Katello
           PulpcoreClient::ImportersPulpImportCheckApi.new(core_api_client)
         end
 
+        def yum_export_api
+          PulpcoreClient::ExportersFilesystemExportsApi.new(core_api_client)
+        end
+
         def export_api
           PulpcoreClient::ExportersPulpExportsApi.new(core_api_client)
         end

data/app/services/katello/pulp3/api/generic.rb

@@ -15,10 +15,6 @@ module Katello
         def self.repository_sync_url_class(repository_type)
           repository_type.repo_sync_url_class
         end
-
-        def self.add_remove_content_class
-          fail NotImplementedError
-        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -3,16 +3,31 @@ module Katello
     module ContentViewVersion
       class Export
         include ImportExportCommon
-        attr_reader :smart_proxy, :content_view_version, :destination_server, :from_content_view_version
+        SYNCABLE = "syncable".freeze
+        IMPORTABLE = "importable".freeze
+        FORMATS = [SYNCABLE, IMPORTABLE].freeze
+
+        attr_reader :smart_proxy, :content_view_version, :destination_server, :from_content_view_version, :repository, :base_path
+        def self.create(options)
+          if options.delete(:format) == SYNCABLE
+            SyncableFormatExport.new(options)
+          else
+            self.new(options)
+          end
+        end
 
         def initialize(smart_proxy:,
                         content_view_version: nil,
                         destination_server: nil,
-                        from_content_view_version: nil)
+                        from_content_view_version: nil,
+                        repository: nil,
+                        base_path: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
           @from_content_view_version = from_content_view_version
+          @repository = repository
+          @base_path = base_path
         end
 
         def repository_hrefs
@@ -37,21 +52,27 @@ module Katello
         end
 
         def generate_exporter_path
-          export_path = "#{content_view_version.content_view}/#{content_view_version.version}/#{destination_server}/#{date_dir}".gsub(/\s/, '_')
-          "#{content_view_version.organization.label}/#{export_path}"
+          return base_path if base_path
+          export_path = "#{content_view_version.content_view}/#{content_view_version.version}/"
+          export_path += "#{destination_server}/" unless destination_server.blank?
+          export_path += "#{date_dir}".gsub(/\s/, '_')
+          @base_path = "#{Setting['pulpcore_export_destination']}/#{content_view_version.organization.label}/#{export_path}"
         end
 
         def date_dir
           DateTime.now.to_s.gsub(/\W/, '-')
         end
 
-        def create_exporter(export_base_dir: Setting['pulpcore_export_destination'])
-          api.exporter_api.create(name: generate_id(content_view_version),
-                                  path: "#{export_base_dir}/#{generate_exporter_path}",
-                                  repositories: repository_hrefs)
+        def create_exporter
+          exporter_api = api.exporter_api
+          options = { name: generate_id(content_view_version), path: generate_exporter_path }
+          options[:repositories] = repository_hrefs
+
+          exporter_api.create(options)
         end
 
-        def create_export(exporter_href, chunk_size: nil)
+        def create_export(exporter_data, chunk_size: nil)
+          exporter_href = exporter_data[:pulp_href]
           options = { versions: version_hrefs }
           options[:chunk_size] = "#{chunk_size}GB" if chunk_size
           if from_content_view_version
@@ -77,14 +98,16 @@ module Katello
             options[:start_versions] = start_versions
             options[:full] = 'false'
           end
-          [api.export_api.create(exporter_href, options)]
+          export_api = api.export_api
+          [export_api.create(exporter_href, options)]
         end
 
         def fetch_export(exporter_href)
           api.export_api.list(exporter_href).results.first
         end
 
-        def destroy_exporter(exporter_href)
+        def destroy_exporter(exporter_data)
+          exporter_href = exporter_data[:pulp_href]
           export_data = fetch_export(exporter_href)
           api.exporter_api.partial_update(exporter_href, :last_export => nil)
           api.export_api.delete(export_data.pulp_href) unless export_data.blank?
@@ -162,23 +185,44 @@ module Katello
                                        generated_for: generated_for).send(select_method)
         end
 
+        def format
+          is_a?(SyncableFormatExport) ? SYNCABLE : IMPORTABLE
+        end
+
         def self.find_library_export_view(create_by_default: false,
                                           destination_server:,
-                                          organization:)
+                                          organization:,
+                                          format: IMPORTABLE)
+          if format == IMPORTABLE
+            generated_for = :library_export
+            name = ::Katello::ContentView::EXPORT_LIBRARY
+          else
+            generated_for = :library_export_syncable
+            name = "#{::Katello::ContentView::EXPORT_LIBRARY}-SYNCABLE"
+          end
+
           find_generated_export_view(create_by_default: create_by_default,
                                      destination_server: destination_server,
                                      organization: organization,
-                                     name: ::Katello::ContentView::EXPORT_LIBRARY,
-                                     generated_for: :library_export)
+                                     name: name,
+                                     generated_for: generated_for)
         end
 
         def self.find_repository_export_view(create_by_default: false,
-                                              repository:)
+                                              repository:,
+                                              format: IMPORTABLE)
+          if format == IMPORTABLE
+            generated_for = :repository_export
+            name = "Export-#{repository.label}-#{repository.library_instance_or_self.id}"
+          else
+            generated_for = :repository_export_syncable
+            name = "Export-SYNCABLE-#{repository.label}-#{repository.library_instance_or_self.id}"
+          end
           find_generated_export_view(create_by_default: create_by_default,
                                      destination_server: nil,
                                      organization: repository.organization,
-                                     name: "Export-#{repository.label}-#{repository.library_instance_or_self.id}",
-                                     generated_for: :repository_export)
+                                     name: name,
+                                     generated_for: generated_for)
         end
 
         def self.generate_product_repo_strings(repositories:)

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -42,11 +42,14 @@ module Katello
           # In other words if metadata had repos {label:foo, product: bar}
           # this would match it to the repo with the label foo and product bar
           # in the library.
-
           queries = metadata_map.repositories.map do |repo|
-            if repo.redhat && repo.product.cp_id
+            if repo.redhat && repo.product.cp_id && repo.content&.id
               library_repositories.where("#{Katello::Product.table_name}.cp_id": repo.product.cp_id,
-                                         "#{Katello::RootRepository.table_name}.label": repo.label)
+                                         "#{::Katello::RootRepository.table_name}" => {
+                                           content_id: repo.content.id,
+                                           arch: repo.arch,
+                                           major: repo.major,
+                                           minor: repo.minor })
             else
               library_repositories.where("#{Katello::Product.table_name}.label": repo.product.label,
                                          "#{Katello::RootRepository.table_name}.label": repo.label)
@@ -84,9 +87,13 @@ module Katello
           relation = content_view_version.importable_repositories.joins(:root, :product)
 
           metadata_map.repositories.each do |metadata_repo|
-            if metadata_repo.redhat && metadata_repo.product.cp_id
+            if metadata_repo.redhat && metadata_repo.product.cp_id && metadata_repo.content&.id
               repo = relation.where("#{::Katello::Product.table_name}" => {cp_id: metadata_repo.product.cp_id},
-                                        "#{::Katello::RootRepository.table_name}" => {label: metadata_repo.label}).first
+                                    "#{::Katello::RootRepository.table_name}" => {
+                                      content_id: metadata_repo.content.id,
+                                      arch: metadata_repo.arch,
+                                      major: metadata_repo.major,
+                                      minor: metadata_repo.minor}).first
             else
               repo = relation.where("#{::Katello::Product.table_name}" => {label: metadata_repo.product.label},
                                         "#{::Katello::RootRepository.table_name}" => {label: metadata_repo.label}).first

data/app/services/katello/pulp3/content_view_version/importable_repositories.rb

@@ -22,12 +22,21 @@ module Katello
             product = product_for_metadata_repo(repo)
             fail _("Unable to find product '%s' in organization '%s'" % [repo.product.label, @organization.name]) if product.blank?
 
-            root = product.root_repositories.find { |r| r.label == repo.label }
+            root = product.root_repositories.find do |r|
+              if repo.content&.id && repo.redhat
+                r.content.cp_content_id == repo.content.id &&
+                  r.arch == repo.arch &&
+                  r.major == repo.major &&
+                  r.minor == repo.minor
+              else
+                r.label == repo.label
+              end
+            end
+
             if root
               updatable << { repository: root, options: update_repo_params(repo) }
             elsif repo.redhat
-              content = repo.content
-              product_content = product_content_by_label(content.label)
+              product_content = fetch_product_content(repo.content, product)
               substitutions = {
                 basearch: repo.arch,
                 releasever: repo.minor
@@ -49,8 +58,14 @@ module Katello
           end
         end
 
-        def product_content_by_label(content_label)
-          ::Katello::Content.find_by_label(content_label)
+        def fetch_product_content(content_metadata, product)
+          query = ::Katello::Content.joins(:product_contents).where("#{Katello::ProductContent.table_name}.product_id": product.id)
+          table_name = Katello::Content.table_name
+          if content_metadata&.id
+            query.find_by("#{table_name}.cp_content_id": content_metadata.id)
+          else
+            query.find_by("#{table_name}.label": content_metadata.label)
+          end
         end
 
         def gpg_key_id(metadata_repo)

data/app/services/katello/pulp3/content_view_version/metadata_generator.rb

@@ -16,10 +16,12 @@ module Katello
 
         def generate!
           ret = { organization: organization.name,
+                  base_path: Setting['pulpcore_export_destination'],
                   repositories: {},
                   content_view: content_view.slice(:name, :label, :description, :generated_for),
                   content_view_version: content_view_version.slice(:major, :minor, :description),
-                  incremental: from_content_view_version.present?
+                  incremental: from_content_view_version.present?,
+                  format: export_service.format
           }
 
           unless from_content_view_version.blank?

data/app/services/katello/pulp3/content_view_version/syncable_format_export.rb

@@ -0,0 +1,34 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class SyncableFormatExport < Export
+        def create_exporter
+          api.yum_exporter_api.create(name: "#{generate_id(content_view_version)}-#{repository.id}",
+                                      path: generate_repository_exporter_path,
+                                      method: :hardlink)
+        end
+
+        def create_export(exporter_data, _options = {})
+          [api.yum_export_api.create(exporter_data[:pulp_href], publication: repository.publication_href)]
+        end
+
+        def fetch_export(exporter_href)
+          api.yum_export_api.list(exporter_href).results.first
+        end
+
+        def destroy_exporter(exporter_data)
+          exporter_href = exporter_data[:pulp_href]
+          export_data = fetch_export(exporter_href)
+          api.yum_export_api.delete(export_data.pulp_href)
+          api.yum_exporter_api.delete(exporter_href)
+        end
+
+        def generate_repository_exporter_path
+          _org, _content, content_path = repository.library_instance_or_self.relative_path.split("/", 3)
+          content_path = content_path.sub(%r|^/|, '')
+          "#{generate_exporter_path}/#{content_path}".gsub(/\s/, '_')
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/deb.rb

@@ -9,6 +9,7 @@ module Katello
       end
 
       def self.content_api_create(opts = {})
+        opts.delete(:relative_path) if opts.key?(:relative_path)
         self.content_api.create(opts)
       end
 
@@ -24,7 +25,8 @@ module Katello
       end
 
       def self.generate_model_row(unit)
-        {
+        unit = unit.try(:with_indifferent_access)
+        return {
           pulp_id: unit[unit_identifier],
           checksum: unit[:sha256],
           filename: unit[:relative_path],

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -146,6 +146,8 @@ module Katello
         elsif repository.generic?
           duplicate_sha_path_content_list = content_backend_service.content_api(repository.repository_type, unit_type_id).list(
             filter_label => checksum)
+        elsif unit_type_id == 'deb'
+          duplicate_sha_path_content_list = content_backend_service.content_api.list(filter_label => checksum)
         else
           duplicate_sha_path_content_list = content_backend_service.content_api.list(
             filter_label => checksum,