katello 4.7.4 → 4.8.0.rc1

data/app/services/katello/pulp/repository/docker.rb

@@ -1,82 +0,0 @@
-module Katello
-  module Pulp
-    class Repository
-      class Docker < ::Katello::Pulp::Repository
-        REPOSITORY_TYPE = 'docker'.freeze
-
-        def unit_type_id(uploads = [])
-          uploads.pluck('digest').any? ? 'docker_tag' : super
-        end
-
-        def unit_keys(uploads)
-          uploads.map do |upload|
-            upload.except('id')
-          end
-        end
-
-        def generate_primary_importer
-          config = {
-            feed: root.url,
-            upstream_name: root.docker_upstream_name,
-            tags: root.include_tags,
-            enable_v1: false
-          }
-          importer_class.new(config.merge(primary_importer_connection_options))
-        end
-
-        #what foreman proxies w/ content pull docker content from
-        def docker_registry_host
-          if SmartProxy.pulp_primary.pulp3_repository_type_support?(Katello::Repository::DOCKER_TYPE)
-            foreman_url = URI.parse(Setting[:foreman_url]).host.downcase
-            "https://#{foreman_url}"
-          else
-            pulp_uri = URI.parse(SmartProxy.pulp_primary.pulp_url)
-            "https://#{pulp_uri.host.downcase}:#{Setting['pulp_docker_registry_port']}"
-          end
-        end
-
-        def generate_mirror_importer
-          config = {
-            feed: docker_registry_host,
-            upstream_name: repo.container_repository_name,
-            enable_v1: false
-          }
-          importer_class.new(config.merge(mirror_importer_connection_options))
-        end
-
-        def generate_distributors
-          [Runcible::Models::DockerDistributor.new(:protected => !root.unprotected,
-                                                  :id => repo.pulp_id,
-                                                  :auto_publish => true,
-                                                  :repo_registry_id => repo.container_repository_name)]
-        end
-
-        def distributors_to_publish(_options)
-          {Runcible::Models::DockerDistributor => {}}
-        end
-
-        def external_url(_force_https = false)
-          pulp_uri = URI.parse(SmartProxy.pulp_primary.pulp_url)
-          "#{pulp_uri.host.downcase}:#{Setting['pulp_docker_registry_port']}/#{repo.container_repository_name}"
-        end
-
-        def importer_class
-          Runcible::Models::DockerImporter
-        end
-
-        def copy_contents(destination_repo, options = {})
-          if options[:filters]
-            clause_gen = ::Katello::Util::DockerManifestClauseGenerator.new(@repo, options[:filters])
-            clause_gen.generate
-            criteria = {filters: {:unit => clause_gen.copy_clause } }
-          end
-
-          @smart_proxy.pulp_api.extensions.docker_tag.copy(
-            @repo.pulp_id,
-            destination_repo.pulp_id,
-            criteria || {})
-        end
-      end
-    end
-  end
-end

data/app/services/katello/pulp/repository/file.rb

@@ -1,52 +0,0 @@
-module Katello
-  module Pulp
-    class Repository
-      class File < ::Katello::Pulp::Repository
-        REPOSITORY_TYPE = 'iso'.freeze
-
-        def unit_keys(uploads)
-          uploads.map do |upload|
-            upload.except('id', 'content_unit_id')
-          end
-        end
-
-        def generate_primary_importer
-          config = { feed: root.url }
-          importer_class.new(config.merge(primary_importer_connection_options))
-        end
-
-        def generate_mirror_importer
-          config = {
-            feed: external_url
-          }
-          importer_class.new(config.merge(mirror_importer_connection_options))
-        end
-
-        def generate_distributors
-          yum_dist_id = repo.pulp_id
-          options = {
-            id: yum_dist_id,
-            auto_publish: true
-          }
-          [Runcible::Models::IsoDistributor.new(repo.relative_path, repo.unprotected, true, options)]
-        end
-
-        def partial_repo_path
-          "/pulp/isos/#{repo.relative_path}/"
-        end
-
-        def importer_class
-          Runcible::Models::IsoImporter
-        end
-
-        def distributors_to_publish(_options)
-          {Runcible::Models::IsoDistributor => {}}
-        end
-
-        def copy_contents(destination_repo, _options = {})
-          @smart_proxy.pulp_api.extensions.file.copy(@repo.pulp_id, destination_repo.pulp_id, {})
-        end
-      end
-    end
-  end
-end

data/app/services/katello/pulp/repository/yum.rb

@@ -1,205 +0,0 @@
-module Katello
-  module Pulp
-    class Repository
-      class Yum < ::Katello::Pulp::Repository
-        REPOSITORY_TYPE = 'yum'.freeze
-
-        def generate_primary_importer
-          config = {
-            download_policy: root.download_policy,
-            remove_missing: root.mirror_on_sync?,
-            feed: root.url,
-            type_skip_list: root.ignorable_content
-          }
-          importer_class.new(config.merge(primary_importer_connection_options))
-        end
-
-        def generate_mirror_importer
-          config = {
-            download_policy: smart_proxy_download_policy,
-            remove_missing: true,
-            feed: external_url(true)
-          }
-          importer_class.new(config.merge(mirror_importer_connection_options))
-        end
-
-        def partial_repo_path
-          "/pulp/content/#{repo.relative_path}/".sub('//', '/')
-        end
-
-        def importer_class
-          Runcible::Models::YumImporter
-        end
-
-        def generate_distributors
-          yum_dist_id = repo.pulp_id
-          options = {
-            protected: true,
-            id: yum_dist_id,
-            auto_publish: true
-          }
-          if smart_proxy.pulp_primary?
-            options[:checksum_type] = repo.saved_checksum_type || root.checksum_type
-          else
-            options[:checksum_type] = nil
-          end
-          distributors = [Runcible::Models::YumDistributor.new(repo.relative_path, root.unprotected, true, options)]
-
-          if smart_proxy.pulp_primary?
-            distributors << Runcible::Models::YumCloneDistributor.new(:id => "#{repo.pulp_id}_clone",
-                                                                   :destination_distributor_id => yum_dist_id)
-            distributors << Runcible::Models::ExportDistributor.new(false, false, repo.relative_path)
-          end
-          distributors
-        end
-
-        def distributors_to_publish(options)
-          source_repo_id = options[:source_repository]&.fetch(:id)
-          if (source_repo_id || !repo.primary?) && smart_proxy.pulp_primary?
-            source_repository = source_repo_id ? ::Katello::Repository.find(source_repo_id) : repo.target_repository
-            source_service = source_repository.backend_service(smart_proxy)
-            source_distributor_id = source_service.lookup_distributor_id(Runcible::Models::YumDistributor.type_id)
-            {Runcible::Models::YumCloneDistributor => {source_repo_id: source_repository.pulp_id,
-                                                       source_distributor_id: source_distributor_id}}
-          else
-            {Runcible::Models::YumDistributor => {}}
-          end
-        end
-
-        def smart_proxy_download_policy
-          policy = smart_proxy.download_policy || Setting[:default_proxy_download_policy]
-          if policy == ::SmartProxy::DOWNLOAD_INHERIT
-            self.root.download_policy
-          else
-            policy
-          end
-        end
-
-        def regenerate_applicability
-          smart_proxy.pulp_api.extensions.repository.regenerate_applicability_by_ids([repo.pulp_id], true)
-        end
-
-        def generate_mapping(destination_repo)
-          source_repo_map = {}
-          repo.siblings.yum_type.each do |sibling|
-            key = sibling.library_instance? ? sibling.id : sibling.library_instance_id
-            source_repo_map[key] = sibling.pulp_id
-          end
-
-          Hash[destination_repo.siblings.yum_type.map { |sibling| [source_repo_map[sibling.library_instance_id], sibling.pulp_id] }]
-        end
-
-        def build_override_config(destination_repo, incremental_update: false,
-                                                    solve_dependencies: false,
-                                                    filters: [])
-          config = {}
-          if incremental_update ||
-              (filters.present? && solve_dependencies)
-            if Setting[:dependency_solving_algorithm] == 'greedy'
-              config[:recursive] = true
-            else
-              config[:recursive_conservative] = true
-            end
-            config[:additional_repos] = generate_mapping(destination_repo)
-          end
-          config
-        end
-
-        def import_distribution_data
-          distribution = smart_proxy.pulp_api.extensions.repository.distributions(repo.pulp_id).first
-          if distribution
-            repo.update!(
-              :distribution_version => distribution["version"],
-              :distribution_arch => distribution["arch"],
-              :distribution_family => distribution["family"],
-              :distribution_variant => distribution["variant"],
-              :distribution_bootable => self.class.distribution_bootable?(distribution)
-            )
-          end
-        end
-
-        def self.distribution_bootable?(distribution)
-          # Not every distribution from Pulp represents a bootable
-          # repo. Determine based on the files in the repo.
-          distribution["files"].any? do |file|
-            if file.is_a? Hash
-              filename = file[:relativepath]
-            else
-              filename = file
-            end
-            filename.include?('vmlinuz') || filename.include?('pxeboot') || filename.include?('kernel.img') || filename.include?('initrd.img')
-          end
-        end
-
-        def purge_empty_contents
-          [purge_partial_errata, purge_empty_package_groups]
-        end
-
-        def should_purge_empty_contents?
-          true
-        end
-
-        def purge_partial_errata
-          task = nil
-          repo.remove_partial_errata! do |errata_to_delete|
-            task = repo.unassociate_by_filter(::Katello::ContentViewErratumFilter::CONTENT_TYPE,
-                                                "id" => { "$in" => errata_to_delete.map(&:errata_id).sort })
-          end
-          task
-        end
-
-        private
-
-        def purge_empty_package_groups
-          rpm_names = repo.rpms.pluck(:name).uniq
-
-          # Remove all  package groups with no packages
-          package_groups_to_delete = repo.package_groups.select do |group|
-            (rpm_names & group.package_names).empty?
-          end
-
-          repo.repository_package_groups.where(:package_group_id => package_groups_to_delete.map(&:id)).delete_all
-
-          criteria = {:association => {"unit_id" => {"$in" => package_groups_to_delete.compact}}}
-          smart_proxy.pulp_api.extensions.repository.unassociate_units(repo.pulp_id, :filters => criteria)
-        end
-
-        def copy_module_contents(destination_repo, filters:, solve_dependencies:)
-          override_config = build_override_config(destination_repo,
-                                                  filters: filters,
-                                                  solve_dependencies: solve_dependencies)
-
-          filters = filters.module_stream.or(filters.errata) if filters
-
-          copy_clauses, remove_clauses = generate_module_stream_copy_clauses(filters)
-          tasks = [smart_proxy.pulp_api.extensions.module.copy(repo.pulp_id, destination_repo.pulp_id,
-                   copy_clauses.merge(:override_config => override_config))]
-
-          if remove_clauses
-            tasks << smart_proxy.pulp_api.extensions.repository.unassociate_units(destination_repo.pulp_id,
-                                                                         type_ids: [::Katello::Pulp::ModuleStream::CONTENT_TYPE],
-                                                                          filters: {unit: remove_clauses})
-          end
-          tasks
-        end
-
-        def generate_module_stream_copy_clauses(filters)
-          if filters&.any?
-            clause_gen = ::Katello::Util::ModuleStreamClauseGenerator.new(repo, filters)
-            clause_gen.generate
-
-            copy = clause_gen.copy_clause
-            copy_clauses = {filters: {unit: copy }} if copy
-
-            remove = clause_gen.remove_clause
-            remove_clauses = {filters: {unit: remove}} if remove
-          else
-            copy_clauses = {}
-            remove_clauses = nil
-          end
-          [copy_clauses, remove_clauses]
-        end
-      end
-    end
-  end
-end

data/app/services/katello/pulp/repository.rb

@@ -1,327 +0,0 @@
-module Katello
-  module Pulp
-    class Repository < ::Actions::Pulp::Abstract
-      include ImporterComparison
-      attr_accessor :repo, :input
-      attr_accessor :smart_proxy
-      delegate :root, to: :repo
-
-      def initialize(repo, smart_proxy)
-        @repo = repo
-        @smart_proxy = smart_proxy
-      end
-
-      def backend_data(force = false)
-        if (repo.pulp_id && (force || @backend_data.nil?))
-          @backend_data = smart_proxy.pulp_api.extensions.repository.retrieve_with_details(repo.pulp_id)
-        end
-        @backend_data
-      rescue RestClient::ResourceNotFound
-        nil
-      end
-
-      def self.instance_for_type(repo, smart_proxy)
-        Katello::RepositoryTypeManager.enabled_repository_types[repo.root.content_type].service_class.new(repo, smart_proxy)
-      end
-
-      def unit_type_id(_uploads = [])
-        @repo.unit_type_id
-      end
-
-      def unit_keys(uploads)
-        uploads.map do |upload|
-          upload.except('id', 'name')
-        end
-      end
-
-      def partial_repo_path
-        fail NotImplementedError
-      end
-
-      def importer_class
-        fail NotImplementedError
-      end
-
-      def primary_importer_configuration
-        fail NotImplementedError
-      end
-
-      def mirror_importer_configuration
-        fail NotImplementedError
-      end
-
-      def generate_mirror_importer
-        fail NotImplementedError
-      end
-
-      def distributors_to_publish
-        fail NotImplementedError
-      end
-
-      def generate_primary_importer
-        fail NotImplementedError
-      end
-
-      def generate_distributors
-        fail NotImplementedError
-      end
-
-      def regenerate_applicability
-        fail NotImplementedError
-      end
-
-      def copy_contents(_destination_repo, _filters)
-        fail NotImplementedError
-      end
-
-      def content_service
-        Katello::Pulp::Content
-      end
-
-      def should_purge_empty_contents?
-        false
-      end
-
-      def create_mirror_entities
-        create
-      end
-
-      def mirror_needs_updates?
-        needs_update?
-      end
-
-      def needs_update?
-        needs_importer_updates? || needs_distributor_updates?
-      end
-
-      def sync(overrides = {})
-        sync_options = {}
-        sync_options[:max_speed] = SETTINGS.dig(:katello, :pulp, :sync_KBlimit)
-        sync_options[:num_threads] = SETTINGS.dig(:katello, :pulp, :sync_threads)
-        sync_options[:feed] = overrides[:feed] if overrides[:feed]
-        sync_options[:validate] = !SETTINGS.dig(:katello, :pulp, :skip_checksum_validation)
-        sync_options.merge!(overrides[:options]) if overrides[:options]
-        [smart_proxy.pulp_api.resources.repository.sync(@repo.pulp_id, override_config: sync_options.compact!)]
-      end
-
-      def create
-        smart_proxy.pulp_api.extensions.repository.create_with_importer_and_distributors(
-          repo.pulp_id,
-          generate_importer,
-          generate_distributors,
-          display_name: root.name)
-      end
-
-      def delete
-        smart_proxy.pulp_api.extensions.repository.delete(repo.pulp_id)
-      rescue RestClient::ResourceNotFound
-        Rails.logger.warn("Tried to delete repository #{repo.pulp_id}, but it did not exist.")
-        []
-      end
-
-      def external_url(force_https = false)
-        uri = URI.parse(::SmartProxy.pulp_primary.pulp_url)
-        uri.scheme = (root.unprotected && !force_https) ? 'http' : 'https'
-        uri.path = partial_repo_path
-        uri.to_s
-      end
-
-      def generate_importer
-        if smart_proxy.pulp_mirror?
-          generate_mirror_importer
-        elsif repo.in_default_view?
-          generate_primary_importer
-        else #content view repositories don't need any importer configuration
-          importer_class.new
-        end
-      end
-
-      def needs_importer_updates?
-        repo_details = backend_data
-        return unless repo_details
-        capsule_importer = repo_details["importers"][0]
-        !importer_matches?(generate_importer, capsule_importer)
-      end
-
-      def needs_distributor_updates?
-        repo_details = backend_data
-        return unless repo_details
-        !distributors_match?(repo_details["distributors"])
-      end
-
-      def primary_importer_connection_options
-        options = {
-          basic_auth_username: root.upstream_username,
-          basic_auth_password: root.upstream_password,
-          ssl_validation: root.verify_ssl_on_sync?
-        }
-        options.merge!(proxy_options)
-        options.merge!(primary_importer_ssl_options)
-      end
-
-      def primary_importer_ssl_options
-        if root.redhat? && Katello::Resources::CDN::CdnResource.redhat_cdn?(root.url)
-          {
-            ssl_client_cert: root.product.certificate,
-            ssl_client_key: root.product.key,
-            ssl_ca_cert: Katello::Repository.feed_ca_cert(root.url)
-          }
-        elsif root.custom?
-          {
-            ssl_client_cert: root.ssl_client_cert&.content,
-            ssl_client_key: root.ssl_client_key&.content,
-            ssl_ca_cert: root.ssl_ca_cert&.content
-          }
-        else
-          {}
-        end
-      end
-
-      def mirror_importer_connection_options
-        ueber_cert = ::Cert::Certs.ueber_cert(root.organization)
-        {
-          ssl_client_cert: ueber_cert[:cert],
-          ssl_client_key: ueber_cert[:key],
-          ssl_ca_cert: ::Cert::Certs.ca_cert,
-          ssl_validation: true
-        }
-      end
-
-      def distributor_publish(options)
-        distributors_to_publish(options).map do |distributor_class, config|
-          config[:force_full] = options.fetch(:force, false)
-          smart_proxy.pulp_api.extensions.repository.publish(repo.pulp_id, lookup_distributor_id(distributor_class.type_id),
-                                                             override_config: config)
-        end
-      end
-
-      def lookup_distributor_id(distributor_type_id)
-        distributor = backend_data["distributors"].find do |dist|
-          dist["distributor_type_id"] == distributor_type_id
-        end
-        distributor['id']
-      end
-
-      def refresh_if_needed
-        refresh if needs_update?
-      end
-
-      def refresh
-        tasks = update_or_associate_importer
-        tasks += update_or_associate_distributors
-        tasks += remove_unnecessary_distributors
-        tasks
-      end
-
-      def refresh_mirror_entities
-        refresh_if_needed
-      end
-
-      def update_or_associate_importer
-        existing_importers = backend_data["importers"]
-        importer = generate_importer
-        found = existing_importers.find { |i| i['importer_type_id'] == importer.id }
-
-        tasks = []
-        if found
-          ssl_ca_cert = importer.config.delete('ssl_ca_cert')
-          ssl_client_cert = importer.config.delete('ssl_client_cert')
-          ssl_client_key = importer.config.delete('ssl_client_key')
-          importer.config['basic_auth_username'] = nil if importer.config['basic_auth_username'].blank?
-          importer.config['basic_auth_password'] = nil if importer.config['basic_auth_password'].blank?
-          # Update ssl options by themselves workaround for https://pulp.plan.io/issues/2727
-          tasks << smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], :ssl_client_cert => ssl_client_cert,
-                                                    :ssl_client_key => ssl_client_key, :ssl_ca_cert => ssl_ca_cert)
-          tasks << smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], importer.config)
-        else
-          tasks << smart_proxy.pulp_api.resources.repository.associate_importer(repo.pulp_id, repo.importers.first['importer_type_id'], importer.config)
-        end
-        tasks
-      end
-
-      def update_or_associate_distributors
-        tasks = []
-        existing_distributors = backend_data["distributors"]
-        generate_distributors.each do |distributor|
-          found = existing_distributors.find { |i| i['distributor_type_id'] == distributor.type_id }
-          if found
-            tasks << smart_proxy.pulp_api.resources.repository.update_distributor(repo.pulp_id, found['id'], distributor.config)
-          else
-            smart_proxy.pulp_api.resources.repository.
-                associate_distributor(repo.pulp_id, distributor.type_id, distributor.config, :distributor_id => distributor.id,
-                                      :auto_publish => distributor.auto_publish)
-          end
-        end
-        tasks
-      end
-
-      def remove_unnecessary_distributors
-        tasks = []
-        existing_distributors = backend_data["distributors"]
-        generated_distributors = generate_distributors
-        existing_distributors.each do |distributor|
-          found = generated_distributors.find { |dist| dist.type_id == distributor['distributor_type_id'] }
-          tasks << smart_proxy.pulp_api.resources.repository.delete_distributor(repo.pulp_id, distributor['id']) unless found
-        end
-        tasks
-      end
-
-      def copy_units(destination_repo, units, incremental_update: false)
-        override_config = {}
-        override_config = build_override_config(destination_repo, incremental_update: incremental_update) if respond_to? :build_override_config
-        content_type = units.first.class::CONTENT_TYPE
-        unit_ids = units.pluck(:pulp_id)
-        smart_proxy.pulp_api.extensions.send(content_type).copy(repo.pulp_id, destination_repo.pulp_id, ids: unit_ids, override_config: override_config)
-      end
-
-      def content_unit_types
-        Katello::RepositoryTypeManager.find(repo.content_type)
-      end
-
-      def proxy_options
-        if repo.root.http_proxy_policy == RootRepository::NO_DEFAULT_HTTP_PROXY
-          return { proxy_host: '' }
-        end
-
-        proxy = repo.root.http_proxy
-        if repo.root.http_proxy_policy == RootRepository::GLOBAL_DEFAULT_HTTP_PROXY
-          proxy = HttpProxy.default_global_content_proxy
-        end
-
-        if proxy
-          uri = URI(proxy.url)
-          proxy_options = {
-            proxy_host: uri.scheme + '://' + uri.host,
-            proxy_port: uri.port,
-            proxy_username: proxy.username,
-            proxy_password: proxy.password
-          }
-          return proxy_options
-        end
-        { proxy_host: '' }
-      end
-
-      private
-
-      def filtered_distribution_config_equal?(generated_config, actual_config)
-        generated = generated_config.clone
-        actual = actual_config.clone
-        #We store 'default' checksum type as nil, but pulp will default to sha256, so if we haven't set it, ignore it
-        if generated.keys.include?('checksum_type') && generated['checksum_type'].nil?
-          generated.delete('checksum_type')
-          actual.delete('checksum_type')
-        end
-        generated.compact == actual.compact
-      end
-
-      def distributors_match?(capsule_distributors)
-        generated_distributor_configs = generate_distributors
-        generated_distributor_configs.all? do |gen_dist|
-          type = gen_dist.class.type_id
-          found_on_capsule = capsule_distributors.find { |dist| dist['distributor_type_id'] == type }
-          found_on_capsule && filtered_distribution_config_equal?(gen_dist.config, found_on_capsule['config'])
-        end
-      end
-    end
-  end
-end

data/app/services/katello/pulp/server.rb

@@ -1,35 +0,0 @@
-module Katello
-  module Pulp
-    class Server
-      def self.config(url, user_remote_id)
-        uri = URI.parse(url)
-
-        runcible_params = {
-          :url => "#{uri.scheme}://#{uri.host.downcase}:#{uri.port}",
-          :api_path => uri.path,
-          :user => user_remote_id,
-          :timeout => SETTINGS[:katello][:rest_client_timeout],
-          :open_timeout => SETTINGS[:katello][:rest_client_timeout],
-          :logging => {
-            :logger => ::Foreman::Logging.logger('katello/pulp_rest'),
-            :exception => true,
-            :info => true,
-            :debug => true
-          },
-          :cert_auth => {
-            :ssl_client_cert => '/etc/pki/katello/certs/pulp-client.crt',
-            :ssl_client_key => '/etc/pki/katello/private/pulp-client.key'
-          }
-        }
-
-        if (ca_cert = SETTINGS[:katello][:pulp][:ca_cert_file])
-          runcible_params[:ca_cert_file] = ca_cert
-        end
-
-        runcible_params[:verify_ssl] = SETTINGS[:katello][:pulp][:verify_ssl] if SETTINGS[:katello][:pulp].key?(:verify_ssl)
-
-        Runcible::Instance.new(runcible_params)
-      end
-    end
-  end
-end