katello 3.11.2 → 3.12.0.rc1

data/app/services/katello/pulp/repository/puppet.rb

@@ -19,7 +19,7 @@ module Katello
         end
 
         def generate_distributors
-          puppet_install_dist = Runcible::Models::PuppetInstallDistributor.new(puppet_install_distributor_path, :subdir => 'modules', :id => repo.pulp_id, :auto_publish => false)
+          puppet_install_dist = Runcible::Models::PuppetInstallDistributor.new(puppet_install_distributor_path, :subdir => 'modules', :id => repo.pulp_id, :auto_publish => smart_proxy.try(:pulp_mirror?) || false)
           puppet_dist = Runcible::Models::PuppetDistributor.new(nil, (root.unprotected || false), true,
                                                                 :id => "#{repo.pulp_id}_puppet", :auto_publish => true)
           [puppet_dist, puppet_install_dist]
@@ -30,6 +30,14 @@ module Katello
           ::File.join(smart_proxy.puppet_path, puppet_env)
         end
 
+        def distributors_to_publish(_options)
+          if !repo.content_view.default? && !repo.archive?
+            {Runcible::Models::PuppetDistributor => {}, Runcible::Models::PuppetInstallDistributor => {}}
+          else
+            {Runcible::Models::PuppetDistributor => {}}
+          end
+        end
+
         def partial_repo_path
           "/pulp/puppet/#{repo.pulp_id}/"
         end

data/app/services/katello/pulp/repository/yum.rb

@@ -36,7 +36,11 @@ module Katello
             id: yum_dist_id,
             auto_publish: true
           }
-          options[:checksum_type] = repo.saved_checksum_type || root.checksum_type if smart_proxy.pulp_master?
+          if smart_proxy.pulp_master?
+            options[:checksum_type] = repo.saved_checksum_type || root.checksum_type
+          else
+            options[:checksum_type] = nil
+          end
           distributors = [Runcible::Models::YumDistributor.new(repo.relative_path, root.unprotected, true, options)]
 
           if smart_proxy.pulp_master?
@@ -47,6 +51,17 @@ module Katello
           distributors
         end
 
+        def distributors_to_publish(_options)
+          if repo.clone && !repo.master?
+            source_service = repo.target_repository.backend_service(smart_proxy)
+            source_distributor_id = source_service.lookup_distributor_id(Runcible::Models::YumDistributor.type_id)
+            {Runcible::Models::YumCloneDistributor => {source_repo_id: repo.target_repository.pulp_id,
+                                                       source_distributor_id: source_distributor_id}}
+          else
+            {Runcible::Models::YumDistributor => {}}
+          end
+        end
+
         def smart_proxy_download_policy
           policy = smart_proxy.download_policy || Setting[:default_proxy_download_policy]
           if policy == ::SmartProxy::DOWNLOAD_INHERIT
@@ -61,20 +76,54 @@ module Katello
         end
 
         def copy_contents(destination_repo, options = {})
+          override_config = Katello::Repository.build_override_config(options)
           rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(options[:filters], options[:rpm_filenames])
-          tasks = [smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id, rpm_copy_clauses)]
+          tasks = [smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id,
+                   rpm_copy_clauses.merge(:override_config => override_config))]
+
           if rpm_remove_clauses
             tasks << smart_proxy.pulp_api.extensions.repository.unassociate_units(destination_repo.pulp_id,
                                                                          type_ids: [::Katello::Pulp::Rpm::CONTENT_TYPE],
                                                                           filters: {unit: rpm_remove_clauses})
           end
 
+          # always copy modular rpms
+          modular_includes = {filters: {unit: { 'filename' => { '$in' => repo.rpms.modular.pluck(:filename) } }}}
+          smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id, modular_includes)
+
           [:srpm, :errata, :package_group, :yum_repo_metadata_file, :distribution, :module, :module_default].each do |type|
             tasks << smart_proxy.pulp_api.extensions.send(type).copy(repo.pulp_id, destination_repo.pulp_id)
           end
           tasks
         end
 
+        def import_distribution_data
+          distribution = smart_proxy.pulp_api.extensions.repository.distributions(repo.pulp_id).first
+          if distribution
+            repo.update_attributes!(
+              :distribution_version => distribution["version"],
+              :distribution_arch => distribution["arch"],
+              :distribution_family => distribution["family"],
+              :distribution_variant => distribution["variant"],
+              :distribution_uuid => distribution["_id"],
+              :distribution_bootable => self.class.distribution_bootable?(distribution)
+            )
+          end
+        end
+
+        def self.distribution_bootable?(distribution)
+          # Not every distribution from Pulp represents a bootable
+          # repo. Determine based on the files in the repo.
+          distribution["files"].any? do |file|
+            if file.is_a? Hash
+              filename = file[:relativepath]
+            else
+              filename = file
+            end
+            filename.include?('vmlinuz') || filename.include?('pxeboot') || filename.include?('kernel.img') || filename.include?('initrd.img')
+          end
+        end
+
         def purge_empty_contents
           [purge_empty_errata, purge_empty_package_groups]
         end

data/app/services/katello/pulp/rpm.rb

@@ -4,26 +4,24 @@ module Katello
       include LazyAccessor
 
       PULP_SELECT_FIELDS = %w(name epoch version release arch checksumtype checksum).freeze
-      PULP_INDEXED_FIELDS = %w(name version release arch epoch summary sourcerpm checksum filename _id).freeze
+      PULP_INDEXED_FIELDS = %w(name version release arch epoch summary sourcerpm checksum filename _id is_modular).freeze
       CONTENT_TYPE = "rpm".freeze
 
-      lazy_accessor :pulp_facts, :initializer => :backend_data
-
       lazy_accessor :description, :license, :buildhost, :vendor, :relativepath, :children, :checksumtype,
                     :changelog, :group, :size, :url, :build_time, :group,
-                    :initializer => :pulp_facts
+                    :initializer => :backend_data
 
       def requires
-        if pulp_facts['requires']
-          pulp_facts['requires'].map { |entry| Katello::Util::Package.format_requires(entry) }.uniq.sort
+        if backend_data['requires']
+          backend_data['requires'].map { |entry| Katello::Util::Package.format_requires(entry) }.uniq.sort
         else
           []
         end
       end
 
       def provides
-        if pulp_facts['provides']
-          pulp_facts['provides'].map { |entry| Katello::Util::Package.build_nvrea(entry, false) }.uniq.sort
+        if backend_data['provides']
+          backend_data['provides'].map { |entry| Katello::Util::Package.build_nvrea(entry, false) }.uniq.sort
         else
           []
         end
@@ -31,16 +29,29 @@ module Katello
 
       def files
         result = []
-        if pulp_facts['files']
-          if pulp_facts['files']['file']
-            result << pulp_facts['files']['file']
+        if backend_data['files']
+          if backend_data['files']['file']
+            result << backend_data['files']['file']
           end
-          if pulp_facts['files']['dir']
-            result << pulp_facts['files']['dir']
+          if backend_data['files']['dir']
+            result << backend_data['files']['dir']
           end
         end
         result.flatten
       end
+
+      def update_model(model)
+        keys = PULP_INDEXED_FIELDS - ['_id', 'is_modular']
+        data = backend_data.slice(*keys)
+        data['modular'] = backend_data['is_modular'] == true
+        if data.any? { |name, value| model.send(name) != value }
+          data[:release_sortable] = Util::Package.sortable_version(data[:release])
+          data[:version_sortable] = Util::Package.sortable_version(data[:version])
+          model.assign_attributes(data)
+          model.nvra = model.build_nvra
+          model.save!
+        end
+      end
     end
   end
 end

data/app/services/katello/pulp/smart_proxy_repository.rb

@@ -0,0 +1,165 @@
+module Katello
+  module Pulp
+    class SmartProxyRepository
+      attr_accessor :smart_proxy
+
+      def initialize(smart_proxy)
+        @smart_proxy = smart_proxy
+      end
+
+      def ==(other)
+        other.class == self.class && other.smart_proxy == smart_proxy
+      end
+
+      def default_capsule?
+        @smart_proxy.pulp_master?
+      end
+
+      def repos_available_to_capsule(environments = nil, content_view = nil)
+        yum_repos = yum_repos_available_to_capsule(environments, content_view) || []
+        puppet_envs = puppet_environments_available_to_capsule(environments, content_view) || []
+        yum_repos + puppet_envs
+      end
+
+      def yum_repos_available_to_capsule(environments = nil, content_view = nil)
+        environments = @smart_proxy.lifecycle_environments if environments.nil?
+        yum_repos = Katello::Repository.in_environment(environments)
+        yum_repos = yum_repos.in_content_views([content_view]) if content_view
+        yum_repos.find_all { |repo| repo.node_syncable? }
+      end
+
+      def puppet_environments_available_to_capsule(environments = nil, content_view = nil)
+        environments = @smart_proxy.lifecycle_environments if environments.nil?
+        puppet_environments = Katello::ContentViewPuppetEnvironment.in_environment(environments)
+        puppet_environments = puppet_environments.in_content_view(content_view) if content_view
+        puppet_environments
+      end
+
+      def affected_repositories(environment, content_view, repository)
+        if repository
+          [repository]
+        else
+          repos_available_to_capsule(environment, content_view)
+        end
+      end
+
+      def repos_needing_updates(environment, content_view, repository)
+        repos = affected_repositories(environment, content_view, repository)
+        need_importer_update = needs_importer_updates(repos)
+        need_distributor_update = needs_distributor_updates(repos)
+        (need_distributor_update + need_importer_update).uniq
+      end
+
+      def current_repositories(environment_id = nil, content_view_id = nil)
+        yum_repos = current_yum_repos(environment_id, content_view_id) || []
+        puppet_envs = current_puppet_environments(environment_id, content_view_id) || []
+        yum_repos + puppet_envs
+      end
+
+      def current_yum_repos(environment_id = nil, content_view_id = nil)
+        katello_repos = Katello::Repository.all
+        katello_repos = katello_repos.where(:environment_id => environment_id) if environment_id
+        katello_repos = katello_repos.in_content_views([content_view_id]) if content_view_id
+
+        pulp_repos = self.smart_proxy.pulp_api.extensions.repository.search_by_repository_ids(katello_repos.pluck(:pulp_id))
+
+        katello_repos.where(:pulp_id => pulp_repos.map { |pulp_repo| pulp_repo['id'] })
+      end
+
+      def current_puppet_environments(environment_id = nil, content_view_id = nil)
+        puppet_repos = Katello::ContentViewPuppetEnvironment.all
+        puppet_repos = puppet_repos.where(:environment_id => environment_id) if environment_id
+        puppet_repos = puppet_repos.in_content_view(content_view_id) if content_view_id
+
+        pulp_repos = self.smart_proxy.pulp_api.extensions.repository.search_by_repository_ids(puppet_repos.pluck(:pulp_id))
+
+        puppet_repos.where(:pulp_id => pulp_repos.map { |pulp_repo| pulp_repo['id'] })
+      end
+
+      def current_repositories_data(environment = nil, content_view = nil)
+        @pulp_repositories ||= smart_proxy.pulp_repositories
+
+        repos = Katello::Repository
+        repos = repos.in_environment(environment) if environment
+        repos = repos.in_content_views([content_view]) if content_view
+        puppet_envs = Katello::ContentViewPuppetEnvironment
+        puppet_envs = puppet_envs.in_environment(environment) if environment
+        puppet_envs = puppet_envs.in_content_view(content_view) if content_view
+
+        repo_ids = repos.pluck(:pulp_id) + puppet_envs.pluck(:pulp_id)
+
+        @pulp_repositories.select { |r| repo_ids.include?(r['id']) }
+      end
+
+      def orphaned_repos
+        @smart_proxy.pulp_repositories.map { |x| x["id"] } - current_repositories.map { |x| x.pulp_id }
+      end
+
+      def delete_orphaned_repos
+        orphaned_repos.map { |repo| self.smart_proxy.pulp_api.extensions.repository.delete(repo) }.compact
+      end
+
+      def needs_importer_updates(repos)
+        repos.select do |repo|
+          repo_details = repo.backend_service(self.smart_proxy).backend_data
+          next unless repo_details
+          capsule_importer = repo_details["importers"][0]
+          !importer_matches?(repo, capsule_importer)
+        end
+      end
+
+      def needs_distributor_updates(repos)
+        repos.select do |repo|
+          repo_details = repo.backend_service(smart_proxy).backend_data
+          next unless repo_details
+          !distributors_match?(repo, repo_details["distributors"])
+        end
+      end
+
+      def importer_matches?(repo, capsule_importer)
+        generated_importer = repo.backend_service(self.smart_proxy).generate_importer
+        capsule_importer.try(:[], 'importer_type_id') == generated_importer.id &&
+            generated_importer.config.compact == capsule_importer['config'].compact
+      end
+
+      def distributors_match?(repo, capsule_distributors)
+        generated_distributor_configs = repo.backend_service(self.smart_proxy).generate_distributors
+        generated_distributor_configs.all? do |gen_dist|
+          type = gen_dist.class.type_id
+          found_on_capsule = capsule_distributors.find { |dist| dist['distributor_type_id'] == type }
+          found_on_capsule && filtered_distribution_config_equal?(gen_dist.config, found_on_capsule['config'])
+        end
+      end
+
+      def get_repository_ids(environment, content_view, repository)
+        if environment
+          repository_ids = repos_available_to_capsule(environment, content_view).map(&:pulp_id)
+        elsif repository
+          repository_ids = [repository.pulp_id]
+          environment = repository.environment
+        else
+          repository_ids = repos_available_to_capsule.map(&:pulp_id)
+        end
+
+        if environment && !self.smart_proxy.lifecycle_environments.include?(environment)
+          fail _("Lifecycle environment '%{environment}' is not attached to this capsule.") % { :environment => environment.name }
+        end
+
+        repository_ids
+      end
+
+      private
+
+      def filtered_distribution_config_equal?(generated_config, actual_config)
+        generated = generated_config.clone
+        actual = actual_config.clone
+        #We store 'default' checksum type as nil, but pulp will default to sha256, so if we haven't set it, ignore it
+        if generated.keys.include?('checksum_type') && generated['checksum_type'].nil?
+          generated.delete('checksum_type')
+          actual.delete('checksum_type')
+        end
+        generated.compact == actual.compact
+      end
+    end
+  end
+end

data/app/services/katello/pulp/srpm.rb

@@ -12,6 +12,18 @@ module Katello
       lazy_accessor :description, :license, :buildhost, :vendor, :relativepath, :children, :checksumtype,
                     :changelog, :group, :size, :url, :build_time, :group,
                     :initializer => :pulp_facts
+
+      def update_model(model)
+        keys = Pulp::Srpm::PULP_INDEXED_FIELDS - ['_id']
+        custom_json = backend_data.slice(*keys)
+        if custom_json.any? { |name, value| model.send(name) != value }
+          custom_json[:release_sortable] = Util::Package.sortable_version(custom_json[:release])
+          custom_json[:version_sortable] = Util::Package.sortable_version(custom_json[:version])
+          model.assign_attributes(custom_json)
+          model.nvra = model.build_nvra
+          model.save!
+        end
+      end
     end
   end
 end

data/app/services/katello/pulp/yum_metadata_file.rb

@@ -2,6 +2,25 @@ module Katello
   module Pulp
     class YumMetadataFile < PulpContentUnit
       CONTENT_TYPE = "yum_repo_metadata_file".freeze
+
+      def update_model(model)
+        shared_attributes = backend_data.keys & model.class.column_names
+        shared_json = backend_data.select { |key, _v| shared_attributes.include?(key) }
+        repo = ::Katello::Repository.find_by(:pulp_id => backend_data['repo_id']).try(:id)
+        model.update_attributes!(shared_json.merge(repository_id: repo,
+                                                  name: find_name_from_json(backend_data)))
+      end
+
+      def find_name_from_json(json)
+        # get the name of the metadata file.
+        # Notice that pulp does not have a way to get name of the metadata file
+        # so we infer it from the _storage_path
+        # for example from the following storage path
+        #  "/var/lib/pulp/content/units/yum_repo_metadata_file/..../050-productid.gz"
+        # we find the right most '/' and return everything after that
+        # i.e ->  "050-productid.gz"
+        File.basename(json["_storage_path"])
+      end
     end
   end
 end

data/app/services/katello/pulp3/repository.rb

@@ -0,0 +1,170 @@
+module Katello
+  module Pulp3
+    class Repository
+      attr_accessor :repo, :input
+      attr_accessor :smart_proxy
+      delegate :root, to: :repo
+      delegate :pulp3_api, to: :smart_proxy
+
+      def initialize(repo, smart_proxy)
+        @repo = repo
+        @smart_proxy = smart_proxy
+      end
+
+      def create_remote
+        fail NotImplementedError
+      end
+
+      def remote_options
+        fail NotImplementedError
+      end
+
+      def update_remote
+        fail NotImplementedError
+      end
+
+      def sync
+        fail NotImplementedError
+      end
+
+      def create_publisher
+        fail NotImplementedError
+      end
+
+      def create_publication
+        fail NotImplementedError
+      end
+
+      def self.instance_for_type(repo, smart_proxy)
+        Katello::RepositoryTypeManager.repository_types[repo.root.content_type].pulp3_service_class.new(repo, smart_proxy)
+      end
+
+      def backend_object_name
+        "#{root.label}-#{repo.id}#{rand(9999)}"
+      end
+
+      def repository_reference
+        RepositoryReference.find_by(:root_repository_id => repo.root_id, :content_view_id => repo.content_view.id)
+      end
+
+      def distribution_reference(path)
+        DistributionReference.find_by(:path => path)
+      end
+
+      def create
+        unless repository_reference
+          response = pulp3_api.repositories_create(Zest::Repository.new(:name => backend_object_name))
+          RepositoryReference.create!(:root_repository_id => repo.root_id, :content_view_id => repo.content_view.id, :repository_href => response._href)
+          response
+        end
+      end
+
+      def list(args)
+        pulp3_api.repositories_list(args).results
+      end
+
+      def delete(href = repository_reference.repository_href)
+        response = pulp3_api.repositories_delete(href)
+        repository_reference.destroy if repository_reference
+        response
+      end
+
+      def update_distribution(path)
+        distribution_reference = distribution_reference(path)
+        pulp3_api.distributions_partial_update(distribution_reference.href, publication: repo.publication_href)
+      end
+
+      def refresh_distributions
+        paths.map do |prefix, path|
+          if distribution_reference(path)
+            update_distribution(path)
+          else
+            create_distribution(prefix, path)
+          end
+        end
+      end
+
+      def create_version
+        pulp3_api.repositories_versions_create(repository_reference.repository_href, {})
+      end
+
+      def paths
+        list = {
+          https: "https/#{repo.relative_path}"
+        }
+        list['http'] = "http/#{repo.relative_path}" if repo.root.unprotected
+        list
+      end
+
+      private def delete_distribution(href)
+        pulp3_api.distributions_delete(href)
+      end
+
+      def lookup_distributions(args)
+        pulp3_api.distributions_list(args).results
+      end
+
+      def get_distribution(href)
+        pulp3_api.distributions_read(href)
+      rescue Zest::ApiError => e
+        raise e if e.code != 404
+        nil
+      end
+
+      def delete_distributions
+        paths.values.each do |path|
+          dists = lookup_distributions(base_path: path.sub(/^\//, ''))
+          delete_distribution(dists.first._href) if dists.first
+          distribution_reference = distribution_reference(path)
+          distribution_reference.destroy if distribution_reference
+        end
+      end
+
+      def save_distribution_references(hrefs)
+        hrefs.each do |href|
+          path = get_distribution(href)&.base_path
+          unless distribution_reference(path)
+            DistributionReference.create!(path: path, href: href, root_repository_id: repo.root.id)
+          end
+        end
+      end
+
+      def create_distribution(prefix, path)
+        path = path.sub(/^\//, '') #remove leading / if present
+        pulp3_api.distributions_create(base_path: path, publication: repo.publication_href, name: "#{prefix}_#{backend_object_name}")
+      end
+
+      def common_remote_options
+        remote_options = {
+          validate: true,
+          ssl_validation: root.verify_ssl_on_sync,
+          name: backend_object_name,
+          url: root.url
+        }
+        if root.upstream_username && root.upstream_password
+          remote_options.merge(username: root.upstream_username,
+                               password: root.upstream_password)
+        end
+        remote_options.merge(ssl_remote_options)
+      end
+
+      def ssl_remote_options
+        if root.redhat? && Katello::Resources::CDN::CdnResource.redhat_cdn?(root.url)
+          {
+            ssl_client_certificate: root.product.certificate,
+            ssl_client_key: root.product.key,
+            ssl_ca_certificate: Katello::Repository.feed_ca_cert(root.url)
+          }
+        elsif root.custom?
+          {
+            ssl_client_certificate: root.ssl_client_cert&.content,
+            ssl_client_key: root.ssl_client_key&.content,
+            ssl_ca_certificate: root.ssl_ca_cert&.content
+          }
+        else
+          {}
+        end
+      end
+    end
+  end
+end