katello 3.15.1 → 3.16.0.rc2.1

data/app/services/katello/pulp3/file_unit.rb

@@ -24,7 +24,7 @@ module Katello
         custom_json['checksum'] = backend_data['sha256']
         custom_json['path'] = backend_data['relative_path']
         custom_json['name'] = File.basename(backend_data['relative_path'].try(:split, '/').try(:[], -1))
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/pulp3/migration.rb

@@ -130,7 +130,8 @@ module Katello
           migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
           migrated_units.results.each do |migrated_unit|
             matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }
-            matching_record.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content) if matching_record
+
+            matching_record&.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content)
           end
         end
       end

data/app/services/katello/pulp3/module_stream.rb

@@ -13,6 +13,15 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
+      def create_stream_rpms(model, packages)
+        packages_found = Katello::Rpm.where(:pulp_id => packages)
+        existing_rpms = model.rpms
+        new_packages = packages_found - existing_rpms
+        packages_to_delete = existing_rpms - packages_found
+        model.rpms.delete(packages_to_delete)
+        model.rpms << new_packages
+      end
+
       def create_stream_artifacts(model, artifacts_json)
         artifacts_json.each do |name|
           Katello::Util::Support.active_record_retry do
@@ -37,10 +46,11 @@ module Katello
       def update_model(model)
         shared_attributes = backend_data.keys & model.class.column_names
         shared_json = backend_data.select { |key, _v| shared_attributes.include?(key) }
-        model.update_attributes!(shared_json)
+        model.update!(shared_json)
 
-        create_stream_artifacts(model, JSON.parse(backend_data['artifacts'])) if backend_data.key?('artifacts')
-        create_profiles(model, JSON.parse(backend_data['profiles'])) if backend_data.key?('profiles')
+        create_stream_artifacts(model, backend_data['artifacts']) if backend_data.key?('artifacts')
+        create_profiles(model, backend_data['profiles']) if backend_data.key?('profiles')
+        create_stream_rpms(model, backend_data['packages']) if backend_data.key?('packages')
       end
     end
   end

data/app/services/katello/pulp3/package_group.rb

@@ -59,7 +59,7 @@ module Katello
         custom_json['pulp_id'] = backend_data['pulp_href']
         custom_json['name'] = backend_data['name']
         custom_json['description'] = backend_data['description']
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -15,6 +15,14 @@ module Katello
         fail NotImplementedError
       end
 
+      def self.backend_unit_identifier
+        nil
+      end
+
+      def self.supports_id_fetch?
+        true
+      end
+
       def update_model
         fail NotImplementedError
       end
@@ -38,6 +46,12 @@ module Katello
         self::CONTENT_TYPE
       end
 
+      def self.pulp_units_batch_all(content_unit_hrefs)
+        Enumerator.new do |yielder|
+          yielder.yield content_unit_hrefs.collect { |href| pulp_data(href) }
+        end
+      end
+
       def self.pulp_units_batch_for_repo(repository, options = {})
         fetch_identifiers = options.fetch(:fetch_identifiers, false)
         page_size = options.fetch(:page_size, SETTINGS[:katello][:pulp][:bulk_load_size])
@@ -52,13 +66,7 @@ module Katello
             break unless (
               (response["count"] && page_opts["offset"] < response["count"]) ||
               page_opts["offset"] == 0)
-            if self == Katello::Pulp3::Rpm
-              # Get all packages where arch != "src"
-              # FIXME change once reverse filtering is in Pulp3
-              page_opts["arch__in"] = Katello::Pulp3::Rpm.rpm_architectures.join(",")
-            elsif self == Katello::Pulp3::Srpm
-              page_opts["arch"] = "src"
-            end
+            page_opts = page_options page_opts
             response = fetch_content_list page_opts
             response = response.as_json.with_indifferent_access
             yielder.yield response[:results]
@@ -67,6 +75,10 @@ module Katello
         end
       end
 
+      def self.page_options(page_opts = {})
+        page_opts
+      end
+
       def self.pulp_data(href)
         content_unit = self.content_api.read(href)
         content_unit.as_json

data/app/services/katello/pulp3/repository.rb

@@ -1,11 +1,9 @@
 require "pulpcore_client"
-# rubocop:disable ClassLength
 
 module Katello
   module Pulp3
     class Repository
       include Katello::Util::HttpProxy
-
       attr_accessor :repo
       attr_accessor :smart_proxy
       delegate :root, to: :repo
@@ -43,6 +41,13 @@ module Katello
         @api ||= self.class.api(smart_proxy)
       end
 
+      def ignore_404_exception(*)
+        yield
+      rescue api.class.api_exception_class => e
+        raise e unless e.code == 404
+        nil
+      end
+
       def content_service
         Katello::Pulp3::Content
       end
@@ -50,14 +55,14 @@ module Katello
       def create_remote
         remote_file_data = api.class.remote_class.new(remote_options)
         response = api.remotes_api.create(remote_file_data)
-        repo.update_attributes!(:remote_href => response.pulp_href)
+        repo.update!(:remote_href => response.pulp_href)
       end
 
       def update_remote
         href = repo.remote_href
         if remote_options[:url].blank?
           if href
-            repo.update_attributes(remote_href: nil)
+            repo.update(remote_href: nil)
             delete_remote href
           end
         else
@@ -74,7 +79,7 @@ module Katello
       end
 
       def delete_remote(href = repo.remote_href)
-        api.remotes_api.delete(href) if href
+        ignore_404_exception { api.remotes_api.delete(href) } if href
       end
 
       def self.instance_for_type(repo, smart_proxy)
@@ -138,7 +143,7 @@ module Katello
 
       def distribution_needs_update?
         if distribution_reference
-          expected = distribution_options(relative_path).except(:name).compact
+          expected = secure_distribution_options(relative_path).except(:name).compact
           actual = get_distribution.to_hash
           expected != actual.slice(*expected.keys)
         elsif repo.environment
@@ -175,13 +180,17 @@ module Katello
         api.repositories_api.list(options).results
       end
 
+      def read
+        api.repositories_api.read(repository_reference.try(:repository_href))
+      end
+
       def delete(href = repository_reference.try(:repository_href))
         repository_reference.try(:destroy)
-        api.repositories_api.delete(href) if href
+        ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
       def sync
-        repository_sync_url_data = api.class.client_module::RepositorySyncURL.new(remote: repo.remote_href, mirror: repo.root.mirror_on_sync)
+        repository_sync_url_data = api.class.repository_sync_url_class.new(remote: repo.remote_href, mirror: repo.root.mirror_on_sync)
         [api.repositories_api.sync(repository_reference.repository_href, repository_sync_url_data)]
       end
 
@@ -204,7 +213,7 @@ module Katello
       end
 
       def create_distribution(path)
-        distribution_data = api.class.distribution_class.new(distribution_options(path))
+        distribution_data = api.class.distribution_class.new(secure_distribution_options(path))
         api.distributions_api.create(distribution_data)
       end
 
@@ -214,7 +223,8 @@ module Katello
 
       def update_distribution
         if distribution_reference
-          options = distribution_options(relative_path).except(:name)
+          options = secure_distribution_options(relative_path).except(:name)
+          distribution_reference.update(:content_guard_href => options[:content_guard])
           api.distributions_api.partial_update(distribution_reference.href, options)
         end
       end
@@ -232,7 +242,7 @@ module Katello
       end
 
       def delete_version
-        api.repository_versions_api.delete(repo.version_href)
+        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
       end
 
       def create_version(options = {})
@@ -241,16 +251,17 @@ module Katello
 
       def save_distribution_references(hrefs)
         hrefs.each do |href|
-          path = api.get_distribution(href)&.base_path
+          pulp3_distribution_data = api.get_distribution(href)
+          path, content_guard_href = pulp3_distribution_data&.base_path, pulp3_distribution_data&.content_guard
           unless distribution_reference
-            DistributionReference.create!(path: path, href: href, repository_id: repo.id)
+            DistributionReference.create!(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href)
           end
         end
       end
 
       def delete_distributions
         if (dist_ref = distribution_reference)
-          api.delete_distribution(dist_ref.href)
+          ignore_404_exception { api.delete_distribution(dist_ref.href) }
           dist_ref.destroy!
         end
       end
@@ -279,6 +290,16 @@ module Katello
         remote_options.merge!(ssl_remote_options)
       end
 
+      def secure_distribution_options(path)
+        secured_distribution_options = {}
+        if root.unprotected
+          secured_distribution_options[:content_guard] = nil
+        else
+          secured_distribution_options[:content_guard] = ::Katello::Pulp3::ContentGuard.first.pulp_href
+        end
+        secured_distribution_options.merge!(distribution_options(path))
+      end
+
       def ssl_remote_options
         if root.redhat? && Katello::Resources::CDN::CdnResource.redhat_cdn?(root.url)
           {
@@ -299,14 +320,14 @@ module Katello
 
       def lookup_version(href)
         api.repository_versions_api.read(href) if href
-      rescue api.api_exception_class => e
+      rescue api.class.api_exception_class => e
         Rails.logger.error "Exception when calling repository_versions_api->read: #{e}"
         nil
       end
 
       def lookup_publication(href)
         api.publications_api.read(href) if href
-      rescue api.api_exception_class => e
+      rescue api.class.api_exception_class => e
         Rails.logger.error "Exception when calling publications_api->read: #{e}"
         nil
       end

data/app/services/katello/pulp3/repository/docker.rb

@@ -10,7 +10,7 @@ module Katello
 
         def remote_options
           options = {url: root.url, upstream_name: root.docker_upstream_name}
-          if root.docker_tags_whitelist && root.docker_tags_whitelist.any?
+          if root.docker_tags_whitelist&.any?
             options[:whitelist_tags] = root.docker_tags_whitelist
           else
             options[:whitelist_tags] = nil

data/app/services/katello/pulp3/repository/yum.rb

@@ -4,6 +4,9 @@ module Katello
   module Pulp3
     class Repository
       class Yum < ::Katello::Pulp3::Repository
+        include Katello::Util::Errata
+        include Katello::Util::PulpcoreContentFilters
+
         def remote_options
           if root.url.blank?
             common_remote_options.merge(url: nil, policy: root.download_policy)
@@ -20,10 +23,20 @@ module Katello
           }
         end
 
+        def mirror_remote_options
+          policy = smart_proxy.download_policy
+
+          if smart_proxy.download_policy == SmartProxy::DOWNLOAD_INHERIT
+            policy = repo.root.download_policy
+          end
+
+          { policy: policy }
+        end
+
         def import_distribution_data
           distribution = ::Katello::Pulp3::Distribution.fetch_content_list(repository_version: repo.version_href)
           if distribution.results.present?
-            repo.update_attributes!(
+            repo.update!(
               :distribution_version => distribution.results.first.release_version,
               :distribution_arch => distribution.results.first.arch,
               :distribution_family => distribution.results.first.release_name,
@@ -32,7 +45,7 @@ module Katello
             )
             unless distribution.results.first.variants.empty?
               unless distribution.results.first.variants.first.name.nil?
-                repo.update_attributes!(:distribution_variant => distribution.results.first.variants.first.name)
+                repo.update!(:distribution_variant => distribution.results.first.variants.first.name)
               end
             end
           end
@@ -45,9 +58,97 @@ module Katello
           end
         end
 
-        def copy_content_for_source
-          # TODO
-          fail NotImplementedError
+        def partial_repo_path
+          "/pulp/repos/#{repo.relative_path}/".sub('//', '/')
+        end
+
+        def copy_units(source_repository_version_href, content_unit_hrefs, dependency_solving)
+          tasks = []
+
+          if content_unit_hrefs.any?
+            data = PulpRpmClient::Copy.new
+            data.config = [
+              source_repo_version: source_repository_version_href,
+              dest_repo: repository_reference.repository_href,
+              dest_base_version: 0,
+              content: content_unit_hrefs.sort
+            ]
+            data.dependency_solving = dependency_solving
+            tasks << api.copy_api.copy_content(data)
+          else
+            data = PulpRpmClient::RepositoryAddRemoveContent.new(
+              remove_content_units: ['*'])
+            tasks << api.repositories_api.modify(repository_reference.repository_href, data)
+          end
+
+          tasks
+        end
+
+        def packageenvironments(options = {})
+          api.content_package_environments_api.list(options)
+        end
+
+        def metadatafiles(options = {})
+          api.content_repo_metadata_files_api.list(options)
+        end
+
+        def distributiontrees(options = {})
+          api.content_distribution_trees_api.list(options)
+        end
+
+        def copy_content_for_source(source_repository, options = {})
+          filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class|
+            filter_class.where(:id => options[:filter_ids])
+          end
+
+          filters.flatten!.compact!
+
+          whitelist_ids = []
+          blacklist_ids = []
+          filters.each do |filter|
+            if filter.inclusion
+              whitelist_ids += filter.content_unit_pulp_ids(source_repository)
+            else
+              blacklist_ids += filter.content_unit_pulp_ids(source_repository)
+            end
+          end
+
+          if whitelist_ids.empty? && filters.select { |filter| filter.inclusion }.empty?
+            whitelist_ids = source_repository.rpms.where(:modular => false).pluck(:pulp_id).sort
+          end
+
+          content_unit_hrefs = whitelist_ids - blacklist_ids
+
+          if content_unit_hrefs.any?
+            content_unit_hrefs += additional_content_hrefs(source_repository, content_unit_hrefs)
+          end
+          content_unit_hrefs += source_repository.srpms.pluck(:pulp_id)
+
+          dependency_solving = options[:solve_dependencies] || false
+          copy_units(source_repository.version_href, content_unit_hrefs.uniq, dependency_solving)
+        end
+
+        def additional_content_hrefs(source_repository, content_unit_hrefs)
+          repo_service = source_repository.backend_service(SmartProxy.pulp_master)
+          options = { :repository_version => source_repository.version_href }
+
+          errata_to_include = filter_errata_by_pulp_href(source_repository.errata, content_unit_hrefs)
+          content_unit_hrefs += errata_to_include.collect { |erratum| erratum.repository_errata.pluck(:erratum_pulp3_href) }.flatten
+
+          package_groups_to_include = filter_package_groups_by_pulp_href(source_repository.package_groups, content_unit_hrefs)
+          content_unit_hrefs += package_groups_to_include.pluck(:pulp_id)
+
+          package_environment_hrefs_to_include = filter_package_environments_by_pulp_hrefs(
+            repo_service.packageenvironments(options).results, package_groups_to_include.pluck(:pulp_id))
+          content_unit_hrefs += package_environment_hrefs_to_include
+
+          metadata_file_hrefs_to_include = filter_metadatafiles_by_pulp_hrefs(
+            repo_service.metadatafiles(options).results, content_unit_hrefs)
+          content_unit_hrefs += metadata_file_hrefs_to_include
+
+          distribution_tree_hrefs_to_include = filter_distribution_trees_by_pulp_hrefs(
+            repo_service.distributiontrees(options).results, content_unit_hrefs)
+          content_unit_hrefs + distribution_tree_hrefs_to_include
         end
 
         def regenerate_applicability

data/app/services/katello/pulp3/repository_mirror.rb

@@ -52,6 +52,9 @@ module Katello
 
       def delete(href = repository_href)
         api.repositories_api.delete(href) if href
+      rescue api.class.api_exception_class => e
+        raise e if e.code != 404
+        nil
       end
 
       def repository_href
@@ -103,8 +106,7 @@ module Katello
       end
 
       def sync
-        api_module = api.class.client_module
-        repository_sync_url_data = api_module::RepositorySyncURL.new(remote: remote_href, mirror: true)
+        repository_sync_url_data = api.class.repository_sync_url_class.new(remote: remote_href, mirror: true)
         [api.repositories_api.sync(repository_href, repository_sync_url_data)]
       end
 

data/app/services/katello/pulp3/rpm.rb

@@ -20,37 +20,25 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def self.rpm_architectures
-        ['i386', 'i486', 'i586', 'i686', 'athlon', 'geode', 'pentium3', 'pentium4',
-         'x86_64', 'amd64',
-         'ia64',
-         'alpha', 'alphaev5', 'alphaev56', 'alphapca56', 'alphaev6', 'alphaev67',
-         'sparc', 'sparcv8', 'sparcv9', 'sparc64', 'sparc64v', 'sun4', 'sun4c', 'sun4d', 'sun4m', 'sun4u',
-         'armv3l', 'armv4b', 'armv4l', 'armv5tel', 'armv5tejl', 'armv6l', 'armv7l',
-         'mips', 'mipsel',
-         'ppc', 'ppciseries', 'ppcpseries', 'ppc64', 'ppc8260', 'ppc8560', 'ppc32dy4',
-         'm68k', 'm68kmint', 'atarist', 'atariste', 'ataritt', 'falcon', 'atariclone', 'milan', 'hades',
-         'Sgi',
-         'rs6000',
-         'i370', 's390x', 's390',
-         'noarch']
+      def self.page_options(page_opts = {})
+        page_opts["arch__ne"] = "src"
+        page_opts
       end
 
       def requires
         results = []
         flags = {'GT' => '>', 'LT' => '>', 'EQ' => '=', 'GE' => '>=', 'LE' => '<='}
-        if backend_data['requires']
-          backend_data['requires'].each do |requirement|
-            requires_str = ""
-            if requirement.count < 3
-              requires_str = requirement.first
-              results << requires_str
-            else
-              requirement[1] = flags[requirement[1]]
-              requirement[0...2].each { |requirement_piece| requires_str += "#{requirement_piece} " }
-              requirement[2...-1].each { |requirement_piece| requires_str += "#{requirement_piece}." }
-              results << requires_str[0...-1]
-            end
+
+        backend_data['requires']&.each do |requirement|
+          requires_str = ""
+          if requirement.count < 3
+            requires_str = requirement.first
+            results << requires_str
+          else
+            requirement[1] = flags[requirement[1]]
+            requirement[0...2].each { |requirement_piece| requires_str += "#{requirement_piece} " }
+            requirement[2...-1].each { |requirement_piece| requires_str += "#{requirement_piece}." }
+            results << requires_str[0...-1]
           end
         end
         results.uniq
@@ -59,18 +47,17 @@ module Katello
       def provides
         results = []
         flags = {'GT' => '>', 'LT' => '>', 'EQ' => '=', 'GE' => '>=', 'LE' => '<='}
-        if backend_data['provides']
-          backend_data['provides'].each do |provided|
-            provides_str = ""
-            if provided.count < 3
-              provides_str = provided.first
-              results << provides_str
-            else
-              provided[1] = flags[provided[1]]
-              provided[0...2].each { |provided_piece| provides_str += "#{provided_piece} " }
-              provided[2...-1].each { |provided_piece| provides_str += "#{provided_piece}." }
-              results << provides_str[0...-1]
-            end
+
+        backend_data['provides']&.each do |provided|
+          provides_str = ""
+          if provided.count < 3
+            provides_str = provided.first
+            results << provides_str
+          else
+            provided[1] = flags[provided[1]]
+            provided[0...2].each { |provided_piece| provides_str += "#{provided_piece} " }
+            provided[2...-1].each { |provided_piece| provides_str += "#{provided_piece}." }
+            results << provides_str[0...-1]
           end
         end
         results.uniq
@@ -136,7 +123,7 @@ module Katello
         custom_json['filename'] = backend_data['location_href']
         custom_json['checksum'] = backend_data['pkgId']
         custom_json['sourcerpm'] = backend_data['rpm_sourcerpm']
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end