katello 3.15.3.1 → 3.16.0.rc1

data/app/services/katello/pulp3/docker_manifest_list.rb

@@ -23,7 +23,7 @@ module Katello
         custom_json['schema_version'], = backend_data['schema_version']
         custom_json['digest'], = backend_data['digest']
         custom_json['docker_manifests'] = ::Katello::DockerManifest.where(:pulp_id => backend_data[:listed_manifests])
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/pulp3/docker_tag.rb

@@ -21,7 +21,7 @@ module Katello
         end
         custom_json['docker_taggable'] = taggable
         custom_json['name'] = backend_data['name']
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/pulp3/erratum.rb

@@ -11,6 +11,10 @@ module Katello
         "id"
       end
 
+      def self.backend_unit_identifier
+        "pulp_href"
+      end
+
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::Yum.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list
@@ -29,7 +33,7 @@ module Katello
           custom_json['errata_id'] = custom_json.delete('id')
           custom_json['errata_type'] = custom_json.delete('type')
           custom_json['updated'] = custom_json['updated'].blank? ? custom_json['issued'] : custom_json['updated']
-          model.update_attributes!(custom_json)
+          model.update!(custom_json)
 
           unless backend_data['references'].blank?
             update_bugzillas(model, backend_data['references'])

data/app/services/katello/pulp3/file_unit.rb

@@ -24,7 +24,7 @@ module Katello
         custom_json['checksum'] = backend_data['sha256']
         custom_json['path'] = backend_data['relative_path']
         custom_json['name'] = File.basename(backend_data['relative_path'].try(:split, '/').try(:[], -1))
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/pulp3/migration.rb

@@ -130,7 +130,8 @@ module Katello
           migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
           migrated_units.results.each do |migrated_unit|
             matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }
-            matching_record.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content) if matching_record
+
+            matching_record&.update_column(:migrated_pulp3_href, migrated_unit.pulp3_content)
           end
         end
       end

data/app/services/katello/pulp3/module_stream.rb

@@ -13,6 +13,15 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
+      def create_stream_rpms(model, packages)
+        packages_found = Katello::Rpm.where(:pulp_id => packages)
+        existing_rpms = model.rpms
+        new_packages = packages_found - existing_rpms
+        packages_to_delete = existing_rpms - packages_found
+        model.rpms.delete(packages_to_delete)
+        model.rpms << new_packages
+      end
+
       def create_stream_artifacts(model, artifacts_json)
         artifacts_json.each do |name|
           Katello::Util::Support.active_record_retry do
@@ -37,10 +46,11 @@ module Katello
       def update_model(model)
         shared_attributes = backend_data.keys & model.class.column_names
         shared_json = backend_data.select { |key, _v| shared_attributes.include?(key) }
-        model.update_attributes!(shared_json)
+        model.update!(shared_json)
 
-        create_stream_artifacts(model, JSON.parse(backend_data['artifacts'])) if backend_data.key?('artifacts')
-        create_profiles(model, JSON.parse(backend_data['profiles'])) if backend_data.key?('profiles')
+        create_stream_artifacts(model, backend_data['artifacts']) if backend_data.key?('artifacts')
+        create_profiles(model, backend_data['profiles']) if backend_data.key?('profiles')
+        create_stream_rpms(model, backend_data['packages']) if backend_data.key?('packages')
       end
     end
   end

data/app/services/katello/pulp3/package_group.rb

@@ -59,7 +59,7 @@ module Katello
         custom_json['pulp_id'] = backend_data['pulp_href']
         custom_json['name'] = backend_data['name']
         custom_json['description'] = backend_data['description']
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -15,6 +15,10 @@ module Katello
         fail NotImplementedError
       end
 
+      def self.backend_unit_identifier
+        nil
+      end
+
       def update_model
         fail NotImplementedError
       end
@@ -38,6 +42,12 @@ module Katello
         self::CONTENT_TYPE
       end
 
+      def self.pulp_units_batch_all(content_unit_hrefs)
+        Enumerator.new do |yielder|
+          yielder.yield content_unit_hrefs.collect { |href| pulp_data(href) }
+        end
+      end
+
       def self.pulp_units_batch_for_repo(repository, options = {})
         fetch_identifiers = options.fetch(:fetch_identifiers, false)
         page_size = options.fetch(:page_size, SETTINGS[:katello][:pulp][:bulk_load_size])

data/app/services/katello/pulp3/repository.rb

@@ -1,11 +1,9 @@
 require "pulpcore_client"
-# rubocop:disable ClassLength
 
 module Katello
   module Pulp3
     class Repository
       include Katello::Util::HttpProxy
-
       attr_accessor :repo
       attr_accessor :smart_proxy
       delegate :root, to: :repo
@@ -43,6 +41,13 @@ module Katello
         @api ||= self.class.api(smart_proxy)
       end
 
+      def ignore_404_exception(*)
+        yield
+      rescue api.class.api_exception_class => e
+        raise e unless e.code == 404
+        nil
+      end
+
       def content_service
         Katello::Pulp3::Content
       end
@@ -50,14 +55,14 @@ module Katello
       def create_remote
         remote_file_data = api.class.remote_class.new(remote_options)
         response = api.remotes_api.create(remote_file_data)
-        repo.update_attributes!(:remote_href => response.pulp_href)
+        repo.update!(:remote_href => response.pulp_href)
       end
 
       def update_remote
         href = repo.remote_href
         if remote_options[:url].blank?
           if href
-            repo.update_attributes(remote_href: nil)
+            repo.update(remote_href: nil)
             delete_remote href
           end
         else
@@ -74,7 +79,7 @@ module Katello
       end
 
       def delete_remote(href = repo.remote_href)
-        api.remotes_api.delete(href) if href
+        ignore_404_exception { api.remotes_api.delete(href) } if href
       end
 
       def self.instance_for_type(repo, smart_proxy)
@@ -177,11 +182,11 @@ module Katello
 
       def delete(href = repository_reference.try(:repository_href))
         repository_reference.try(:destroy)
-        api.repositories_api.delete(href) if href
+        ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
       def sync
-        repository_sync_url_data = api.class.client_module::RepositorySyncURL.new(remote: repo.remote_href, mirror: repo.root.mirror_on_sync)
+        repository_sync_url_data = api.class.repository_sync_url_class.new(remote: repo.remote_href, mirror: repo.root.mirror_on_sync)
         [api.repositories_api.sync(repository_reference.repository_href, repository_sync_url_data)]
       end
 
@@ -232,7 +237,7 @@ module Katello
       end
 
       def delete_version
-        api.repository_versions_api.delete(repo.version_href)
+        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
       end
 
       def create_version(options = {})
@@ -250,7 +255,7 @@ module Katello
 
       def delete_distributions
         if (dist_ref = distribution_reference)
-          api.delete_distribution(dist_ref.href)
+          ignore_404_exception { api.delete_distribution(dist_ref.href) }
           dist_ref.destroy!
         end
       end
@@ -299,14 +304,14 @@ module Katello
 
       def lookup_version(href)
         api.repository_versions_api.read(href) if href
-      rescue api.api_exception_class => e
+      rescue api.class.api_exception_class => e
         Rails.logger.error "Exception when calling repository_versions_api->read: #{e}"
         nil
       end
 
       def lookup_publication(href)
         api.publications_api.read(href) if href
-      rescue api.api_exception_class => e
+      rescue api.class.api_exception_class => e
         Rails.logger.error "Exception when calling publications_api->read: #{e}"
         nil
       end

data/app/services/katello/pulp3/repository/docker.rb

@@ -10,7 +10,7 @@ module Katello
 
         def remote_options
           options = {url: root.url, upstream_name: root.docker_upstream_name}
-          if root.docker_tags_whitelist && root.docker_tags_whitelist.any?
+          if root.docker_tags_whitelist&.any?
             options[:whitelist_tags] = root.docker_tags_whitelist
           else
             options[:whitelist_tags] = nil

data/app/services/katello/pulp3/repository/yum.rb

@@ -20,10 +20,20 @@ module Katello
           }
         end
 
+        def mirror_remote_options
+          policy = smart_proxy.download_policy
+
+          if smart_proxy.download_policy == SmartProxy::DOWNLOAD_INHERIT
+            policy = repo.root.download_policy
+          end
+
+          { policy: policy }
+        end
+
         def import_distribution_data
           distribution = ::Katello::Pulp3::Distribution.fetch_content_list(repository_version: repo.version_href)
           if distribution.results.present?
-            repo.update_attributes!(
+            repo.update!(
               :distribution_version => distribution.results.first.release_version,
               :distribution_arch => distribution.results.first.arch,
               :distribution_family => distribution.results.first.release_name,
@@ -32,7 +42,7 @@ module Katello
             )
             unless distribution.results.first.variants.empty?
               unless distribution.results.first.variants.first.name.nil?
-                repo.update_attributes!(:distribution_variant => distribution.results.first.variants.first.name)
+                repo.update!(:distribution_variant => distribution.results.first.variants.first.name)
               end
             end
           end
@@ -45,6 +55,10 @@ module Katello
           end
         end
 
+        def partial_repo_path
+          "/pulp/repos/#{repo.relative_path}/".sub('//', '/')
+        end
+
         def copy_content_for_source
           # TODO
           fail NotImplementedError

data/app/services/katello/pulp3/repository_mirror.rb

@@ -52,6 +52,9 @@ module Katello
 
       def delete(href = repository_href)
         api.repositories_api.delete(href) if href
+      rescue api.class.api_exception_class => e
+        raise e if e.code != 404
+        nil
       end
 
       def repository_href
@@ -103,8 +106,7 @@ module Katello
       end
 
       def sync
-        api_module = api.class.client_module
-        repository_sync_url_data = api_module::RepositorySyncURL.new(remote: remote_href, mirror: true)
+        repository_sync_url_data = api.class.repository_sync_url_class.new(remote: remote_href, mirror: true)
         [api.repositories_api.sync(repository_href, repository_sync_url_data)]
       end
 

data/app/services/katello/pulp3/rpm.rb

@@ -39,18 +39,17 @@ module Katello
       def requires
         results = []
         flags = {'GT' => '>', 'LT' => '>', 'EQ' => '=', 'GE' => '>=', 'LE' => '<='}
-        if backend_data['requires']
-          backend_data['requires'].each do |requirement|
-            requires_str = ""
-            if requirement.count < 3
-              requires_str = requirement.first
-              results << requires_str
-            else
-              requirement[1] = flags[requirement[1]]
-              requirement[0...2].each { |requirement_piece| requires_str += "#{requirement_piece} " }
-              requirement[2...-1].each { |requirement_piece| requires_str += "#{requirement_piece}." }
-              results << requires_str[0...-1]
-            end
+
+        backend_data['requires']&.each do |requirement|
+          requires_str = ""
+          if requirement.count < 3
+            requires_str = requirement.first
+            results << requires_str
+          else
+            requirement[1] = flags[requirement[1]]
+            requirement[0...2].each { |requirement_piece| requires_str += "#{requirement_piece} " }
+            requirement[2...-1].each { |requirement_piece| requires_str += "#{requirement_piece}." }
+            results << requires_str[0...-1]
           end
         end
         results.uniq
@@ -59,18 +58,17 @@ module Katello
       def provides
         results = []
         flags = {'GT' => '>', 'LT' => '>', 'EQ' => '=', 'GE' => '>=', 'LE' => '<='}
-        if backend_data['provides']
-          backend_data['provides'].each do |provided|
-            provides_str = ""
-            if provided.count < 3
-              provides_str = provided.first
-              results << provides_str
-            else
-              provided[1] = flags[provided[1]]
-              provided[0...2].each { |provided_piece| provides_str += "#{provided_piece} " }
-              provided[2...-1].each { |provided_piece| provides_str += "#{provided_piece}." }
-              results << provides_str[0...-1]
-            end
+
+        backend_data['provides']&.each do |provided|
+          provides_str = ""
+          if provided.count < 3
+            provides_str = provided.first
+            results << provides_str
+          else
+            provided[1] = flags[provided[1]]
+            provided[0...2].each { |provided_piece| provides_str += "#{provided_piece} " }
+            provided[2...-1].each { |provided_piece| provides_str += "#{provided_piece}." }
+            results << provides_str[0...-1]
           end
         end
         results.uniq
@@ -136,7 +134,7 @@ module Katello
         custom_json['filename'] = backend_data['location_href']
         custom_json['checksum'] = backend_data['pkgId']
         custom_json['sourcerpm'] = backend_data['rpm_sourcerpm']
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/pulp3/smart_proxy_mirror_repository.rb

@@ -27,7 +27,7 @@ module Katello
           version_hrefs = api.repository_versions
           orphan_version_hrefs = api.list_all.collect do |pulp_repo|
             mirror_repo_versions = api.versions_list_for_repository(pulp_repo.pulp_href, ordering: :_created)
-            version_hrefs = mirror_repo_versions.collect { |version| version.pulp_href }
+            version_hrefs = mirror_repo_versions.select { |repo_version| repo_version.number != 0 }.collect { |version| version.pulp_href }
 
             version_hrefs - [pulp_repo.latest_version_href]
           end

data/app/services/katello/pulp3/smart_proxy_repository.rb

@@ -45,9 +45,9 @@ module Katello
         tasks.flatten
       end
 
-      def pulp3_enabled_repo_types
+      def pulp3_enabled_repo_types(check_pulp2_preferred = true)
         Katello::RepositoryTypeManager.repository_types.values.select do |repository_type|
-          smart_proxy.pulp3_repository_type_support?(repository_type)
+          smart_proxy.pulp3_repository_type_support?(repository_type, check_pulp2_preferred)
         end
       end
 
@@ -56,12 +56,32 @@ module Katello
         repo_version_map = {}
         pulp3_enabled_repo_types.each do |repo_type|
           api = repo_type.pulp3_service_class.api(smart_proxy)
-          version_hrefs = api.repository_versions
+          version_hrefs = api.repository_versions.select { |repo_version| repo_version.number != 0 }.map(&:pulp_href)
           repo_version_map[api] = version_hrefs - ::Katello::Repository.where(version_href: version_hrefs).pluck(:version_href)
         end
 
         repo_version_map
       end
+
+      def delete_orphan_repositories
+        tasks = []
+        orphan_repositories.each do |api, hrefs|
+          tasks << hrefs.collect do |href|
+            api.repositories_api.delete(href)
+          end
+        end
+        tasks.flatten
+      end
+
+      def orphan_repositories
+        repo_map = {}
+        pulp3_enabled_repo_types(false).each do |repo_type|
+          api = repo_type.pulp3_service_class.api(smart_proxy)
+          repo_hrefs = api.list_all.map(&:pulp_href)
+          repo_map[api] = repo_hrefs - ::Katello::Pulp3::RepositoryReference.where(repository_href: repo_hrefs).pluck(:repository_href)
+        end
+        repo_map
+      end
     end
   end
 end

data/app/services/katello/pulp3/srpm.rb

@@ -3,6 +3,8 @@ module Katello
     class Srpm < PulpContentUnit
       include LazyAccessor
 
+      CONTENT_TYPE = "srpm".freeze
+
       PULP_INDEXED_FIELDS = %w(pulp_href name version release arch epoch summary location_href pkgId).freeze
 
       lazy_accessor :pulp_facts, :initializer => :backend_data
@@ -67,7 +69,7 @@ module Katello
         custom_json['nvra'] = model.build_nvra
         custom_json['filename'] = backend_data['location_href']
         custom_json['checksum'] = backend_data['pkgId']
-        model.update_attributes!(custom_json)
+        model.update!(custom_json)
       end
     end
   end

data/app/services/katello/ui_notifications/subscriptions/expire_soon.rb

@@ -8,7 +8,7 @@ module Katello
               if (notification = notification_already_exists?(organization))
                 next unless organization.expiring_subscriptions.count.to_s ==
                   notification.message.split(' ').first
-                notification.update_attributes(
+                notification.update(
                   :message => message(organization),
                   :actions => actions
                 )