katello 4.9.0.rc1 → 4.9.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ca8e5c339f17e38c34b16b000f420cdd7dd1d728826db1bd7bc04869e2d93c1
-  data.tar.gz: 01a11df500f4474eb063ca3a213babb50b72181dc03468a6d59e48c70aa15636
+  metadata.gz: d870b3e8317aae2cc5675bdebddc2541257c470714209df06d79135ebc751643
+  data.tar.gz: e19a405204719de4c723e6b7a0c4612e40983698df33562f74e28e0eb2fb83f6
 SHA512:
-  metadata.gz: aa294960e367aab5f65d0b3a7fd1513d4cd5f1f5534cb591b055bab41ea3599428645d712cbb57a0a8a8b8739a61fbc8c5a4a5234b4a6897f617586bcddeb1e6
-  data.tar.gz: 4f1cb92f021810d29b49843d2e439dae4ea155b120bfbcebe008a25ec0b85235f5ae20e6b063f3a399099318c3bfe68d550772bb2609eb4b154bdd0eebcb031d
+  metadata.gz: ee637b366b689e48fdbde217e2255b683c35095ffcbc3ad1234b9446ba7757be992a5b33a61650dc1fc5ac25ff4474185e2e58772991475c0f6a78627bbd70c4
+  data.tar.gz: 6eac40d9467c4365c19c82ecdce5d53f42418f511c7e35e12f987c5e58b432c3919f0ff0a2c1d581084e14e8f1ca29147f298a505331ec1d6907a6f132b4e209

data/app/lib/actions/katello/content_view/update.rb

@@ -4,6 +4,30 @@ module Actions
       class Update < Actions::EntryAction
         def plan(content_view, content_view_params)
           action_subject content_view
+          content_view_params = content_view_params.with_indifferent_access
+
+          # If we are removing repositories, remove their filter rules
+          if content_view.filters.present? && content_view.repository_ids.present? && content_view_params.key?(:repository_ids)
+            repo_ids_to_remove = content_view.repository_ids - content_view_params[:repository_ids]
+            if repo_ids_to_remove.present?
+              # Only yum-type repositories have by-ID filter rules
+              old_repos = content_view.repositories.yum_type
+              new_repos = ::Katello::Repository.where(id: content_view_params[:repository_ids]).yum_type
+
+              lost_module_stream_ids = (::Katello::ModuleStream.in_repositories(old_repos) -
+                ::Katello::ModuleStream.in_repositories(new_repos)).pluck(:id)
+              ::Katello::ContentViewModuleStreamFilterRule.in_content_views([content_view.id]).where(module_stream_id: lost_module_stream_ids).delete_all
+
+              lost_errata_ids = (::Katello::Erratum.in_repositories(old_repos) -
+                ::Katello::Erratum.in_repositories(new_repos)).pluck(:errata_id)
+              ::Katello::ContentViewErratumFilterRule.in_content_views([content_view.id]).where(errata_id: lost_errata_ids).delete_all
+
+              lost_package_group_hrefs = (::Katello::PackageGroup.in_repositories(old_repos) -
+                ::Katello::PackageGroup.in_repositories(new_repos)).pluck(:pulp_id)
+              ::Katello::ContentViewPackageGroupFilterRule.in_content_views([content_view.id]).where(uuid: lost_package_group_hrefs).delete_all
+            end
+          end
+
           content_view.update!(content_view_params)
         end
       end

data/app/lib/actions/pulp3/orchestration/orphan_cleanup/remove_orphans.rb

@@ -7,13 +7,13 @@ module Actions
             if proxy.pulp3_enabled?
               sequence do
                 plan_action(Actions::Pulp3::OrphanCleanup::DeleteOrphanRepositoryVersions, proxy)
-                plan_action(Actions::Pulp3::OrphanCleanup::RemoveOrphans, proxy)
                 if proxy.pulp_mirror?
                   plan_action(Actions::Pulp3::OrphanCleanup::RemoveUnneededRepos, proxy)
                   plan_action(Actions::Pulp3::OrphanCleanup::DeleteOrphanDistributions, proxy)
                   plan_action(Actions::Pulp3::OrphanCleanup::DeleteOrphanAlternateContentSources, proxy)
                   plan_action(Actions::Pulp3::OrphanCleanup::DeleteOrphanRemotes, proxy)
                 end
+                plan_action(Actions::Pulp3::OrphanCleanup::RemoveOrphans, proxy)
               end
             end
           end

data/app/models/katello/concerns/pulp_database_unit.rb

@@ -167,6 +167,14 @@ module Katello
         where("#{self.table_name}.id in (?) or #{self.table_name}.pulp_id in (?)", id_integers, ids)
       end
 
+      def orphaned
+        if many_repository_associations
+          where.not(:id => repository_association_class.where(:repository_id => ::Katello::Repository.all).select(unit_id_field))
+        else
+          where.not(:repository_id => ::Katello::Repository.all)
+        end
+      end
+
       def in_repositories(repos)
         if many_repository_associations
           where(:id => repository_association_class.where(:repository_id => repos).select(unit_id_field))

data/app/models/katello/content_view.rb

@@ -95,11 +95,13 @@ module Katello
                                :repository_import,
                                :repository_export_syncable])
     }
-    scope :ignore_generated, -> {
-      where.not(:generated_for => [:repository_export,
-                                   :repository_import,
-                                   :library_export_syncable,
-                                   :repository_export_syncable])
+    scope :ignore_generated, ->(include_library_generated: false) {
+      ignored_values = [:repository_export,
+                        :repository_import,
+                        :library_export_syncable,
+                        :repository_export_syncable]
+      ignored_values += [:library_export, :library_import] if include_library_generated
+      where.not(generated_for: ignored_values)
     }
     scope :generated_for_library, -> { where(:generated_for => [:library_export, :library_import, :library_export_syncable]) }
 

data/app/models/katello/content_view_erratum_filter_rule.rb

@@ -26,6 +26,11 @@ module Katello
         :message => (_("must be one of the following: %s") % DATE_TYPES.join(', '))
       }
 
+    def self.in_content_views(content_view_ids)
+      joins('INNER JOIN katello_content_view_filters ON katello_content_view_erratum_filter_rules.content_view_filter_id = katello_content_view_filters.id').
+        where("katello_content_view_filters.content_view_id IN (#{content_view_ids.join(',')})")
+    end
+
     def filter_has_date_or_type_rule?
       filter.erratum_rules.any? { |rule| rule.start_date || rule.end_date || !rule.types.blank? }
     end

data/app/models/katello/content_view_module_stream_filter_rule.rb

@@ -7,5 +7,10 @@ module Katello
                :foreign_key => :content_view_filter_id
     belongs_to :module_stream, :class_name => "Katello::ModuleStream", :inverse_of => :rules
     validates :module_stream_id, :presence => true, :uniqueness => { :scope => :content_view_filter_id }
+
+    def self.in_content_views(content_view_ids)
+      joins('INNER JOIN katello_content_view_filters ON katello_content_view_module_stream_filter_rules.content_view_filter_id = katello_content_view_filters.id').
+        where("katello_content_view_filters.content_view_id IN (#{content_view_ids.join(',')})")
+    end
   end
 end

data/app/models/katello/content_view_package_group_filter_rule.rb

@@ -8,5 +8,10 @@ module Katello
                :foreign_key => :content_view_filter_id
 
     validates :uuid, :presence => true, :uniqueness => { :scope => :content_view_filter_id }
+
+    def self.in_content_views(content_view_ids)
+      joins('INNER JOIN katello_content_view_filters ON katello_content_view_package_group_filter_rules.content_view_filter_id = katello_content_view_filters.id').
+        where("katello_content_view_filters.content_view_id IN (#{content_view_ids.join(',')})")
+    end
   end
 end

data/app/models/katello/repository.rb

@@ -673,7 +673,7 @@ module Katello
 
     def self.smart_proxy_syncable
       joins(:content_view_version => :content_view).
-        merge(ContentView.ignore_generated).
+        merge(ContentView.ignore_generated(include_library_generated: true)).
         where.not(environment_id: nil)
     end
 
@@ -935,6 +935,7 @@ module Katello
         repository_type.content_types_to_index.each do |type|
           Katello::Logging.time("CONTENT_INDEX", data: {type: type.model_class}) do
             Katello::ContentUnitIndexer.new(content_type: type, repository: self, optimized: !full_index).import_all
+            type.model_class.orphaned.destroy_all
           end
         end
         repository_type.index_additional_data_proc&.call(self)

data/app/services/katello/content_unit_indexer.rb

@@ -142,14 +142,39 @@ module Katello
     def sync_repository_associations(assocication_tracker, additive: false)
       unless additive
         ActiveRecord::Base.connection.uncached do
-          @model_class.repository_association_class.where(repository_id: @repository.id).where.
-            not(@model_class.unit_id_field => assocication_tracker.unit_ids).delete_all
+          repo_associations_to_destroy = @model_class.repository_association_class.where(repository_id: @repository.id).where.
+            not(@model_class.unit_id_field => assocication_tracker.unit_ids)
+          clean_filter_rules(repo_associations_to_destroy) if repo_associations_to_destroy.present? && [::Katello::ModuleStream, ::Katello::Erratum, ::Katello::PackageGroup].include?(@model_class)
+          repo_associations_to_destroy.destroy_all
         end
       end
       return if assocication_tracker.db_values.empty?
       @model_class.repository_association_class.upsert_all(assocication_tracker.db_values, :unique_by => association_class_uniqiness_attributes)
     end
 
+    def clean_filter_rules(repo_associations_to_destroy)
+      affected_content_view_ids = @repository.content_views.non_default.pluck(:id)
+      return false if affected_content_view_ids.empty?
+      case @model_class.to_s
+      when 'Katello::ModuleStream'
+        module_stream_ids = repo_associations_to_destroy.pluck(:module_stream_id)
+        filter_rules = ::Katello::ContentViewModuleStreamFilterRule.
+          in_content_views(affected_content_view_ids).where(module_stream_id: module_stream_ids)
+        filter_rules.delete_all
+      when 'Katello::Erratum'
+        errata_ids = ::Katello::Erratum.where(id: repo_associations_to_destroy.select(:erratum_id)).pluck(:errata_id)
+        filter_rules = ::Katello::ContentViewErratumFilterRule.in_content_views(affected_content_view_ids).where(errata_id: errata_ids)
+        filter_rules.delete_all
+      when 'Katello::PackageGroup'
+        package_group_uuids = ::Katello::PackageGroup.where(id: repo_associations_to_destroy.select(:package_group_id)).pluck(:pulp_id)
+        filter_rules = ::Katello::ContentViewPackageGroupFilterRule.
+          in_content_views(affected_content_view_ids).where(uuid: package_group_uuids)
+        filter_rules.delete_all
+      else
+        return false
+      end
+    end
+
     def association_class_uniqiness_attributes
       columns = [@model_class.unit_id_field, 'repository_id']
       found = ActiveRecord::Base.connection.indexes(@model_class.repository_association_class.table_name).find do |index|

data/app/services/katello/pulp3/api/core.rb

@@ -168,7 +168,7 @@ module Katello
         end
 
         def delete_orphans
-          [orphans_api.cleanup(PulpcoreClient::OrphansCleanup.new(orphan_protection_time: Setting[:orphan_protection_time]))]
+          [orphans_api.cleanup(PulpcoreClient::OrphansCleanup.new(orphan_protection_time: (smart_proxy.pulp_mirror? ? 0 : Setting[:orphan_protection_time])))]
         end
 
         def delete_remote(remote_href)

data/app/services/katello/pulp3/repository/docker.rb

@@ -67,6 +67,13 @@ module Katello
         def add_content(content_unit_href)
           content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
           api.repositories_api.add(repository_reference.repository_href, content_units: content_unit_href)
+        rescue api.client_module::ApiError => e
+          if e.message.include? 'Could not find the following content units'
+            raise ::Katello::Errors::Pulp3Error, "Content units that do not exist in Pulp were requested to be copied."\
+              " Please run a complete sync on the following repository: #{repository_reference.root_repository.name}. Original error: #{e.message}"
+          else
+            raise e
+          end
         end
 
         def copy_units_recursively(unit_hrefs, clear_repo = false)

data/app/services/katello/pulp3/repository.rb

@@ -493,11 +493,26 @@ module Katello
         else
           api.repositories_api.modify(repository_reference.repository_href, add_content_units: content_unit_href)
         end
+      rescue api.client_module::ApiError => e
+        if e.message.include? 'Could not find the following content units'
+          raise ::Katello::Errors::Pulp3Error, "Content units that do not exist in Pulp were requested to be copied."\
+            " Please run a complete sync on the following repository: #{repository_reference.root_repository.name}. Original error: #{e.message}"
+        else
+          raise e
+        end
       end
 
       def add_content_for_repo(repository_href, content_unit_href)
         content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
         api.repositories_api.modify(repository_href, add_content_units: content_unit_href)
+      rescue api.client_module::ApiError => e
+        if e.message.include? 'Could not find the following content units'
+          raise ::Katello::Errors::Pulp3Error, "Content units that do not exist in Pulp were requested to be copied."\
+            " Please run a complete sync on the following repository:"\
+            " #{::Katello::Pulp3::RepositoryReference.find_by(repository_href: repository_href).root_repository.name}. Original error: #{e.message}"
+        else
+          raise e
+        end
       end
 
       def unit_keys(uploads)

data/app/services/katello/smart_proxy_helper.rb

@@ -21,6 +21,11 @@ module Katello
       end
     end
 
+    def library_export_repo(repository = nil)
+      cv = repository.content_view if repository
+      cv.library_export?
+    end
+
     def clear_smart_proxy_sync_histories(repo_list = [])
       if repo_list.empty?
         @smart_proxy.smart_proxy_sync_histories.delete_all
@@ -32,7 +37,7 @@ module Katello
 
     def combined_repos_available_to_capsule(environment = nil, content_view = nil, repository = nil)
       lifecycle_environment_check(environment, repository)
-      if repository
+      if repository && !library_export_repo(repository)
         [repository]
       else
         repositories_available_to_capsule(environment, content_view)

data/lib/katello/plugin.rb

@@ -670,7 +670,7 @@ Foreman::Plugin.register :katello do
         type: :integer,
         default: 1440,
         full_name: N_('Orphaned Content Protection Time'),
-        description: N_('Time in minutes to consider orphan content as orphaned.')
+        description: N_('Time in minutes before content that is not contained within a repository and has not been accessed is considered orphaned.')
 
       setting 'remote_execution_prefer_registered_through_proxy',
         type: :boolean,

data/lib/katello/tasks/upgrades/4.9/clean_orphaned_filter_rules.rake

@@ -0,0 +1,40 @@
+namespace :katello do
+  namespace :upgrades do
+    namespace '4.9' do
+      desc "Clean orphaned filter rules that cause Pulp copy errors during content view publishing"
+      task :clean_orphaned_filter_rules => ['environment'] do
+        module_stream_count = 0
+        erratum_count = 0
+        package_group_count = 0
+
+        ::Katello::ContentViewModuleStreamFilterRule.all.each do |rule|
+          # Delete if rule exists in a CV that does not have the matching module stream in its repositories
+          content_view = rule.filter.content_view
+          unless ::Katello::ModuleStream.in_repositories(content_view.repositories)&.pluck(:id)&.include?(rule.module_stream_id)
+            rule.delete
+            module_stream_count += 1
+          end
+        end
+        puts "#{module_stream_count} orphaned content view module stream filter rules were deleted."
+
+        ::Katello::ContentViewErratumFilterRule.all.each do |rule|
+          content_view = rule.filter.content_view
+          unless ::Katello::Erratum.in_repositories(content_view.repositories)&.pluck(:errata_id)&.include?(rule.errata_id)
+            rule.delete
+            erratum_count += 1
+          end
+        end
+        puts "#{erratum_count} orphaned content view erratum filter rules were deleted."
+
+        ::Katello::ContentViewPackageGroupFilterRule.all.each do |rule|
+          content_view = rule.filter.content_view
+          unless ::Katello::PackageGroup.in_repositories(content_view.repositories)&.pluck(:pulp_id)&.include?(rule.uuid)
+            rule.delete
+            package_group_count += 1
+          end
+        end
+        puts "#{package_group_count} orphaned content view package group filter rules were deleted."
+      end
+    end
+  end
+end

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "4.9.0.rc1".freeze
+  VERSION = "4.9.0.rc2".freeze
 end

data/locale/Makefile

@@ -10,6 +10,7 @@ DOMAIN = katello
 VERSION = $(shell git describe --abbrev=0 --tags)
 POTFILE = $(DOMAIN).pot
 MOFILE = $(DOMAIN).mo
+ACTIONFILE = action_names.rb
 POFILES = $(shell find . -name '$(DOMAIN).po')
 MOFILES = $(patsubst %.po,%.mo,$(POFILES))
 POXFILES = $(patsubst %.po,%.pox,$(POFILES))
@@ -50,11 +51,11 @@ tx-pull: $(EDITFILES)
 
 tx-update: tx-pull
 	@echo
-	@echo Run rake plugin:gettext[$(DOMAIN)] from the Foreman installation, then make -C locale mo-files to finish
+	@echo Run rake plugin:gettext[$(DOMAIN)] and rake plugin:po_to_json[$(DOMAIN)] from the Foreman installation, then make -C locale po-files to finish
 	@echo
 
-mo-files: $(MOFILES)
-	git add $(POFILES) $(POTFILE) ../locale/*/LC_MESSAGES
+commit-translation-files: $(POFILES)
+	git add $(POFILES) $(POTFILE) $(ACTIONFILE) ../app/assets/javascripts/katello/locale
 	git commit -m "i18n - pulling from tx"
 	@echo
 	@echo Changes commited!

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: katello
 version: !ruby/object:Gem::Version
-  version: 4.9.0.rc1
+  version: 4.9.0.rc2
 platform: ruby
 authors:
 - N/A
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-26 00:00:00.000000000 Z
+date: 2023-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -4482,6 +4482,7 @@ files:
 - lib/katello/tasks/upgrades/4.4/publish_import_cvvs.rake
 - lib/katello/tasks/upgrades/4.8/fix_incorrect_providers.rake
 - lib/katello/tasks/upgrades/4.8/regenerate_imported_repository_metadata.rake
+- lib/katello/tasks/upgrades/4.9/clean_orphaned_filter_rules.rake
 - lib/katello/tasks/upgrades/4.9/update_custom_products_enablement.rake
 - lib/katello/tasks/virt_who_report.rake
 - lib/katello/url_constrained_cookie_store.rb
@@ -5502,7 +5503,7 @@ homepage: http://www.katello.org
 licenses:
 - GPL-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -5517,8 +5518,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: Content and Subscription Management plugin for Foreman
 test_files: []