katello 3.18.0.rc1 → 3.18.0.rc2

data/app/lib/actions/katello/capsule_content/sync_capsule.rb

@@ -9,9 +9,8 @@ module Actions
           content_view = options[:content_view]
           repository = options[:repository]
           skip_metadata_check = options.fetch(:skip_metadata_check, false)
-
           sequence do
-            repos = repos_to_sync(smart_proxy, environment, content_view, repository)
+            repos = repos_to_sync(smart_proxy, environment, content_view, repository, skip_metadata_check)
 
             repos.in_groups_of(Setting[:foreman_proxy_content_batch_size], false) do |repo_batch|
               concurrence do
@@ -20,7 +19,6 @@ module Actions
                                     Actions::Pulp3::CapsuleContent::Sync],
                                      repo, smart_proxy,
                                      skip_metadata_check: skip_metadata_check)
-
                   if repo.is_a?(::Katello::Repository) &&
                     repo.distribution_bootable? &&
                     repo.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_ON_DEMAND
@@ -34,22 +32,33 @@ module Actions
           end
         end
 
-        def repos_to_sync(smart_proxy, environment, content_view, repository)
+        def repos_to_sync(smart_proxy, environment, content_view, repository, skip_metatadata_check = false)
           smart_proxy_helper = ::Katello::SmartProxyHelper.new(smart_proxy)
           smart_proxy_helper.lifecycle_environment_check(environment, repository)
-
           if repository
-            [repository]
+            if skip_metatadata_check || !repository.smart_proxy_sync_histories.where(:smart_proxy_id => smart_proxy).any? { |sph| !sph.finished_at.nil? }
+              [repository]
+            end
           else
             repositories = smart_proxy_helper.repositories_available_to_capsule(environment, content_view).by_rpm_count
             puppet_envs = smart_proxy_helper.puppet_environments_available_to_capsule(environment, content_view)
-            repositories + puppet_envs
+            repositories_to_skip = []
+            if skip_metatadata_check
+              smart_proxy_helper.clear_smart_proxy_sync_histories repositories
+            else
+              repositories_to_skip = ::Katello::Repository.synced_on_capsule smart_proxy
+            end
+            repositories - repositories_to_skip + puppet_envs
           end
         end
 
         def resource_locks
           :link
         end
+
+        def rescue_strategy
+          Dynflow::Action::Rescue::Skip
+        end
       end
     end
   end

data/app/lib/actions/katello/content_view/publish.rb

@@ -6,10 +6,10 @@ module Actions
         include ::Katello::ContentViewHelper
         attr_accessor :version
         # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity
-        def plan(content_view, description = "", options = {})
+        def plan(content_view, description = "", options = {importing: false})
           action_subject(content_view)
-          import_only = options.fetch(:import_only, false)
-          content_view.check_ready_to_publish! unless import_only
+
+          content_view.check_ready_to_publish!(importing: options[:importing])
 
           if options[:repos_units].present?
             valid_labels_from_cv = content_view.repositories.map(&:label)
@@ -52,7 +52,8 @@ module Actions
 
             if separated_repo_map[:pulp3_yum].keys.flatten.present? &&
                 SmartProxy.pulp_primary.pulp3_support?(separated_repo_map[:pulp3_yum].keys.flatten.first)
-              if import_only
+
+              if options[:importing]
                 handle_import(version, options.slice(:path, :metadata))
               else
                 plan_action(Repository::MultiCloneToVersion, separated_repo_map[:pulp3_yum], version)

data/app/lib/actions/katello/content_view_version/import.rb

@@ -2,28 +2,21 @@ module Actions
   module Katello
     module ContentViewVersion
       class Import < Actions::EntryAction
-        PULP_USER = 'pulp'.freeze
-
         def plan(content_view, path:, metadata:)
           content_view.check_ready_to_import!
           unless SmartProxy.pulp_primary.pulp3_repository_type_support?(::Katello::Repository::YUM_TYPE)
             fail ::Katello::HttpErrors::BadRequest, _("This API endpoint is only valid for Pulp 3 repositories.")
           end
-          ::Katello::Pulp3::ContentViewVersion::Import.check_permissions!(path)
+
+          ::Katello::Pulp3::ContentViewVersion::Import.check!(content_view: content_view, metadata: metadata, path: path)
 
           major = metadata[:content_view_version][:major]
           minor = metadata[:content_view_version][:minor]
 
-          if ::Katello::ContentViewVersion.where(major: major, minor: minor, content_view: content_view).exists?
-            cvv_name = "#{content_view.name} #{major}.#{minor}"
-            fail _("Content View Version specified in the metadata - '%{name}' already exists. "\
-                    "If you wish to replace the existing version, delete %{name} and try again. " % { name: cvv_name })
-          end
-
           plan_action(::Actions::Katello::ContentView::Publish, content_view, '',
                         path: path,
                         metadata: metadata,
-                        import_only: true,
+                        importing: true,
                         major: major,
                         minor: minor)
         end

data/app/lib/actions/katello/host/upload_package_profile.rb

@@ -39,7 +39,9 @@ module Actions
             Rails.logger.warn("Host with ID %s has no content facet, continuing" % host_id)
           else
             begin
-              ::Katello::Pulp::Consumer.new(host.content_facet.uuid).upload_package_profile(profile)
+              unless SmartProxy.pulp_primary&.pulp3_repository_type_support?(::Katello::Repository::YUM_TYPE)
+                ::Katello::Pulp::Consumer.new(host.content_facet.uuid).upload_package_profile(profile)
+              end
               simple_packages = profile.map { |item| ::Katello::Pulp::SimplePackage.new(item) }
               host.import_package_profile(simple_packages)
             rescue RestClient::ResourceNotFound

data/app/lib/actions/katello/host/upload_profiles.rb

@@ -60,12 +60,14 @@ module Actions
 
           module_stream_profile = updated_profiles + unassociated_profiles
 
-          unless module_stream_profile.empty?
-            begin
-              ::Katello::Pulp::Consumer.new(host.content_facet.uuid).
-                upload_module_stream_profile(module_stream_profile)
-            rescue RestClient::ResourceNotFound
-              Rails.logger.warn("Host with ID %s was not known to Pulp, continuing" % host.id)
+          unless SmartProxy.pulp_primary&.pulp3_repository_type_support?(::Katello::Repository::YUM_TYPE)
+            unless module_stream_profile.empty?
+              begin
+                ::Katello::Pulp::Consumer.new(host.content_facet.uuid).
+                  upload_module_stream_profile(module_stream_profile)
+              rescue RestClient::ResourceNotFound
+                Rails.logger.warn("Host with ID %s was not known to Pulp, continuing" % host.id)
+              end
             end
           end
         end

data/app/lib/actions/katello/repository/import_upload.rb

@@ -4,8 +4,10 @@ module Actions
     module Repository
       class ImportUpload < Actions::EntryAction
         include Actions::Katello::PulpSelector
+        # rubocop:disable Metrics/MethodLength
         def plan(repository, uploads, options = {})
           action_subject(repository)
+          repository.clear_smart_proxy_sync_histories
           repo_service = repository.backend_service(::SmartProxy.pulp_primary)
 
           upload_ids = uploads.pluck('id')

data/app/lib/actions/katello/repository/remove_content.rb

@@ -23,7 +23,7 @@ module Actions
             remove_content_args = {
               :contents => content_unit_ids,
               :content_unit_type => content_unit_type}
-
+            repository.clear_smart_proxy_sync_histories
             pulp_action = plan_pulp_action(
               [Pulp::Orchestration::Repository::RemoveUnits,
                Pulp3::Orchestration::Repository::RemoveUnits],

data/app/lib/actions/katello/repository/sync.rb

@@ -79,7 +79,9 @@ module Actions
         end
 
         def run
-          ForemanTasks.async_task(Repository::CapsuleSync, ::Katello::Repository.find(input[:id])) if Setting[:foreman_proxy_content_auto_sync]
+          repo = ::Katello::Repository.find(input[:id])
+          repo.clear_smart_proxy_sync_histories if input[:contents_changed]
+          ForemanTasks.async_task(Repository::CapsuleSync, repo) if Setting[:foreman_proxy_content_auto_sync]
         rescue ::Katello::Errors::CapsuleCannotBeReached # skip any capsules that cannot be connected to
         end
 

data/app/lib/actions/katello/repository/update.rb

@@ -60,6 +60,7 @@ module Actions
         def run
           repository = ::Katello::Repository.find(input[:repository_id])
           ForemanTasks.async_task(Katello::Repository::MetadataGenerate, repository)
+          repository.clear_smart_proxy_sync_histories
         end
 
         private

data/app/lib/actions/katello/repository/upload_files.rb

@@ -9,6 +9,7 @@ module Actions
         include Actions::Katello::PulpSelector
         def plan(repository, files, content_type = nil)
           action_subject(repository)
+          repository.clear_smart_proxy_sync_histories
           tmp_files = prepare_tmp_files(files)
 
           content_type ||= ::Katello::RepositoryTypeManager.find(repository.content_type).default_managed_content_type.label

data/app/lib/actions/middleware/record_smart_proxy_sync_history.rb

@@ -0,0 +1,15 @@
+module Actions
+  module Middleware
+    class RecordSmartProxySyncHistory < Dynflow::Middleware
+      def run(*args)
+        if (action.input[:repository_id] && (action.input[:smart_proxy_id] || action.input[:capsule_id]) && !self.action.output[:smart_proxy_history_id])
+          repo = ::Katello::Repository.find(action.input[:repository_id])
+          smart_proxy_id = action.input[:smart_proxy_id] || action.input[:capsule_id]
+          smart_proxy = ::SmartProxy.find(smart_proxy_id)
+          self.action.output[:smart_proxy_history_id] = repo.create_smart_proxy_sync_history(smart_proxy)
+        end
+        pass(*args)
+      end
+    end
+  end
+end

data/app/lib/actions/pulp/consumer/sync_capsule.rb

@@ -2,14 +2,16 @@ module Actions
   module Pulp
     module Consumer
       class SyncCapsule < ::Actions::Pulp::AbstractAsyncTask
+        include ::Actions::Helpers::SmartProxySyncHistoryHelper
         input_format do
           param :capsule_id, Integer
           param :repo_pulp_id, String
+          param :repository_id, Integer
           param :sync_options
         end
 
         def plan(repository, smart_proxy, options)
-          plan_self(:capsule_id => smart_proxy.id, :repo_pulp_id => repository.pulp_id, :sync_options => options)
+          plan_self(:capsule_id => smart_proxy.id, :repo_pulp_id => repository.pulp_id, :repository_id => repository.id, :sync_options => options)
         end
 
         def humanized_name
@@ -17,7 +19,7 @@ module Actions
         end
 
         def invoke_external_task
-          pulp_resources.repository.sync(input[:repo_pulp_id], override_config: input[:sync_options])
+          output[:pulp_tasks] = pulp_resources.repository.sync(input[:repo_pulp_id], override_config: input[:sync_options])
         end
 
         def run_progress

data/app/lib/actions/pulp/repository/distributor_publish.rb

@@ -20,7 +20,7 @@ module Actions
           else
             repo = ::Katello::ContentViewPuppetEnvironment.find_by(:id => input[:content_view_puppet_environment_id]).nonpersisted_repository
           end
-
+          repo.clear_smart_proxy_sync_histories if smart_proxy(input[:smart_proxy_id]).pulp_primary?
           repo.backend_service(smart_proxy(input[:smart_proxy_id])).distributor_publish(input[:options])
         end
 

data/app/lib/actions/pulp3/abstract_async_task.rb

@@ -74,6 +74,7 @@ module Actions
 
       def cancel
         pulp_tasks.each { |task| task.cancel }
+        task_groups.each { |task_group| task_group.cancel }
       end
 
       def rescue_external_task(error)

data/app/lib/actions/pulp3/capsule_content/sync.rb

@@ -2,6 +2,7 @@ module Actions
   module Pulp3
     module CapsuleContent
       class Sync < Pulp3::AbstractAsyncTask
+        include ::Actions::Helpers::SmartProxySyncHistoryHelper
         def plan(repository, smart_proxy, options = {})
           sequence do
             plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :options => options)

data/app/lib/actions/pulp3/content_view_version/export.rb

@@ -4,6 +4,7 @@ module Actions
       class Export < Pulp3::AbstractAsyncTask
         input_format do
           param :content_view_version_id, Integer
+          param :from_content_view_version_id, Integer
           param :smart_proxy_id, Integer
           param :exporter_data, Hash
           param :chunk_size, Integer
@@ -11,8 +12,11 @@ module Actions
 
         def invoke_external_task
           cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+          from_cvv = ::Katello::ContentViewVersion.find(input[:from_content_view_version_id]) unless input[:from_content_view_version_id].blank?
           ::Katello::Pulp3::ContentViewVersion::Export.new(smart_proxy: smart_proxy,
-                                                   content_view_version: cvv).create_export(input[:exporter_data][:pulp_href], input[:chunk_size])
+                                                   content_view_version: cvv,
+                                                   from_content_view_version: from_cvv).create_export(input[:exporter_data][:pulp_href],
+                                                                                            chunk_size: input[:chunk_size])
         end
       end
     end

data/app/lib/actions/pulp3/orchestration/content_view_version/export.rb

@@ -6,6 +6,7 @@ module Actions
           input_format do
             param :smart_proxy_id, Integer
             param :content_view_version_id, Integer
+            param :from_content_view_version_id, Integer
             param :export_history_id, Integer
             param :exporter_data, Hash
             param :destination_server, String
@@ -16,7 +17,8 @@ module Actions
             param :export_path, String
           end
 
-          def plan(content_view_version, destination_server:, chunk_size: nil)
+          # rubocop:disable Metrics/MethodLength
+          def plan(content_view_version, destination_server:, chunk_size: nil, from_history: nil)
             action_subject(content_view_version)
             unless File.directory?(Setting['pulpcore_export_destination'])
               fail ::Foreman::Exception, N_("Unable to export. 'pulpcore_export_destination' setting is not set to a valid directory.")
@@ -24,6 +26,16 @@ module Actions
 
             sequence do
               smart_proxy = SmartProxy.pulp_primary!
+              from_content_view_version = from_history&.content_view_version
+              if from_content_view_version.present?
+                export_service = ::Katello::Pulp3::ContentViewVersion::Export.new(
+                                                       smart_proxy: smart_proxy,
+                                                       content_view_version: content_view_version,
+                                                       destination_server: destination_server,
+                                                       from_content_view_version: from_content_view_version)
+                export_service.validate_incremental_export!
+              end
+
               action_output = plan_action(::Actions::Pulp3::ContentViewVersion::CreateExporter,
                                      content_view_version_id: content_view_version.id,
                                      smart_proxy_id: smart_proxy.id,
@@ -33,12 +45,13 @@ module Actions
                                      content_view_version_id: content_view_version.id,
                                      smart_proxy_id: smart_proxy.id,
                                      exporter_data: action_output[:exporter_data],
-                                     chunk_size: chunk_size
-                                     )
+                                     chunk_size: chunk_size,
+                                     from_content_view_version_id: from_content_view_version&.id)
 
               plan_self(exporter_data: action_output[:exporter_data], smart_proxy_id: smart_proxy.id,
                         destination_server: destination_server,
-                        content_view_version_id: content_view_version.id)
+                        content_view_version_id: content_view_version.id,
+                        from_content_view_version_id: from_content_view_version&.id)
 
               plan_action(::Actions::Pulp3::ContentViewVersion::DestroyExporter,
                             smart_proxy_id: smart_proxy.id,
@@ -55,9 +68,13 @@ module Actions
             path = File.dirname(file_name.to_s)
             output[:export_path] = path
             cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+            from_cvv = ::Katello::ContentViewVersion.find(input[:from_content_view_version_id]) unless input[:from_content_view_version_id].blank?
 
-            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(:content_view_version => cvv,
-                                                               :smart_proxy => smart_proxy).generate_metadata
+            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(
+                                                       content_view_version: cvv,
+                                                       smart_proxy: smart_proxy,
+                                                       from_content_view_version: from_cvv).generate_metadata
+            export_metadata[:incremental] = from_cvv.present?
             toc = Dir.glob("#{path}/*toc.json").first
             export_metadata[:toc] = File.basename(toc) if toc
             ::Katello::ContentViewVersionExportHistory.create!(

data/app/lib/actions/pulp3/orchestration/repository/generate_metadata.rb

@@ -8,7 +8,7 @@ module Actions
             publication_content_type = !::Katello::RepositoryTypeManager.find(repository.content_type).pulp3_skip_publication
             sequence do
               if options[:source_repository] && publication_content_type
-                plan_self(source_repository_id: options[:source_repository].id, target_repository_id: repository.id)
+                plan_self(source_repository_id: options[:source_repository].id, target_repository_id: repository.id, smart_proxy_id: smart_proxy.id)
               elsif publication_content_type
                 plan_action(Actions::Pulp3::Repository::CreatePublication, repository, smart_proxy, options)
               end
@@ -21,6 +21,9 @@ module Actions
             #we don't have to actually generate a publication, we can reuse the old one
             target_repo = ::Katello::Repository.find(input[:target_repository_id])
             source_repo = ::Katello::Repository.find(input[:source_repository_id])
+            if (target_repo.publication_href != source_repo.publication_href && smart_proxy.pulp_primary?)
+              target_repo.clear_smart_proxy_sync_histories
+            end
             target_repo.update!(publication_href: source_repo.publication_href)
           end
         end

data/app/models/katello/authorization/content_view_filter.rb

@@ -0,0 +1,15 @@
+module Katello
+  module Authorization::ContentViewFilter
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def readable
+        where(:content_view_id => ::Katello::ContentView.readable)
+      end
+
+      def editable
+        where(:content_view_id => ::Katello::ContentView.editable)
+      end
+    end
+  end
+end

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -49,6 +49,8 @@ module Katello
         has_many :content_facets, :class_name => "::Katello::Host::ContentFacet", :foreign_key => :content_source_id,
                                   :inverse_of => :content_source, :dependent => :nullify
 
+        has_many :smart_proxy_sync_histories, :class_name => "::Katello::SmartProxySyncHistory", :inverse_of => :smart_proxy, dependent: :delete_all
+
         has_many :hostgroup_content_facets, :class_name => "::Katello::Hostgroup::ContentFacet", :foreign_key => :content_source_id,
                               :inverse_of => :content_source, :dependent => :nullify
         has_many :hostgroups, :class_name => "::Hostgroup", :through => :hostgroup_content_facets

data/app/models/katello/content_view.rb

@@ -67,6 +67,14 @@ module Katello
     validates :composite,
               inclusion: { in: [false], message: "Composite Content Views can not solve dependencies"},
               if: :solve_dependencies
+    validates :import_only, :inclusion => [true, false]
+    validates :import_only,
+              inclusion: { in: [false], message: "Import-only Content Views can not be Composite"},
+              if: :composite
+    validates :import_only,
+              inclusion: { in: [false], message: "Import-only Content Views can not solve dependencies"},
+              if: :solve_dependencies
+    validate :import_only_immutable
 
     validates_with Validators::KatelloNameFormatValidator, :attributes => :name
     validates_with Validators::KatelloLabelFormatValidator, :attributes => :label
@@ -580,16 +588,23 @@ module Katello
     end
 
     def check_ready_to_import!
-      fail _("User must be logged in.") if ::User.current.nil?
       fail _("Cannot import a composite content view") if composite?
+      fail _("This Content View must be set to Import-only before performing an import") unless import_only?
       true
     end
 
-    def check_ready_to_publish!
+    def check_ready_to_publish!(importing: false)
       fail _("User must be logged in.") if ::User.current.nil?
       fail _("Cannot publish default content view") if default?
-      check_composite_action_allowed!(organization.library)
-      check_docker_repository_names!([organization.library])
+
+      if importing
+        check_ready_to_import!
+      else
+        fail _("Import-only content views can not be published directly") if import_only?
+        check_composite_action_allowed!(organization.library)
+        check_docker_repository_names!([organization.library])
+      end
+
       true
     end
 
@@ -698,6 +713,12 @@ module Katello
 
     private
 
+    def import_only_immutable
+      if import_only_changed? && self.persisted?
+        errors.add(:import_only, _("Import-only can not be changed after creation"))
+      end
+    end
+
     def generate_cp_environment_label(env)
       # The label for a default view, will simply be the env label; otherwise, it
       # will be a combination of env and view label.  The reason being, the label