katello 3.18.0.rc1 → 3.18.0.rc2

data/app/models/katello/content_view_filter.rb

@@ -1,5 +1,6 @@
 module Katello
   class ContentViewFilter < Katello::Model
+    include Authorization::ContentViewFilter
     audited :associations => [:repositories]
     DOCKER = 'docker'.freeze
     RPM = Rpm::CONTENT_TYPE
@@ -157,6 +158,10 @@ module Katello
       if self.content_view.composite?
         errors.add(:base, _("cannot contain filters if composite view"))
       end
+
+      if self.content_view.import_only?
+        errors.add(:base, _("cannot add filter to import-only view"))
+      end
     end
 
     def validate_filter_repos(errors, content_view)

data/app/models/katello/content_view_puppet_module.rb

@@ -17,6 +17,8 @@ module Katello
 
     before_validation :set_attributes
 
+    validate :import_only_content_view
+
     def puppet_module
       PuppetModule.find_by(:pulp_id => self.uuid)
     end
@@ -42,6 +44,12 @@ module Katello
 
     private
 
+    def import_only_content_view
+      if self.content_view.import_only?
+        errors.add(:base, "Cannot add puppet modules to an import-only content view.")
+      end
+    end
+
     def set_attributes
       if self.uuid.present?
         puppet_module = PuppetModule.with_identifiers(self.uuid).first

data/app/models/katello/content_view_repository.rb

@@ -7,6 +7,10 @@ module Katello
                                 Repository::DEB_TYPE
                                ].freeze
 
+    ALLOWED_IMPORT_REPOSITORY_TYPES = [
+      Repository::YUM_TYPE
+    ].freeze
+
     belongs_to :content_view, :inverse_of => :content_view_repositories,
                               :class_name => "Katello::ContentView"
     belongs_to :repository, :inverse_of => :content_view_repositories,
@@ -27,7 +31,7 @@ module Katello
     end
 
     def ensure_repository_type
-      unless ALLOWED_REPOSITORY_TYPES.include?(repository.content_type)
+      unless allowed_repository_types.include?(repository.content_type)
         errors.add(:base, _("Cannot add %s repositories to a content view.") % repository.content_type)
       end
     end
@@ -41,5 +45,13 @@ module Katello
         errors.add(:base, _("Repositories from published Content Views are not allowed."))
       end
     end
+
+    def allowed_repository_types
+      if self.content_view.import_only?
+        ALLOWED_IMPORT_REPOSITORY_TYPES
+      else
+        ALLOWED_REPOSITORY_TYPES
+      end
+    end
   end
 end

data/app/models/katello/ping.rb

@@ -5,10 +5,17 @@ module Katello
     PACKAGES = %w(katello candlepin pulp qpid foreman tfm hammer).freeze
 
     class << self
+      def pulpcore_enabled # for downstream 6.9, remove in 6.10
+        SETTINGS[:katello][:use_pulp_2_for_content_type].nil? || (!SETTINGS[:katello][:use_pulp_2_for_content_type][:yum] &&
+        !SETTINGS[:katello][:use_pulp_2_for_content_type][:docker] &&
+        !SETTINGS[:katello][:use_pulp_2_for_content_type][:file]) ||
+        system('systemctl is-enabled pulpcore-api.service')
+      end
+
       def services(capsule_id = nil)
         proxy = fetch_proxy(capsule_id)
         services = [:candlepin, :candlepin_auth, :foreman_tasks, :katello_events, :candlepin_events]
-        services += [:pulp3] if proxy&.pulp3_enabled?
+        services += [:pulp3] if proxy&.pulp3_enabled? && pulpcore_enabled
         if proxy.nil? || proxy.has_feature?(SmartProxy::PULP_NODE_FEATURE) || proxy.has_feature?(SmartProxy::PULP_FEATURE)
           services += [:pulp, :pulp_auth]
         end
@@ -16,9 +23,7 @@ module Katello
         services
       end
 
-      #
       # Calls "status" services in all backend engines.
-      #
       def ping(services: nil, capsule_id: nil)
         services ||= self.services(capsule_id)
         result = {}

data/app/models/katello/repository.rb

@@ -109,6 +109,8 @@ module Katello
     has_many :distribution_references, :class_name => 'Katello::Pulp3::DistributionReference', :foreign_key => :repository_id,
              :dependent => :destroy, :inverse_of => :repository
 
+    has_many :smart_proxy_sync_histories, :class_name => "::Katello::SmartProxySyncHistory", :inverse_of => :repository, :dependent => :delete_all
+
     validates_with Validators::ContainerImageNameValidator, :attributes => :container_repository_name, :allow_blank => false, :if => :docker?
     validates :container_repository_name, :if => :docker?, :uniqueness => {message: ->(object, _data) do
       _("for repository '%{name}' is not unique and cannot be created in '%{env}'. Its Container Repository Name (%{container_name}) conflicts with an existing repository.  Consider changing the Lifecycle Environment's Registry Name Pattern to something more specific.") %
@@ -531,6 +533,30 @@ module Katello
       clone
     end
 
+    def self.synced_on_capsule(smart_proxy)
+      smart_proxy.smart_proxy_sync_histories.map { |sph| sph.repository unless sph.finished_at.nil? }
+    end
+
+    def clear_smart_proxy_sync_histories(smart_proxy = nil)
+      if smart_proxy
+        self.smart_proxy_sync_histories.where(:smart_proxy_id => smart_proxy.id).try(:delete_all)
+      else
+        self.smart_proxy_sync_histories.delete_all
+      end
+    end
+
+    def create_smart_proxy_sync_history(smart_proxy)
+      clear_smart_proxy_sync_histories(smart_proxy)
+      sp_history_args = {
+        :smart_proxy_id => smart_proxy.id,
+        :repository_id => self.id,
+        :started_at => Time.now
+      }
+      sp_history = ::Katello::SmartProxySyncHistory.create sp_history_args
+      sp_history.save!
+      sp_history.id
+    end
+
     def latest_sync_audit
       self.audits.where(:action => AUDIT_SYNC_ACTION).order(:created_at).last
     end

data/app/models/katello/smart_proxy_sync_history.rb

@@ -0,0 +1,8 @@
+module Katello
+  class SmartProxySyncHistory < Katello::Model
+    self.table_name = 'katello_smart_proxy_sync_history'
+
+    belongs_to :smart_proxy, :class_name => "::SmartProxy", :inverse_of => :smart_proxy_sync_histories
+    belongs_to :repository, :class_name => "Katello::Repository", :inverse_of => :smart_proxy_sync_histories
+  end
+end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -4,10 +4,11 @@ module Katello
       class Export
         include ImportExportCommon
 
-        def initialize(smart_proxy:, content_view_version: nil, destination_server: nil)
+        def initialize(smart_proxy:, content_view_version: nil, destination_server: nil, from_content_view_version: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
+          @from_content_view_version = from_content_view_version
         end
 
         def generate_exporter_path
@@ -25,9 +26,32 @@ module Katello
                                   repositories: repository_hrefs)
         end
 
-        def create_export(exporter_href, chunk_size = nil)
+        def create_export(exporter_href, chunk_size: nil)
           options = { versions: version_hrefs }
           options[:chunk_size] = "#{chunk_size}MB" if chunk_size
+          if @from_content_view_version
+            from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
+            start_versions = from_exporter.version_hrefs
+
+            # current_cvv - cvv_from , i.e. repos in current cvv that are not in from
+            # implying something got added to the current cvv
+            # make sure you set the start_versions as 0
+            added_repo_hrefs = repository_hrefs - from_exporter.repository_hrefs
+            added_repo_hrefs.each do |added_repo_href|
+              start_versions << zero_version_href(added_repo_href)
+            end
+
+            # cvv_from - current_cvv , i.e. repos in from cvv that are not in current
+            # implying something got removed the current cvv
+            # make sure the start_versions doesn't contain those
+            deleted_repo_hrefs = from_exporter.repository_hrefs - repository_hrefs
+            start_versions.select! do |href|
+              !deleted_repo_hrefs.include?(version_href_to_repository_href(href))
+            end
+
+            options[:start_versions] = start_versions
+            options[:full] = 'false'
+          end
           [api.export_api.create(exporter_href, options)]
         end
 
@@ -42,15 +66,49 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
+        def validate_incremental_export!
+          return if @from_content_view_version.blank?
+          from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
+
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories)
+          to_exporter_repos = generate_repo_mapping(repositories)
+
+          invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
+            from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
+          end
+
+          if invalid_repos_exist
+            fail _("The exported Content View Version '%{content_view} %{current}' cannot be incrementally updated from version '%{from}'."\
+                   " Please do a full export." % { content_view: @content_view_version.content_view.name,
+                                                   current: @content_view_version.version,
+                                                   from: @from_content_view_version.version})
+          end
+        end
+
+        def generate_repo_mapping(repositories)
+          # return a repo mapping with key being the  library_instance_id and value being the repostiory_href
+          # used by validate_incremental_export
+          repo_map = {}
+          repositories.each do |repo|
+            repo_map[repo.library_instance_id] = version_href_to_repository_href(repo.version_href)
+          end
+          repo_map
+        end
+
         def generate_metadata
           ret = { organization: @content_view_version.organization.name,
                   repository_mapping: {},
                   content_view: @content_view_version.content_view.name,
-                  content_view_version: {
-                    major: @content_view_version.major,
-                    minor: @content_view_version.minor
-                  }
+                  content_view_version: @content_view_version.slice(:major, :minor)
           }
+
+          unless @from_content_view_version.blank?
+            ret[:from_content_view_version] = {
+              major: @from_content_view_version.major,
+              minor: @from_content_view_version.minor
+            }
+          end
+
           repositories.each do |repo|
             next if repo.version_href.blank?
             pulp3_repo = fetch_repository_info(repo.version_href).name

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -3,7 +3,6 @@ module Katello
     module ContentViewVersion
       class Import
         include ImportExportCommon
-        BASEDIR = '/var/lib/pulp'.freeze
 
         def initialize(smart_proxy:, content_view_version: nil, path: nil, metadata: nil)
           @smart_proxy = smart_proxy
@@ -44,32 +43,8 @@ module Katello
           api.importer_api.delete(importer_href)
         end
 
-        class << self
-          def check_permissions!(path)
-            fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
-            fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
-            fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
-
-            Dir.glob("#{path}/*").each do |file|
-              fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
-            end
-          end
-
-          def pulp_user_accessible?(path)
-            pulp_info = fetch_pulp_user_info
-            return false if pulp_info.blank?
-
-            stat = File.stat(path)
-            stat.gid.to_s == pulp_info.gid ||
-              stat.uid.to_s == pulp_info.uid ||
-              stat.mode.to_s(8)[-1].to_i >= 4
-          end
-
-          def fetch_pulp_user_info
-            pulp_user = nil
-            Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
-            pulp_user
-          end
+        def self.check!(content_view:, metadata:, path:)
+          ImportValidator.new(content_view: content_view, metadata: metadata, path: path).check!
         end
       end
     end

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -38,6 +38,12 @@ module Katello
         def version_href_to_repository_href(version_href)
           version_href.split("/")[0..-3].join("/") + "/"
         end
+
+        def zero_version_href(repository_href)
+          #  /pulp/api/v3/repositories/rpm/rpm/e59c4334-81d2-4d6b-a1a1-b61fa55ed664/versions/0/
+          repository_href += "/" unless repository_href.ends_with?('/')
+          "#{repository_href}versions/0/"
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -0,0 +1,107 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ImportValidator
+        BASEDIR = '/var/lib/pulp'.freeze
+        attr_accessor :metadata, :path, :content_view
+        def initialize(content_view:, path:, metadata:)
+          self.content_view = content_view
+          self.path = path
+          self.metadata = metadata
+        end
+
+        def check!
+          check_permissions!
+          ensure_importing_cvv_does_not_exist!
+          ensure_from_cvv_exists!
+          ensure_repositories_metadata_and_content_view_match!
+        end
+
+        def ensure_importing_cvv_does_not_exist!
+          major = metadata[:content_view_version][:major]
+          minor = metadata[:content_view_version][:minor]
+
+          if ::Katello::ContentViewVersion.where(major: major, minor: minor, content_view: content_view).exists?
+            fail _("Content View Version specified in the metadata - '%{name}' already exists. "\
+                    "If you wish to replace the existing version, delete %{name} and try again. " % { name: "#{content_view.name} #{major}.#{minor}" })
+          end
+        end
+
+        def ensure_from_cvv_exists!
+          major = metadata[:content_view_version][:major]
+          minor = metadata[:content_view_version][:minor]
+
+          if metadata[:from_content_view_version].present?
+            from_major = metadata[:from_content_view_version][:major]
+            from_minor = metadata[:from_content_view_version][:minor]
+
+            unless ::Katello::ContentViewVersion.where(major: from_major, minor: from_minor, content_view: content_view).exists?
+              fail _("Prior Content View Version specified in the metadata - '%{name}' does not exist. "\
+                      "Please import the metadata for '%{name}' before importing '%{current}' " % { name: "#{content_view.name} #{from_major}.#{from_minor}",
+                                                                                                    current: "#{content_view.name} #{major}.#{minor}"})
+            end
+          end
+        end
+
+        def ensure_repositories_metadata_and_content_view_match!
+          product_repos_in_content_view = content_view.repositories.yum_type.map { |repo| [repo.product.name, repo.name, repo.redhat?] }
+          product_repos_in_metadata = metadata[:repository_mapping].values.map { |repo| [repo[:product], repo[:repository], repo[:redhat]] }
+
+          product_repos_in_content_view.sort!
+          product_repos_in_metadata.sort!
+          # product_repos_in_content_view & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+
+          if product_repos_in_content_view != product_repos_in_metadata
+            repos_in_content_view = generate_product_repo_i18n_string(product_repos_in_content_view)
+            repos_in_import = generate_product_repo_i18n_string(product_repos_in_metadata)
+
+            fail _("Repositories in the importing content view do not match the repositories provided in the import metadata.\n "\
+                    "Repositories in Content View '%{content_view}': %{repos_in_content_view}\n "\
+                    "Repositories in the Import Metadata: %{repos_in_import}" % { content_view: content_view.name,
+                                                                                  repos_in_content_view: repos_in_content_view.join(""),
+                                                                                  repos_in_import: repos_in_import.join("")}
+                  )
+          end
+        end
+
+        def check_permissions!
+          fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
+          fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
+          fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
+
+          Dir.glob("#{path}/*").each do |file|
+            fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
+          end
+          toc_path = "#{path}/#{metadata[:toc]}"
+          fail _("The TOC file specified in the metadata does not exist. %s " % toc_path) unless File.exist?(toc_path)
+        end
+
+        def pulp_user_accessible?(path)
+          pulp_info = fetch_pulp_user_info
+          return false if pulp_info.blank?
+
+          stat = File.stat(path)
+          stat.gid.to_s == pulp_info.gid ||
+            stat.uid.to_s == pulp_info.uid ||
+            stat.mode.to_s(8)[-1].to_i >= 4
+        end
+
+        def fetch_pulp_user_info
+          pulp_user = nil
+          Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
+          pulp_user
+        end
+
+        def generate_product_repo_i18n_string(product_repos)
+          # product_repos look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos.map do |product, repo, redhat|
+            repo_type = redhat ? _("Red Hat") : _("Custom")
+            _("\n* Product = '%{product}', Repository = '%{repository}', Repository Type = '%{repo_type}'" % { product: product,
+                                                                                                               repository: repo,
+                                                                                                               repo_type: repo_type})
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/migration.rb

@@ -165,10 +165,16 @@ module Katello
         process_distributions(pulp3_api, migrated_repo.pulp3_distribution_hrefs)
       end
 
+      def distribution_path_from_cache(href, pulp3_api)
+        @distribution_cache ||= {}
+        @distribution_cache[href] ||= pulp3_api.distributions_api.read(href).base_path
+        @distribution_cache[href]
+      end
+
       def process_distributions(pulp3_api, dist_hrefs_list)
         #distribution_path_hash(pulp3_api, dist_hrefs_list).each do |relative_path, href|
         dist_hrefs_list.each do |href|
-          relative_path = pulp3_api.distributions_api.read(href).base_path
+          relative_path = distribution_path_from_cache(href, pulp3_api)
 
           dist_ref = Katello::Pulp3::DistributionReference.find_or_initialize_by(:path => relative_path)
           if (distribution_repo = Katello::Repository.find_by(:relative_path => relative_path) || Katello::Repository.find_by(:container_repository_name => relative_path))

data/app/services/katello/pulp3/repository/yum.rb

@@ -159,8 +159,7 @@ module Katello
 
             data_dup.config[i][:content] = leftover_units.pop(copy_amount)
             unit_copy_counter += copy_amount
-            # Do copy call if limit is reached or if we're under the limit but on the last repo config.
-            if unit_copy_counter >= UNIT_LIMIT || (i == data_dup.config.size - 1 && leftover_units.empty?)
+            if unit_copy_counter != 0
               tasks << api.copy_api.copy_content(data_dup)
               unit_copy_counter = 0
             end

data/app/services/katello/pulp3/task.rb

@@ -79,7 +79,7 @@ module Katello
 
       def error
         if task_data[:state] == CANCELED
-          self.new(_("Task canceled"))
+          _("Task canceled")
         elsif task_data[:state] == FAILED
           if task_data[:error][:description].blank?
             _("Pulp task error")
@@ -90,8 +90,8 @@ module Katello
       end
 
       def cancel
-        data = PulpcoreClient::Task.new(state: 'canceled')
-        tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
+        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
+        tasks_api.tasks_cancel(task_data['pulp_href'], data)
         #the main task may have completed, so cancel spawned tasks too
         task_data['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
       end