katello 4.2.1 → 4.3.0.rc1

data/app/lib/actions/pulp3/repository/import_upload.rb

@@ -4,15 +4,19 @@ module Actions
   module Pulp3
     module Repository
       class ImportUpload < Pulp3::AbstractAsyncTask
-        def plan(content_unit, repository, smart_proxy)
-          plan_self(:content_unit => content_unit,
+        def plan(save_artifact_output, repository, smart_proxy)
+          plan_self(:save_artifact_output => save_artifact_output,
                     :repository_id => repository.id,
                     :smart_proxy_id => smart_proxy.id)
         end
 
         def invoke_external_task
-          content_unit = input[:content_unit]
-          content_unit_href = content_unit.is_a?(String) ? content_unit : content_unit.last[:created_resources].first
+          if input[:save_artifact_output][:pulp_tasks]&.any?
+            content_unit_href = input[:save_artifact_output][:pulp_tasks].last[:created_resources].first
+          else
+            content_unit_href = input[:save_artifact_output][:content_unit_href]
+          end
+
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
           output[:content_unit_href] = content_unit_href

data/app/lib/actions/pulp3/repository/multi_copy_content.rb

@@ -17,7 +17,7 @@ module Actions
           repo_id_map = {}
 
           input[:repo_id_map].each do |source_repo_ids, dest_repo_map|
-            repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map.deep_dup
+            repo_id_map[source_repo_ids] = dest_repo_map.deep_dup
           end
 
           output[:pulp_tasks] = ::Katello::Repository.find(repo_id_map.values.first[:dest_repo]).backend_service(smart_proxy).copy_content_from_mapping(repo_id_map, input)

data/app/lib/actions/pulp3/repository/multi_copy_units.rb

@@ -25,7 +25,7 @@ module Actions
           repo_map = {}
 
           input[:repo_map].each do |source_repo_ids, dest_repo_map|
-            repo_map[JSON.parse(source_repo_ids)] = dest_repo_map
+            repo_map[source_repo_ids] = dest_repo_map
           end
 
           if input[:unit_map][:errata].any?

data/app/lib/actions/pulp3/repository/presenters/abstract_sync_presenter.rb

@@ -15,6 +15,24 @@ module Actions
             fail NotImplementedError
           end
 
+          def index_action
+            plan = action.execution_plan
+            index_step = plan.run_steps.find { |s| s.action_class == Actions::Katello::Repository::IndexContent }
+            index_step&.action(plan)
+          end
+
+          def added_content_message
+            if (content_added = index_action&.output&.[](:new_content))
+              content_added = content_added.select { |_type, number| number > 0 }
+              if content_added&.any?
+                count_messages = content_added.map { |type, number| "#{type.to_s.humanize.pluralize}: #{number}" }
+                _("Added %s") % count_messages.join(', ')
+              else
+                _("No content added.")
+              end
+            end
+          end
+
           def sync_task
             tasks = action.external_task.select do |task|
               if task.key? 'name'

data/app/lib/actions/pulp3/repository/presenters/content_unit_presenter.rb

@@ -12,7 +12,12 @@ module Actions
           def humanized_details
             ret = []
             ret << _("Cancelled.") if cancelled?
-            ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
+            ret << added_content_message
+            if total_units == 0
+              ret << _("Waiting to start.")
+            else
+              ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
+            end
             ret << "--------------------------------"
             progress_reports = sync_task.try(:[], 'progress_reports') || []
             progress_reports = progress_reports.sort_by { |pr| pr.try(:[], 'message') }
@@ -24,7 +29,7 @@ module Actions
               end
             end
 
-            ret.join("\n")
+            ret.compact.join("\n")
           end
 
           def total_units

data/app/lib/actions/pulp3/repository/presenters/repair_presenter.rb

@@ -39,7 +39,12 @@ module Actions
           def humanized_details
             ret = []
             ret << _("Cancelled.") if cancelled?
-            ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
+            if total_units == 0
+              ret << _("Waiting to start.")
+            else
+              ret << _("Total steps: ") + "#{finished_units}/#{total_units}"
+            end
+
             ret << "--------------------------------"
             progress_reports = repair_task.try(:[], 'progress_reports') || []
             progress_reports = progress_reports.sort_by { |pr| pr.try(:[], 'message') }

data/app/lib/actions/pulp3/repository/save_artifact.rb

@@ -9,14 +9,24 @@ module Actions
         end
 
         def invoke_external_task
+          repository = ::Katello::Repository.find(input[:repository_id])
           artifact_href = input[:options][:artifact_href] || fetch_artifact_href
           fail _("Content not uploaded to pulp") unless artifact_href
           content_type = input[:unit_type_id]
           content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
-          output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
-                                                                            artifact: artifact_href,
-                                                                            repository_id: input[:repository_id],
-                                                                            content_type: content_type)]
+
+          existing_content = ::Katello::Pulp3::PulpContentUnit.find_duplicate_unit(repository, input['unit_type_id'], {filename: input[:options][:file_name]}, input[:options][:sha256])
+          existing_content_href = existing_content&.results&.first&.pulp_href
+
+          if existing_content_href
+            output[:content_unit_href] = existing_content_href
+            []
+          else
+            output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
+                                                                              artifact: artifact_href,
+                                                                              repository_id: input[:repository_id],
+                                                                              content_type: content_type)]
+          end
         end
 
         def fetch_artifact_href

data/app/lib/katello/errors.rb

@@ -135,6 +135,8 @@ module Katello
       end
     end
 
+    class UpstreamConsumerNotFound < StandardError; end
+
     class UpstreamEntitlementGone < StandardError; end
 
     class ContainerRegistryNotConfigured < StandardError

data/app/lib/katello/resources/candlepin/consumer.rb

@@ -4,8 +4,6 @@ module Katello
       class Consumer < CandlepinResource
         extend ConsumerResource
 
-        GET_PARAM_BATCH_SIZE = 75
-
         class << self
           def all_uuids
             cp_consumers = Organization.all.map do |org|
@@ -27,14 +25,6 @@ module Katello
             end
           end
 
-          def get_all(uuids)
-            consumers = []
-            uuids.each_slice(GET_PARAM_BATCH_SIZE) do |slice|
-              consumers += get(:uuid => slice)
-            end
-            consumers
-          end
-
           # workaround for https://bugzilla.redhat.com/1647724
           def get_all_with_facts(uuids)
             consumers = []
@@ -139,16 +129,6 @@ module Katello
             self.delete(uri, self.default_headers).code.to_i
           end
 
-          def remove_entitlements(uuid)
-            uri = join_path(path(uuid), 'entitlements')
-            self.delete(uri, self.default_headers).code.to_i
-          end
-
-          def remove_certificate(uuid, serial_id)
-            uri = join_path(path(uuid), 'certificates') + "/#{serial_id}"
-            self.delete(uri, self.default_headers).code.to_i
-          end
-
           def virtual_guests(uuid)
             response = Candlepin::CandlepinResource.get(join_path(path(uuid), 'guests'), self.default_headers).body
             ::Katello::Util::Data.array_with_indifferent_access JSON.parse(response)

data/app/lib/katello/resources/candlepin/proxy.rb

@@ -20,10 +20,12 @@ module Katello
           client.delete({:accept => :json, :content_type => :json}.merge(User.cp_oauth_header))
         end
 
-        def self.get(path)
+        def self.get(path, extra_headers = {})
           logger.debug "Sending GET request to Candlepin: #{path}"
           client = CandlepinResource.rest_client(Net::HTTP::Get, :get, path_with_cp_prefix(path))
-          client.get({:accept => :json}.merge(User.cp_oauth_header))
+          client.get(extra_headers.merge!(default_request_headers))
+        rescue RestClient::NotModified => e
+          e.response
         end
 
         def self.put(path, body)
@@ -35,6 +37,10 @@ module Katello
         def self.path_with_cp_prefix(path)
           CandlepinResource.prefix + path
         end
+
+        def self.default_request_headers
+          @default_request_headers ||= User.cp_oauth_header.merge(accept: :json)
+        end
       end
     end
   end

data/app/lib/katello/resources/candlepin/upstream_consumer.rb

@@ -13,6 +13,8 @@ module Katello
             resource.head
           rescue RestClient::Unauthorized, RestClient::Gone
             raise ::Katello::Errors::UpstreamConsumerGone
+          rescue RestClient::NotFound
+            raise ::Katello::Errors::UpstreamConsumerNotFound
           end
 
           # Overrides the HttpResource get method to check if the upstream

data/app/lib/katello/resources/cdn/katello_cdn.rb

@@ -0,0 +1,68 @@
+module Katello
+  module Resources
+    module CDN
+      class KatelloCdn < CdnResource
+        def initialize(url, options)
+          @organization_label = options.delete(:organization_label)
+          fail ArgumentError, "No upstream organization was specified" if @organization_label.nil?
+
+          super
+        end
+
+        def fetch_paths(content_path)
+          url = "/katello/api/v2/repository_sets?organization_id=#{organization['id']}&search=#{CGI.escape("path = #{content_path}")}"
+          response = get(url)
+          repo_set = JSON.parse(response)['results'].first
+
+          fail _("Upstream organization %s does not provide this content path") % @organization_label if repo_set.nil?
+
+          # now get available repositories when we know the upstream repo set ID
+          url = "/katello/api/v2/repository_sets/#{repo_set['id']}/available_repositories?organization_id=#{organization['id']}"
+          response = get(url)
+          json_body = JSON.parse(response)
+          results = json_body['results']
+
+          results.map do |r|
+            {
+              path: r['path'],
+              substitutions: r['substitutions']
+            }
+          end
+        end
+
+        def valid_path?(_path, _postfix)
+          true
+        end
+
+        def debug_certificate
+          get("/katello/api/v2/organizations/#{organization['id']}/download_debug_certificate")
+        end
+
+        def repository_url(content_label:)
+          response = get("/katello/api/v2/organizations/#{organization['id']}/repositories?search=#{CGI.escape("content_label = #{content_label}")}")
+          repository = JSON.parse(response)['results'].first
+
+          if repository.nil?
+            fail _("Repository with content label %{content_label} was not found in upstream organization %{org_label}") % { content_label: content_label, org_label: @organization_label }
+          end
+
+          repository['full_path']
+        end
+
+        private
+
+        def organization
+          @organization ||= find_organization(@organization_label)
+        end
+
+        def find_organization(label)
+          response = get("/api/v2/organizations?search=#{CGI.escape("label = #{label}")}")
+          JSON.parse(response)['results'].first
+        rescue => e
+          Rails.logger.error("Couldn't load upstream organization with label=#{label} error=#{e.message}")
+          raise e
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/resources/cdn.rb

@@ -2,6 +2,7 @@ module Katello
   module Resources
     module CDN
       SUPPORTED_SSL_VERSIONS = ['SSLv23', 'TLSv1'].freeze
+
       class Utils
         # takes releasever from contentUrl (e.g. 6Server, 6.0, 6.1)
         # returns hash e.g. {:major => 6, :minor => "6.1"}
@@ -18,53 +19,75 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product, :options, :proxy
-
-        def substitutor(logger = nil)
-          @logger = logger
-          Util::CdnVarSubstitutor.new(self)
+        def substitutor
+          @substitutor ||= Util::CdnVarSubstitutor.new(self)
         end
 
         def initialize(url, options = {})
-          @proxy = ::HttpProxy.default_global_content_proxy
           @ssl_version = Setting[:cdn_ssl_version]
           if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version)
             fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting")
           end
+
           options.reverse_merge!(:verify_ssl => 9)
-          options.assert_valid_keys(:ssl_client_key, :ssl_client_cert, :ssl_ca_file, :verify_ssl,
-                                    :product)
-
-          ca = Katello::Repository.feed_ca_file(url)
-          if ca && URI(url).scheme == 'https'
-            options.reverse_merge!(:ssl_ca_file => ca)
-          elsif URI(url).scheme == 'https'
-            store = OpenSSL::X509::Store.new
-            store.set_default_paths
-            options.reverse_merge!(:cert_store => store)
+          options.assert_valid_keys(:ssl_client_key,
+                                    :ssl_client_cert,
+                                    :ssl_ca_file,
+                                    :ssl_ca_cert,
+                                    :verify_ssl,
+                                    :username,
+                                    :password,
+                                    :organization_label,
+                                    :ssl_ca_cert)
+
+          if options[:ssl_ca_cert].present?
+            ca_cert = OpenSSL::X509::Certificate.new(options[:ssl_ca_cert])
+            @cert_store = OpenSSL::X509::Store.new
+            @cert_store.add_cert(ca_cert)
           end
 
-          @product = options[:product]
-
           @url = url
           @uri = URI.parse(url)
           @options = options
         end
 
+        def self.create(product: nil, cdn_configuration:)
+          options = {}
+          if cdn_configuration.redhat?
+            options[:ssl_client_cert] = OpenSSL::X509::Certificate.new(product.certificate)
+            options[:ssl_client_key] = OpenSSL::PKey::RSA.new(product.key)
+            options[:ssl_ca_file] = self.ca_file
+            self.new(cdn_configuration.url, options)
+          else
+            options[:username] = cdn_configuration.username
+            options[:password] = cdn_configuration.password
+            options[:organization_label] = cdn_configuration.upstream_organization_label
+            options[:ssl_ca_cert] = cdn_configuration.ssl_ca
+            CDN::KatelloCdn.new(cdn_configuration.url, options)
+          end
+        end
+
+        def self.redhat_cdn_url
+          SETTINGS[:katello][:redhat_repository_url]
+        end
+
         def self.redhat_cdn?(url)
-          url.include?(SETTINGS[:katello][:redhat_repository_url])
+          url.include?(redhat_cdn_url)
+        end
+
+        def proxy
+          ::HttpProxy.default_global_content_proxy
         end
 
         def http_downloader
           net = net_http_class.new(@uri.host, @uri.port)
           net.use_ssl = @uri.is_a?(URI::HTTPS)
 
-          if CdnResource.redhat_cdn?(@uri.to_s)
+          if @uri.is_a?(URI::HTTPS)
             net.cert = @options[:ssl_client_cert]
             net.key = @options[:ssl_client_key]
-            net.ca_file = @options[:ssl_ca_file] if @options[:ssl_ca_file]
-          else
-            net.cert_store = @options[:cert_store]
+            net.ca_file = @options[:ssl_ca_file]
+            net.cert_store = @cert_store
           end
 
           # NOTE: This was added because some proxies dont support SSLv23 and do not handle TLS 1.2
@@ -93,8 +116,13 @@ module Katello
           net = http_downloader
           path = File.join(@uri.request_uri, path)
           used_url = File.join("#{@uri.scheme}://#{@uri.host}:#{@uri.port}", path)
-          Rails.logger.debug "CDN: Requesting path #{used_url}"
+          Rails.logger.info "CDN: Requesting path #{used_url}"
           req = Net::HTTP::Get.new(path)
+
+          if @options[:username] && @options[:password]
+            req.basic_auth(@options[:username], @options[:password])
+          end
+
           begin
             net.start do |http|
               res = http.request(req, nil) { |http_response| http_response.read_body }
@@ -125,6 +153,14 @@ module Katello
           end
         end
 
+        def valid_path?(path, postfix)
+          get(File.join(path, postfix)).present?
+        rescue RestClient::MovedPermanently
+          return true
+        rescue Errors::NotFound
+          return false
+        end
+
         def fetch_substitutions(base_path)
           get(File.join(base_path, "listing")).split("\n")
         rescue Errors::NotFound => e # some of listing file points to not existing content
@@ -136,12 +172,8 @@ module Katello
           "#{Katello::Engine.root}/ca/redhat-uep.pem"
         end
 
-        def self.ca_file_contents
-          File.read(ca_file)
-        end
-
         def net_http_class
-          if self.proxy
+          if proxy
             uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url
             Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password)
           else

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -48,7 +48,7 @@ module Katello
       end
 
       def any_valid_metadata_file?(repo_path)
-        ['repodata/repomd.xml', 'PULP_MANIFEST', '.treeinfo', 'treeinfo'].any? { |filename| valid_path?(repo_path, filename) }
+        ['repodata/repomd.xml', 'PULP_MANIFEST', '.treeinfo', 'treeinfo'].any? { |filename| @resource.valid_path?(repo_path, filename) }
       end
 
       protected
@@ -59,10 +59,10 @@ module Katello
 
         return resolved if to_resolve.empty?
 
-        to_resolve.in_groups_of(8) do |group|
+        to_resolve.in_groups_of(8, false) do |group|
           futures = group.map do |path_with_substitution|
             Concurrent::Promises.future do
-              path_with_substitution.resolve_substitutions(@resource)
+              resolve_path(path_with_substitution)
             end
           end
 
@@ -75,12 +75,18 @@ module Katello
         find_substitutions(resolved.compact.flatten)
       end
 
-      def valid_path?(path, postfix)
-        @resource.get(File.join(path, postfix)).present?
-      rescue RestClient::MovedPermanently
-        return true
-      rescue Errors::NotFound
-        return false
+      def resolve_path(path_with_substitutions)
+        if @resource.respond_to?(:fetch_paths)
+          @resource.fetch_paths(path_with_substitutions.path).compact.map do |element|
+            PathWithSubstitutions.new(element[:path], element[:substitutions])
+          end
+        elsif @resource.respond_to?(:fetch_substitutions)
+          @resource.fetch_substitutions(path_with_substitutions.base_path).compact.map do |value|
+            path_with_substitutions.resolve_token(value)
+          end
+        else
+          fail _("Unsupported CDN resource")
+        end
       end
     end
   end

data/app/lib/katello/util/package.rb

@@ -170,6 +170,28 @@ module Katello
         end
         pieces.join(".")
       end
+
+      def self.parse_dependencies(dependencies)
+        # dependencies is either 'requires' or 'provides' metadata attribute of rpm package in pulp
+        results = []
+        flags = {'GT' => '>', 'LT' => '<', 'EQ' => '=', 'GE' => '>=', 'LE' => '<='}
+
+        dependencies&.each do |dependency|
+          dependencies_str = ""
+          if dependency.count < 3
+            dependencies_str = dependency.first
+            results << dependencies_str
+          else
+            dependency[1] = flags[dependency[1]]
+            dependency[0...2].each { |dependency_piece| dependencies_str += "#{dependency_piece} " }
+            dependencies_str += "#{dependency[2]}:" unless dependency[2] == "0" # epoch
+            dependencies_str += "#{dependency[3]}-" # version
+            dependency[4...-1].each { |dependency_piece| dependencies_str += "#{dependency_piece}." }
+            results << dependencies_str[0...-1]
+          end
+        end
+        results.uniq
+      end
     end
   end
 end

data/app/lib/katello/util/path_with_substitutions.rb

@@ -3,15 +3,19 @@ module Katello
     class PathWithSubstitutions
       include Comparable
 
-      attr_accessor :substitutions
-      attr_accessor :path
+      attr_reader :base_path, :path, :substitutions
+
+      SUBSTITUTABLE_REGEX = /^(.*?)\$([^\/]*)/.freeze
 
       #path /content/rhel/server/$arch/$releasever/os
       #substitutions  {$arch => 'x86_64'}
       def initialize(path, substitutions)
         @substitutions = substitutions
         @path = path
-        @resolved = []
+
+        if @path =~ SUBSTITUTABLE_REGEX
+          @base_path, @token = Regexp.last_match[1], Regexp.last_match[2]
+        end
       end
 
       def split_path
@@ -25,19 +29,13 @@ module Katello
       end
 
       def substitutable?
-        path =~ /^(.*?)\$([^\/]*)/
+        @token.present?
       end
 
-      def resolve_substitutions(cdn_resource)
-        if @resolved.empty? && path =~ /^(.*?)\$([^\/]*)/
-          base_path, var = Regexp.last_match[1], Regexp.last_match[2]
-          cdn_resource.fetch_substitutions(base_path).compact.map do |value|
-            new_substitutions = substitutions.merge(var => value)
-            new_path = path.sub("$#{var}", value)
-            @resolved << PathWithSubstitutions.new(new_path, new_substitutions)
-          end
-        end
-        @resolved
+      def resolve_token(value)
+        new_substitutions = substitutions.merge(@token => value)
+        new_path = path.sub("$#{@token}", value)
+        PathWithSubstitutions.new(new_path, new_substitutions)
       end
 
       def unused_substitutions

data/app/models/katello/activation_key.rb

@@ -184,5 +184,15 @@ module Katello
         self.content_view = self.environment.try(:default_content_view)
       end
     end
+
+    apipie :class, desc: "A class representing #{model_name.human} object" do
+      name 'Activation Key'
+      refs 'ActivationKey'
+      sections only: %w[all additional]
+      property :name, String, desc: 'Returns the name of the Activation Key.'
+    end
+    class Jail < ::Safemode::Jail
+      allow :name
+    end
   end
 end

data/app/models/katello/ansible_collection.rb

@@ -3,7 +3,7 @@ module Katello
     include Concerns::PulpDatabaseUnit
 
     self.table_name = 'katello_ansible_collections'
-    CONTENT_TYPE = 'ansible collection'.freeze
+    CONTENT_TYPE = 'ansible_collection'.freeze
 
     has_many :ansible_collection_tags, :class_name => "Katello::AnsibleCollectionTag", :dependent => :delete_all
     has_many :tags, :through => :ansible_collection_tags

data/app/models/katello/authorization/repository.rb

@@ -4,12 +4,12 @@ module Katello
 
     delegate :editable?, to: :product
 
-    def deletable?
-      product.editable? && !promoted?
+    def deletable?(remove_from_content_view_versions = false)
+      product.editable? && (remove_from_content_view_versions || !promoted?)
     end
 
-    def redhat_deletable?
-      !self.promoted? && self.product.editable?
+    def redhat_deletable?(remove_from_content_view_versions = false)
+      (remove_from_content_view_versions || !self.promoted?) && self.product.editable?
     end
 
     def readable?

data/app/models/katello/candlepin/repository_mapper.rb

@@ -74,7 +74,11 @@ module Katello
       end
 
       def feed_url
-        product.repo_url(path)
+        @feed_url ||= if product.cdn_resource&.respond_to?(:repository_url)
+                        product.cdn_resource.repository_url(content_label: content.label)
+                      else
+                        product.repo_url(path)
+                      end
       end
 
       def arch

data/app/models/katello/cdn_configuration.rb

@@ -0,0 +1,32 @@
+module Katello
+  class CdnConfiguration < Katello::Model
+    include Encryptable
+
+    belongs_to :organization, :inverse_of => :cdn_configuration
+
+    belongs_to :ssl_ca_credential, :class_name => "Katello::ContentCredential", :inverse_of => :ssl_ca_cdn_configurations
+
+    encrypts :password
+
+    validates :url, presence: true
+    validates_with Validators::KatelloUrlFormatValidator, attributes: :url
+    validates_with Validators::KatelloLabelFormatValidator, attributes: :upstream_organization_label, if: proc { upstream_organization_label.present? }
+    validate :non_redhat_configuration, unless: :redhat?
+
+    def ssl_ca
+      ssl_ca_credential&.content
+    end
+
+    def redhat?
+      username.blank? && password.blank? && upstream_organization_label.blank? && ssl_ca_credential_id.blank?
+    end
+
+    private
+
+    def non_redhat_configuration
+      if username.blank? || password.blank? || upstream_organization_label.blank? || ssl_ca_credential_id.blank?
+        errors.add(:base, _("Username, Password, Upstream Organization Label, and SSL CA Credential are required when using a non-Red Hat CDN."))
+      end
+    end
+  end
+end