katello 3.3.2 → 3.4.0.rc1

data/app/lib/katello/resources/cdn.rb

@@ -17,7 +17,7 @@ module Katello
       class CdnResource
         CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze
 
-        attr_reader :url, :product
+        attr_reader :url, :product, :options
         attr_accessor :proxy_host, :proxy_port, :proxy_user, :proxy_password
 
         def substitutor(logger = nil)
@@ -37,25 +37,29 @@ module Katello
 
           @url = url
           @uri = URI.parse(url)
-          @net = net_http_class.new(@uri.host, @uri.port)
-          @net.use_ssl = @uri.is_a?(URI::HTTPS)
+          @options = options
+        end
+
+        def http_downloader
+          net = net_http_class.new(@uri.host, @uri.port)
+          net.use_ssl = @uri.is_a?(URI::HTTPS)
 
-          @net.cert = options[:ssl_client_cert]
-          @net.key = options[:ssl_client_key]
-          @net.ca_file = options[:ssl_ca_file]
+          net.cert = @options[:ssl_client_cert]
+          net.key = @options[:ssl_client_key]
+          net.ca_file = @options[:ssl_ca_file]
 
           # NOTE: This was added because some proxies dont support SSLv23 and do not handle TLS 1.2
           # Valid values in ruby 1.9.3 are 'SSLv23' or 'TLSV1'
           # Run the following command in rails console to figure out other
           # valid constants in other ruby versions
           # "OpenSSL::SSL::SSLContext::METHODS"
-          @net.ssl_version = SETTINGS[:katello][:cdn_ssl_version] if SETTINGS[:katello].key?(:cdn_ssl_version)
+          net.ssl_version = SETTINGS[:katello][:cdn_ssl_version] if SETTINGS[:katello].key?(:cdn_ssl_version)
 
-          if (options[:verify_ssl] == false) || (options[:verify_ssl] == OpenSSL::SSL::VERIFY_NONE)
-            @net.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          elsif options[:verify_ssl].is_a? Integer
-            @net.verify_mode = options[:verify_ssl]
-            @net.verify_callback = lambda do |preverify_ok, ssl_context|
+          if (@options[:verify_ssl] == false) || (@options[:verify_ssl] == OpenSSL::SSL::VERIFY_NONE)
+            net.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          elsif @options[:verify_ssl].is_a? Integer
+            net.verify_mode = @options[:verify_ssl]
+            net.verify_callback = lambda do |preverify_ok, ssl_context|
               if !preverify_ok || ssl_context.error != 0
                 err_msg = "SSL Verification failed -- Preverify: #{preverify_ok}, Error: #{ssl_context.error_string} (#{ssl_context.error})"
                 fail RestClient::SSLCertificateNotVerified, err_msg
@@ -63,15 +67,17 @@ module Katello
               true
             end
           end
+          net
         end
 
         def get(path, _headers = {})
+          net = http_downloader
           path = File.join(@uri.request_uri, path)
           used_url = File.join("#{@uri.scheme}://#{@uri.host}:#{@uri.port}", path)
           Rails.logger.debug "CDN: Requesting path #{used_url}"
           req = Net::HTTP::Get.new(path)
           begin
-            @net.start do |http|
+            net.start do |http|
               res = http.request(req, nil) { |http_response| http_response.read_body }
               code = res.code.to_i
               if code == 200
@@ -100,6 +106,13 @@ module Katello
           end
         end
 
+        def fetch_substitutions(base_path)
+          get(File.join(base_path, "listing")).split("\n")
+        rescue Errors::NotFound => e # some of listing file points to not existing content
+          log :error, e.message
+          [] # return no substitution for unreachable listings
+        end
+
         def self.ca_file
           "#{Katello::Engine.root}/ca/redhat-uep.pem"
         end

data/app/lib/katello/util/cdn_var_substitutor.rb

@@ -8,104 +8,40 @@ module Katello
       # /content/rhel/6.2/listing) and returns the body response)
       def initialize(cdn_resource)
         @resource = cdn_resource
-        @substitutions = Thread.current[:cdn_var_substitutor_cache] || {}
-        @good_listings = Set.new
-        @bad_listings = Set.new
-      end
-
-      # using substitutor from whithin the block makes sure that every
-      # request is made only once.
-      def self.with_cache(&_block)
-        Thread.current[:cdn_var_substitutor_cache] = {}
-        yield
-      ensure
-        Thread.current[:cdn_var_substitutor_cache] = nil
-      end
-
-      # precalcuclate all paths at once - let's you discover errors and stop
-      # before it causes more pain
-      def precalculate(paths_with_vars)
-        paths_with_vars.uniq.reduce({}) do |ret, path_with_vars|
-          ret[path_with_vars] = substitute_vars(path_with_vars)
-          ret
-        end
       end
 
       # takes path e.g. "/rhel/server/5/$releasever/$basearch/os"
-      # returns hash substituting variables:
+      # returns PathWithSubstitutions objects
       #
       #  { {"releasever" => "6Server", "basearch" => "i386"} =>  "/rhel/server/5/6Server/i386/os",
       #    {"releasever" => "6Server", "basearch" => "x86_64"} =>  "/rhel/server/5/6Server/x84_64/os"}
       #
       # values are loaded from CDN
-      def substitute_vars(path_with_vars)
-        if path_with_vars =~ /^(.*\$\w+)(.*)$/
-          prefix_with_vars, suffix_without_vars = Regexp.last_match[1], Regexp.last_match[2]
-        else
-          prefix_with_vars, suffix_without_vars = "", path_with_vars
-        end
-
-        prefixes_without_vars = substitute_vars_in_prefix(prefix_with_vars)
-        paths_without_vars = prefixes_without_vars.reduce({}) do |h, (substitutions, prefix_without_vars)|
-          h[substitutions] = prefix_without_vars + suffix_without_vars
-          h
-        end
-        return paths_without_vars
+      def substitute_vars(path)
+        find_substitutions([PathWithSubstitutions.new(path, {})])
       end
 
-      # prefix_with_vars is the part of url containing some vars. We can cache
-      # calcualted values for this parts. So for example for:
-      #   "/a/$b/$c/d"
-      #   "/a/$b/$c/e"
-      # prefix_with_vars is "/a/$b/$c" and we store the result after resolving
-      # for the first path.
-      def substitute_vars_in_prefix(prefix_with_vars)
-        paths_with_vars = { {} => prefix_with_vars}
-        prefixes_without_vars = @substitutions[prefix_with_vars]
-
-        unless prefixes_without_vars
-          prefixes_without_vars = {}
-          until paths_with_vars.empty?
-            substitutions, path = paths_with_vars.shift
+      def validate_substitutions(content, substitutions)
+        path_with_subs = PathWithSubstitutions.new(content.contentUrl, substitutions)
+        real_path = path_with_subs.apply_substitutions
+        unused_substitutions = path_with_subs.unused_substitutions
+        needed_substitutions = PathWithSubstitutions.new(real_path, {}).substitutions_needed
 
-            if substituable? path
-              for_each_substitute_of_next_var substitutions, path do |new_substitution, new_path|
-                begin
-                  paths_with_vars[new_substitution] = new_path
-                rescue Errors::SecurityViolation
-                  # Some paths may not be accessible
-                  @resource.log :warn, "#{new_path} is not accessible, ignoring"
-                end
-              end
-            else
-              prefixes_without_vars[substitutions] = path
-            end
-          end
-          @substitutions[prefix_with_vars] = prefixes_without_vars
+        if unused_substitutions.any?
+          fail Errors::CdnSubstitutionError, _("%{unused_substitutes} cannot be specified for %{content_name}"\
+                 " as that information is not substitutable in %{content_url} ") %
+              { unaccepted_substitutions: unused_substitutions, content_name: content.name, content_url: content.contentUrl }
         end
-        return prefixes_without_vars
-      end
-
-      def substituable?(path)
-        path.include?("$")
-      end
 
-      def valid_substitutions(content, substitutions)
-        validate_all_substitutions_accepted(content, substitutions)
-        content_url = content.contentUrl
-        real_path = gsub_vars(content_url, substitutions)
-
-        if substituable?(real_path)
+        if needed_substitutions.any?
           fail Errors::CdnSubstitutionError, _("Missing arguments %{substitutions} for %{content_url}") %
-              { substitutions: substitutions_needed(real_path).join(', '),
-                content_url: real_path }
-        else
-          unless any_valid_metadata_file?(real_path)
-            @resource.log :error, "No valid metadata files found for #{real_path}"
-            fail Errors::CdnSubstitutionError, _("The path %{real_path} does not seem to be a valid repository."\
-                   " If you think this is an error, please try refreshing your manifest.") %
-              {real_path: real_path}
-          end
+              { substitutions: needed_substitutions.join(','), content_url: real_path }
+        end
+
+        unless any_valid_metadata_file?(real_path)
+          @resource.log :error, "No valid metadata files found for #{real_path}"
+          fail Errors::CdnSubstitutionError, _("The path %{real_path} does not seem to be a valid repository."\
+                 " If you think this is an error, please try refreshing your manifest.") % {real_path: real_path}
         end
       end
 
@@ -115,21 +51,28 @@ module Katello
 
       protected
 
-      def substitutions_needed(content_url)
-        # e.g. if content_url = "/content/dist/rhel/server/7/$releasever/$basearch/kickstart"
-        #      return ['releasever', 'basearch']
-        content_url.split('/').map { |word| word.start_with?('$') ? word[1..-1] : nil }.compact
-      end
+      def find_substitutions(paths_with_substitutions)
+        to_resolve = paths_with_substitutions.select { |path| path.substitutable? }
+        resolved = paths_with_substitutions - to_resolve
+
+        return resolved if to_resolve.empty?
 
-      def validate_all_substitutions_accepted(content, substitutions)
-        unaccepted_substitutions = substitutions.keys.reject do |key|
-          content.contentUrl.include?("$#{key}")
+        futures = to_resolve.map do |path_with_substitution|
+          Concurrent.future do
+            path_with_substitution.resolve_substitutions(@resource)
+          end
         end
-        if unaccepted_substitutions.size > 0
-          fail Errors::CdnSubstitutionError, _("%{unaccepted_substitutions} cannot be specified for %{content_name}"\
-                 " as that information is not substituable in %{content_url} ") %
-              { unaccepted_substitutions: unaccepted_substitutions, content_name: content.name, content_url: content.contentUrl }
+
+        futures.each do |future|
+          begin
+            resolved << future.value
+          rescue StandardError => e
+            Rails.logger.error("Error Recieved: #{e.to_s}")
+            Rails.logger.error("Error Recieved: #{e.backtrace.join("\n")}")
+          end
         end
+
+        find_substitutions(resolved.compact.flatten)
       end
 
       def valid_path?(path, postfix)
@@ -139,34 +82,6 @@ module Katello
       rescue Errors::NotFound
         return false
       end
-
-      def gsub_vars(content_url, substitutions)
-        substitutions.reduce(content_url) do |url, (key, value)|
-          url.gsub("$#{key}", value)
-        end
-      end
-
-      def for_each_substitute_of_next_var(substitutions, path)
-        if path =~ /^(.*?)\$([^\/]*)/
-          base_path, var = Regexp.last_match[1], Regexp.last_match[2]
-          get_substitutions_from(base_path).compact.each do |value|
-            new_substitutions = substitutions.merge(var => value)
-            new_path = path.sub("$#{var}", value)
-
-            yield new_substitutions, new_path
-          end
-        end
-      end
-
-      def get_substitutions_from(base_path)
-        ret = @resource.get(File.join(base_path, "listing")).split("\n")
-        @good_listings << base_path
-        ret
-      rescue Errors::NotFound => e # some of listing file points to not existing content
-        @bad_listings << base_path
-        @resource.log :error, e.message
-        [] # return no substitution for unreachable listings
-      end
     end
   end
 end

data/app/lib/katello/util/path_with_substitutions.rb

@@ -0,0 +1,58 @@
+module Katello
+  module Util
+    class PathWithSubstitutions
+      include Comparable
+
+      attr_accessor :substitutions
+      attr_accessor :path
+
+      #path /content/rhel/server/$arch/$releasever/os
+      #substitutions  {$arch => 'x86_64'}
+      def initialize(path, substitutions)
+        @substitutions = substitutions
+        @path = path
+        @resolved = []
+      end
+
+      def substitutions_needed
+        # e.g. if content_url = "/content/dist/rhel/server/7/$releasever/$basearch/kickstart"
+        #      return ['releasever', 'basearch']
+        path.split('/').map { |word| word.start_with?('$') ? word[1..-1] : nil }.compact
+      end
+
+      def substitutable?
+        path =~ /^(.*?)\$([^\/]*)/
+      end
+
+      def resolve_substitutions(cdn_resource)
+        if @resolved.empty? && path =~ /^(.*?)\$([^\/]*)/
+          base_path, var = Regexp.last_match[1], Regexp.last_match[2]
+          cdn_resource.fetch_substitutions(base_path).compact.map do |value|
+            new_substitutions = substitutions.merge(var => value)
+            new_path = path.sub("$#{var}", value)
+            @resolved << PathWithSubstitutions.new(new_path, new_substitutions)
+          end
+        end
+        @resolved
+      end
+
+      def unused_substitutions
+        substitutions.keys.reject do |key|
+          path.include?("$#{key}")
+        end
+      end
+
+      def apply_substitutions
+        substitutions.reduce(path) do |url, (key, value)|
+          url.gsub("$#{key}", value)
+        end
+      end
+
+      def <=>(other)
+        key1 = path + substitutions.to_s
+        key2 = other.path + other.substitutions.to_s
+        key1 <=> key2
+      end
+    end
+  end
+end

data/app/models/katello/activation_key.rb

@@ -49,10 +49,10 @@ module Katello
     scope :in_environment, ->(env) { where(:environment_id => env) }
 
     scoped_search :on => :name, :complete_value => true
-    scoped_search :on => :organization_id, :complete_value => true, :only_explicit => true
-    scoped_search :rename => :environment, :on => :name, :in => :environment, :complete_value => true
-    scoped_search :rename => :content_view, :on => :name, :in => :content_view, :complete_value => true
-    scoped_search :on => :content_view_id, :complete_value => true, :only_explicit => true
+    scoped_search :on => :organization_id, :complete_value => true, :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER
+    scoped_search :rename => :environment, :on => :name, :relation => :environment, :complete_value => true
+    scoped_search :rename => :content_view, :on => :name, :relation => :content_view, :complete_value => true
+    scoped_search :on => :content_view_id, :complete_value => true, :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER
     scoped_search :on => :description, :complete_value => true
 
     def environment_exists
@@ -85,7 +85,7 @@ module Katello
 
     def available_subscriptions
       all_pools = self.get_pools.map { |pool| pool["id"] }
-      added_pools = self.get_key_pools.map { |pool| pool["id"] }
+      added_pools = self.pools.pluck(:cp_id)
       available_pools = all_pools - added_pools
       Pool.where(:cp_id => available_pools,
                  :subscription_id => Subscription.with_subscribable_content)
@@ -94,15 +94,11 @@ module Katello
     def products
       all_products = []
 
-      cp_pools = self.get_key_pools
-      if cp_pools
-        pools = cp_pools.collect { |cp_pool| Pool.find_by(:cp_id => cp_pool['id']) }
-        pools.each do |pool|
-          if pool.subscription
-            all_products << pool.subscription.products
-          else
-            Rails.logger.error("Pool #{pool.id} is missing its subscription id.")
-          end
+      self.pools.each do |pool|
+        if pool.subscription
+          all_products << pool.subscription.products
+        else
+          Rails.logger.error("Pool #{pool.id} is missing its subscription id.")
         end
       end
       all_products.flatten!
@@ -112,7 +108,7 @@ module Katello
       self.products ? self.products.map(&:available_content).flatten.uniq { |product| product.content.id } : []
     end
 
-    def valid_content_label?(content_label)
+    def valid_content_override_label?(content_label)
       self.available_content.map(&:content).any? { |content| content.label == content_label }
     end
 

data/app/models/katello/candlepin/content.rb

@@ -6,8 +6,8 @@ module Katello
       load_attributes(params)
     end
 
-    def self.find(id)
-      found = Resources::Candlepin::Content.get(id)
+    def self.find(owner_key, id)
+      found = Resources::Candlepin::Content.get(owner_key, id)
       Candlepin::Content.new(found)
     end
 
@@ -46,8 +46,10 @@ module Katello
 
       def find_repository
         ::Katello::Repository.where(product_id: product.id,
+                                    content_id: content.id,
                                     environment_id: product.organization.library.id,
-                                    pulp_id: pulp_id).first
+                                    minor: minor,
+                                    arch: arch).first
       end
 
       def build_repository
@@ -56,7 +58,6 @@ module Katello
         repository = Repository.new(
           :environment => product.organization.library,
           :product => product,
-          :pulp_id => pulp_id,
           :cp_label => content.label,
           :content_id => content.id,
           :arch => arch,
@@ -93,7 +94,7 @@ module Katello
 
       def validate!
         return if katello_content_type == Repository::OSTREE_TYPE
-        substitutor.valid_substitutions(content, substitutions)
+        substitutor.validate_substitutions(content, substitutions)
       end
 
       def substitutor
@@ -107,10 +108,6 @@ module Katello
         repo_name_parts.join(" ").gsub(/[^a-z0-9\-\._ ]/i, "")
       end
 
-      def pulp_id
-        product.repo_id(name)
-      end
-
       def path
         substitutions.inject(content.contentUrl) do |url, (key, value)|
           url.gsub("$#{key}", value)
@@ -212,7 +209,6 @@ module Katello
         end
         ::Katello::Repository.new(:environment => product.organization.library,
                                  :product => product,
-                                 :pulp_id => pulp_id,
                                  :cp_label => content.label,
                                  :content_id => content.id,
                                  :relative_path => relative_path,
@@ -237,10 +233,6 @@ module Katello
         "#{content.name} - (#{registry['name']})"
       end
 
-      def pulp_id
-        product.repo_id(content.name, nil, registry['name'])
-      end
-
       def feed_url
         cdn_uri = URI.parse(product.provider.repository_url)
         docker_repo_uri =  URI.parse(registry["url"])