katello 4.13.0.rc1 → 4.13.1

data/app/lib/katello/repo_discovery.rb

@@ -4,22 +4,11 @@ module Katello
   class RepoDiscovery
     include Katello::Util::HttpProxy
 
-    attr_reader :found, :crawled, :to_follow
-
-    # rubocop:disable Metrics/ParameterLists
-    def initialize(url, content_type = 'yum', upstream_username = nil,
-      upstream_password = nil, search = '*', crawled = [],
-      found = [], to_follow = [])
-      @uri = uri(url)
-      @content_type = content_type
-      @upstream_username = upstream_username.empty? ? nil : upstream_username
-      @upstream_password = upstream_password.empty? ? nil : upstream_password
-      @search = search
-      @found = found
-      @crawled = crawled
-      @to_follow = to_follow
+    def self.class_for(content_type)
+      repo_discovery_class = RepositoryTypeManager.find_repository_type(content_type)&.repo_discovery_class
+      fail _("Content type does not support repo discovery") unless repo_discovery_class
+      repo_discovery_class
     end
-    # rubocop:enable Metrics/ParameterLists
 
     def uri(url)
       #add a / on the end, as directories require it or else
@@ -27,180 +16,5 @@ module Katello
       url += '/' unless url.ends_with?('/')
       URI(url)
     end
-
-    def run(resume_point)
-      if @content_type == 'docker'
-        docker_search
-      else
-        if @uri.scheme == 'file'
-          file_crawl(uri(resume_point))
-        elsif %w(http https).include?(@uri.scheme)
-          http_crawl(uri(resume_point))
-        else
-          fail _("Unsupported URL protocol %s.") % @uri.scheme
-        end
-      end
-    end
-
-    private
-
-    def parse_parameter(field_value)
-      field_value.lstrip!
-      i = field_value.index(/[ \t=;,]/) || field_value.length
-      name = field_value.slice!(0, i).downcase(:ascii)
-      field_value.lstrip!
-      if field_value.delete_prefix!('=')
-        field_value.lstrip!
-        if field_value.delete_prefix!('"')
-          value = ''
-          until field_value.empty?
-            break if field_value.delete_prefix!('"')
-            field_value.delete_prefix!("\\")
-            value += field_value.slice!(0, 1) || ''
-          end
-        else
-          i = field_value.index(/[;,]/) || field_value.length
-          value = field_value.slice!(0, i)
-        end
-      end
-      {name: name, value: value || ''}
-    end
-
-    def parse_parameters(field_value)
-      seen_rel = false
-      has_next_rel = false
-      has_anchor = false
-      until field_value.empty?
-        field_value.lstrip!
-        break if field_value.delete_prefix!(';').nil?
-        param = parse_parameter(field_value)
-        case
-        when param[:name] == 'rel' && !seen_rel
-          seen_rel = true
-          has_next_rel = param[:value].downcase(:ascii).split(/[ \t]/).include?('next')
-        when param[:name] == 'anchor'
-          has_anchor = true
-        end
-      end
-      {has_next_rel: has_next_rel, has_anchor: has_anchor}
-    end
-
-    def get_next_link(link_header)
-      # This code mostly follows Appendix B "Algorithms of Parsing Link Header Fields" of RFC 8288
-      # "Web Linking", <https://www.rfc-editor.org/rfc/rfc8288#appendix-B> (that RFC appears to be
-      # silent about multiple "next" links, so just use the first one and ignore any additional
-      # ones, in the general spirit of being lenient):
-      return nil if link_header.nil?
-      field_value = link_header.clone
-      until field_value.empty?
-        # The following ignores any junk preceding the next <...> link URL:
-        m = field_value.match(/<(.*)>/)
-        break unless m
-        target_string = m[1]
-        field_value = m.post_match
-        params = parse_parameters(field_value)
-        if params[:has_next_rel]
-          # To keep it simple, ignore a link with an (unlikely) anchor parameter; but the RFC
-          # mandates that we "MUST NOT process the link without applying the anchor", so just raise
-          # an exception in that (unlikely) case:
-          fail "anchor not supported" if params[:has_anchor]
-          return target_string
-        end
-      end
-      nil
-    end
-
-    def docker_search
-      request_params = {
-        method: :get,
-        headers: { accept: :json },
-        url: "#{@uri}v1/search?q=#{@search}"
-      }
-
-      request_params[:user] = @upstream_username unless @upstream_username.empty?
-      request_params[:password] = @upstream_password unless @upstream_password.empty?
-      request_params[:proxy] = proxy_uri if proxy
-
-      begin
-        results = RestClient::Request.execute(request_params)
-        JSON.parse(results)['results'].each do |result|
-          @found << result['name']
-        end
-      rescue
-        # Note: v2 endpoint does not support search
-        request_params[:url] = "#{@uri}v2/_catalog"
-        loop do
-          results = RestClient::Request.execute(request_params)
-          JSON.parse(results)['repositories'].each do |result|
-            @found << result
-          end
-          next_uri = get_next_link(results.headers[:link])
-          break if next_uri.nil?
-          request_params[:url] = URI(request_params[:url]).merge(next_uri).to_s
-        end
-      end
-      @found.sort!
-    end
-
-    def anemone_proxy_details
-      details = {}
-
-      if proxy
-        details[:proxy_host] = proxy_host
-        details[:proxy_port] = proxy_port
-        details[:proxy_user] = proxy.username
-        details[:proxy_password] = proxy.password
-      end
-
-      details
-    end
-
-    def http_crawl(resume_point)
-      resume_point_uri = URI(resume_point)
-      resume_point_uri.user = @upstream_username if @upstream_username
-      resume_point_uri.password = @upstream_password if @upstream_password
-
-      Anemone.crawl(resume_point_uri, anemone_proxy_details) do |anemone|
-        anemone.focus_crawl do |page|
-          @crawled << page.url.path
-
-          page.links.each do |link|
-            if link.path.ends_with?('/repodata/')
-              page_url = page.url.clone
-              page_url.user = nil
-              page_url.password = nil
-              @found << page_url.to_s
-            else
-              @to_follow << link.to_s if should_follow?(link.path)
-            end
-          end
-          page.discard_doc! #saves memory, doc not needed
-          []
-        end
-      end
-    end
-
-    def file_crawl(resume_point)
-      if resume_point.path.ends_with?('/repodata/')
-        found_path = Pathname(resume_point.path).parent.to_s
-        @found << "file://#{found_path}"
-      end
-      if resume_point.path == @uri.path
-        Dir.glob("#{@uri.path}**/").each { |path| @to_follow << path }
-        @to_follow.shift
-      end
-      @crawled << resume_point.path
-    end
-
-    def should_follow?(path)
-      #Verify:
-      # * link's path starts with the base url
-      # * link hasn't already been crawled
-      # * link ends with '/' so it should be a directory
-      # * link doesn't end with '/Packages/', as this increases
-      #       processing time and memory usage considerably
-      return path.starts_with?(@uri.path) && !@crawled.include?(path) &&
-           path.ends_with?('/') && !path.ends_with?('/Packages/')
-    end
   end
 end

data/app/lib/katello/resources/discovery/container.rb

@@ -0,0 +1,127 @@
+module Katello
+  module Resources
+    module Discovery
+      class Container < RepoDiscovery
+        attr_reader :found, :crawled, :to_follow
+        def initialize(url, crawled = [], found = [], to_follow = [],
+                       upstream_credentials_and_search = {
+                         upstream_username: nil,
+                         upstream_password: nil,
+                         search: '*'
+                       })
+          @uri = uri(url)
+          @upstream_username = upstream_credentials_and_search[:upstream_username].presence
+          @upstream_password = upstream_credentials_and_search[:upstream_password].presence
+          @search = upstream_credentials_and_search.fetch(:search, '*')
+          @found = found
+          @crawled = crawled
+          @to_follow = to_follow
+        end
+
+        def run(_resume_point)
+          docker_search
+        end
+
+        private
+
+        def parse_parameter(field_value)
+          field_value.lstrip!
+          i = field_value.index(/[ \t=;,]/) || field_value.length
+          name = field_value.slice!(0, i).downcase(:ascii)
+          field_value.lstrip!
+          if field_value.delete_prefix!('=')
+            field_value.lstrip!
+            if field_value.delete_prefix!('"')
+              value = ''
+              until field_value.empty?
+                break if field_value.delete_prefix!('"')
+                field_value.delete_prefix!("\\")
+                value += field_value.slice!(0, 1) || ''
+              end
+            else
+              i = field_value.index(/[;,]/) || field_value.length
+              value = field_value.slice!(0, i)
+            end
+          end
+          {name: name, value: value || ''}
+        end
+
+        def parse_parameters(field_value)
+          seen_rel = false
+          has_next_rel = false
+          has_anchor = false
+          until field_value.empty?
+            field_value.lstrip!
+            break if field_value.delete_prefix!(';').nil?
+            param = parse_parameter(field_value)
+            case
+            when param[:name] == 'rel' && !seen_rel
+              seen_rel = true
+              has_next_rel = param[:value].downcase(:ascii).split(/[ \t]/).include?('next')
+            when param[:name] == 'anchor'
+              has_anchor = true
+            end
+          end
+          {has_next_rel: has_next_rel, has_anchor: has_anchor}
+        end
+
+        def get_next_link(link_header)
+          # This code mostly follows Appendix B "Algorithms of Parsing Link Header Fields" of RFC 8288
+          # "Web Linking", <https://www.rfc-editor.org/rfc/rfc8288#appendix-B> (that RFC appears to be
+          # silent about multiple "next" links, so just use the first one and ignore any additional
+          # ones, in the general spirit of being lenient):
+          return nil if link_header.nil?
+          field_value = link_header.clone
+          until field_value.empty?
+            # The following ignores any junk preceding the next <...> link URL:
+            m = field_value.match(/<(.*)>/)
+            break unless m
+            target_string = m[1]
+            field_value = m.post_match
+            params = parse_parameters(field_value)
+            if params[:has_next_rel]
+              # To keep it simple, ignore a link with an (unlikely) anchor parameter; but the RFC
+              # mandates that we "MUST NOT process the link without applying the anchor", so just raise
+              # an exception in that (unlikely) case:
+              fail "anchor not supported" if params[:has_anchor]
+              return target_string
+            end
+          end
+          nil
+        end
+
+        def docker_search
+          request_params = {
+            method: :get,
+            headers: { accept: :json },
+            url: "#{@uri}v1/search?q=#{@search}"
+          }
+
+          request_params[:user] = @upstream_username if @upstream_username
+          request_params[:password] = @upstream_password if @upstream_password
+          request_params[:proxy] = proxy_uri if proxy
+
+          begin
+            results = RestClient::Request.execute(request_params)
+            JSON.parse(results)['results'].each do |result|
+              @found << result['name']
+            end
+          rescue
+            # Note: v2 endpoint does not support search
+            request_params[:url] = "#{@uri}v2/_catalog"
+            loop do
+              results = RestClient::Request.execute(request_params)
+              JSON.parse(results)['repositories'].each do |result|
+                @found << result
+              end
+              next_uri = get_next_link(results.headers[:link])
+              break if next_uri.nil?
+              request_params[:url] = URI(request_params[:url]).merge(next_uri).to_s
+            end
+          end
+          @found.sort!
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/resources/discovery/yum.rb

@@ -0,0 +1,95 @@
+require 'spidr'
+
+module Katello
+  module Resources
+    module Discovery
+      class Yum < RepoDiscovery
+        attr_reader :found, :crawled, :to_follow
+        def initialize(url, crawled = [], found = [], to_follow = [],
+                       upstream_credentials_and_search = {
+                         upstream_username: nil,
+                         upstream_password: nil
+                       })
+          @uri = uri(url)
+          @upstream_username = upstream_credentials_and_search[:upstream_username].presence
+          @upstream_password = upstream_credentials_and_search[:upstream_password].presence
+          @found = found
+          @crawled = crawled
+          @to_follow = to_follow
+        end
+
+        def run(resume_point)
+          if @uri.scheme == 'file'
+            crawl_file_path(uri(resume_point))
+          elsif %w(http https).include?(@uri.scheme)
+            spidr_crawl_pages(resume_point)
+          end
+        end
+
+        private
+
+        def crawl_file_path(url)
+          if url.path.ends_with?('/repodata/')
+            found_path = Pathname(url.path).parent.to_s
+            @found << "file://#{found_path}"
+          end
+          if url.path == @uri.path
+            Dir.glob("#{@uri.path}**/").each { |path| @to_follow << path }
+            @to_follow.shift
+          end
+          @crawled << url.path
+        end
+
+        def spidr_proxy_details
+          details = {}
+
+          if proxy
+            details[:host] = proxy_host
+            details[:port] = proxy_port
+            details[:user] = proxy.username
+            details[:password] = proxy.password
+          end
+
+          details
+        end
+
+        def spidr_crawl_pages(url)
+          user, password = @upstream_username, @upstream_password
+          Spidr.site(url, proxy: spidr_proxy_details) do |spider|
+            spider.authorized.add(url, user, password) if user && password
+            spider.every_page do |page|
+              page.url.query = nil
+              @crawled << page.url.to_s
+              process_page_urls(page.urls)
+              spider.skip_page!
+            end
+          end
+        end
+
+        def process_page_urls(urls)
+          urls.each do |url|
+            # Remove query parameters to avoid duplicate processing of URLs with sorting parameters etc
+            url.query = nil
+            if url.path.ends_with? 'repodata/'
+              @found << url.to_s.split('repodata/').first
+            else
+              @to_follow << url.to_s if should_follow?(url)
+            end
+          end
+        end
+
+        def should_follow?(url)
+          #Verify:
+          # * link's path includes the base url
+          # * link hasn't already been crawled
+          # * link ends with '/' so it should be a directory
+          # * link doesn't end with '/Packages/', as this increases
+          #       processing time and memory usage considerably
+
+          return url.hostname == @uri.hostname && !@crawled.include?(url.to_s) &&
+            url.path.ends_with?('/') && !url.path.ends_with?('/Packages/')
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/util/http_helper.rb

@@ -0,0 +1,15 @@
+#
+# Commons used helper methods pertaining to HTTP/Rest client etc
+#
+module Katello
+  module Util
+    module HttpHelper
+      def ignore_404_exception(*)
+        yield
+      rescue api.api_exception_class => e
+        raise e unless e.code == 404
+        nil
+      end
+    end
+  end
+end

data/app/models/732bd3db9f64c621c64d2be4f2a838727aac0845.patch

@@ -0,0 +1,61 @@
+From 732bd3db9f64c621c64d2be4f2a838727aac0845 Mon Sep 17 00:00:00 2001
+From: Quinn James <qjames@redhat.com>
+Date: Fri, 23 Jun 2023 19:06:34 +0000
+Subject: [PATCH] Fixes #36530 - Fixed up
+ /katello/api/sync_plans/:sync_plan_id/products and
+ /katello/api/repositories/:repository_id/sync API endpoints.
+
+---
+ app/controllers/katello/api/v2/sync_controller.rb | 1 -
+ app/models/katello/repository.rb                  | 4 ++++
+ app/views/katello/api/v2/sync/index.json.rabl     | 1 -
+ config/routes/api/v2.rb                           | 1 +
+ 4 files changed, 5 insertions(+), 2 deletions(-)
+ delete mode 100644 app/views/katello/api/v2/sync/index.json.rabl
+
+diff --git a/app/controllers/katello/api/v2/sync_controller.rb b/app/controllers/katello/api/v2/sync_controller.rb
+index b43d502264f..cf324a004f6 100644
+--- a/app/controllers/katello/api/v2/sync_controller.rb
++++ b/app/controllers/katello/api/v2/sync_controller.rb
+@@ -4,7 +4,6 @@ class Api::V2::SyncController < Api::V2::ApiController
+     before_action :find_object, :only => [:index]
+     before_action :ensure_library, :only => [:index]
+ 
+-    api :GET, "/organizations/:organization_id/products/:product_id/sync", N_("Get status of repo synchronisation for given product")
+     api :GET, "/repositories/:repository_id/sync", N_("Get status of synchronisation for given repository")
+     def index
+       respond_for_async(:resource => @obj.sync_status)
+diff --git a/app/models/katello/repository.rb b/app/models/katello/repository.rb
+index 1fbf499ec44..ccd76734ba6 100644
+--- a/app/models/katello/repository.rb
++++ b/app/models/katello/repository.rb
+@@ -947,6 +947,10 @@ def in_content_view?(content_view)
+       content_view.repositories.include? self
+     end
+ 
++    def sync_status
++      return latest_dynflow_sync
++    end
++
+     protected
+ 
+     def unit_type_for_removal(type_class = nil)
+diff --git a/app/views/katello/api/v2/sync/index.json.rabl b/app/views/katello/api/v2/sync/index.json.rabl
+deleted file mode 100644
+index 4296aca962b..00000000000
+--- a/app/views/katello/api/v2/sync/index.json.rabl
++++ /dev/null
+@@ -1 +0,0 @@
+-extends 'katello/api/v2/common/async'
+diff --git a/config/routes/api/v2.rb b/config/routes/api/v2.rb
+index 67ad91dbc2c..9ec5312b4e0 100644
+--- a/config/routes/api/v2.rb
++++ b/config/routes/api/v2.rb
+@@ -484,6 +484,7 @@ class ActionDispatch::Routing::Mapper
+         end
+ 
+         api_resources :sync_plans, :only => [:index, :show, :update, :destroy] do
++          api_resources :products, :only => [:index]
+           get :auto_complete_search, :on => :collection
+           put :sync
+         end

data/app/models/katello/content_view.rb

@@ -454,6 +454,8 @@ module Katello
           ).each do |facet|
             facet.update_applicability_counts
             facet.update_errata_status
+          rescue NoMethodError
+            Rails.logger.warn _('Errata statuses not updated for deleted content facet with UUID %s') % facet.uuid
           end
         end
       end

data/app/models/katello/glue/pulp/repos.rb

@@ -133,7 +133,14 @@ module Katello
 
         repo_param[:mirroring_policy] = Katello::RootRepository::MIRRORING_POLICY_ADDITIVE if repo_param[:mirroring_policy].blank?
 
-        RootRepository.new(repo_param.merge(:product_id => self.id))
+        repo_param = repo_param.merge(:product_id => self.id)
+
+        # Container push may concurrently call root add several times before the db can update.
+        if repo_param[:is_container_push]
+          RootRepository.create_or_find_by!(repo_param)
+        else
+          RootRepository.new(repo_param)
+        end
       end
     end
   end

data/app/models/katello/repository.rb

@@ -118,7 +118,7 @@ module Katello
     end}
 
     before_validation :set_pulp_id
-    before_validation :set_container_repository_name, :if => :docker?
+    before_validation :set_container_repository_name, :unless => :skip_container_name?
 
     scope :has_url, -> { joins(:root).where.not("#{RootRepository.table_name}.url" => nil) }
     scope :on_demand, -> { joins(:root).where("#{RootRepository.table_name}.download_policy" => ::Katello::RootRepository::DOWNLOAD_ON_DEMAND) }
@@ -268,6 +268,10 @@ module Katello
       self.content_view_version.content_view
     end
 
+    def skip_container_name?
+      self.library_instance? && self.root.docker? && self.root.is_container_push
+    end
+
     def library_instance?
       self.content_view.default?
     end