katello 4.13.0 → 4.13.1

data/app/models/katello/root_repository.rb

@@ -60,6 +60,8 @@ module Katello
     has_many :repository_references, :class_name => 'Katello::Pulp3::RepositoryReference',
              :dependent => :destroy, :inverse_of => :root_repository
 
+    before_validation :remove_sha1_checksum_type, if: :sha1_checksum?
+
     validates_lengths_from_database :except => [:label]
     validates_with Validators::KatelloLabelFormatValidator, :attributes => :label
     validates_with Validators::KatelloNameFormatValidator, :attributes => :name
@@ -157,6 +159,14 @@ module Katello
       joins(:product).where("#{Katello::Product.table_name}.organization_id" => org)
     end
 
+    def sha1_checksum?
+      checksum_type == 'sha1'
+    end
+
+    def remove_sha1_checksum_type
+      self.checksum_type = nil
+    end
+
     def ensure_content_attribute_restrictions
       CONTENT_ATTRIBUTE_RESTRICTIONS.each do |attribute, value|
         if self.send(attribute).present? && !value.include?(self.content_type)

data/app/services/katello/pulp3/content_view_version/import_validator.rb.bak

@@ -0,0 +1,166 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ImportValidator
+        def initialize(import:)
+          @content_view = import.content_view
+          @path = import.path
+          @smart_proxy = import.smart_proxy
+          @organization = import.organization
+          @metadata_map = import.metadata_map
+          @interested_repos = import.intersecting_repos_library_and_metadata
+          @redhat_library_products = redhat_library_products
+        end
+
+        def check!
+          if @metadata_map.content_view.blank? && !metadata_map.syncable_format?
+            fail _("Content view not provided in the metadata")
+          end
+
+          ensure_non_syncable_path_valid! unless @metadata_map.syncable_format?
+          ensure_pulp_importable!
+          if @content_view && !@content_view.default?
+            ensure_non_composite!
+            ensure_importing_cvv_does_not_exist!
+            ensure_from_cvv_exists!
+          end
+          ensure_manifest_imported!
+          ensure_metadata_matches_repos_in_library!
+          ensure_redhat_products_metadata_are_in_the_library!
+        end
+
+        def ensure_non_syncable_path_valid!
+          uri = URI(@path)
+          unless uri.scheme.blank? || uri.scheme == "file"
+            fail _("Invalid path provided. Content can be only imported from file system. ")
+          end
+        end
+
+        def ensure_non_composite!
+          return if @content_view.blank?
+          fail _("Content cannot be imported into a Composite Content View. ") if @content_view.composite?
+        end
+
+        def ensure_pulp_importable!
+          return if @metadata_map.syncable_format?
+          api = ::Katello::Pulp3::Api::Core.new(@smart_proxy).importer_check_api
+          response = api.pulp_import_check_post(toc: "#{@path}/#{@metadata_map.toc}")
+          unless response.toc.is_valid
+            fail response.toc.messages.join("\n")
+          end
+        end
+
+        def ensure_importing_cvv_does_not_exist!
+          major = @metadata_map.content_view_version.major
+          minor = @metadata_map.content_view_version.minor
+
+          if ::Katello::ContentViewVersion.where(major: major, minor: minor, content_view: @content_view).exists?
+            fail _("Content View Version specified in the metadata - '%{name}' already exists. "\
+                    "If you wish to replace the existing version, delete %{name} and try again. " % { name: "#{@content_view.name} #{major}.#{minor}" })
+          end
+        end
+
+        def ensure_from_cvv_exists!
+          major = @metadata_map.content_view_version.major
+          minor = @metadata_map.content_view_version.minor
+
+          if @metadata_map.from_content_view_version
+            from_major = @metadata_map.from_content_view_version.major
+            from_minor = @metadata_map.from_content_view_version.minor
+
+            unless ::Katello::ContentViewVersion.where(major: from_major, minor: from_minor, content_view: @content_view).exists?
+              fail _("Prior Content View Version specified in the metadata - '%{name}' does not exist. "\
+                      "Please import the metadata for '%{name}' before importing '%{current}' " % { name: "#{@content_view.name} #{from_major}.#{from_minor}",
+                                                                                                    current: "#{@content_view.name} #{major}.#{minor}"})
+            end
+          end
+        end
+
+        def ensure_manifest_imported!
+          any_rh_repos = @metadata_map.repositories.any?(&:redhat)
+          if any_rh_repos && !@organization.manifest_imported?
+            fail _("No manifest found. Import a manifest with the appropriate subscriptions "\
+                   "before importing content.")
+          end
+        end
+
+        def ensure_metadata_matches_repos_in_library!
+          bad_repos = @interested_repos.select do |katello_repo|
+            metadata_repo = metadata_repo_for_katello_repo(katello_repo)
+
+            next unless metadata_repo
+
+            !(katello_repo.content_type == metadata_repo.content_type &&
+              katello_repo.redhat? == metadata_repo.redhat)
+          end
+
+          if bad_repos.any?
+            fail _("The following repositories provided in the import metadata have an incorrect content type or provider type. "\
+                    "Make sure the export and import repositories are of the same type before importing\n "\
+                    "%{repos}" % { repos: generate_product_repo_i18n_string(bad_repos).join("")}
+                  )
+          end
+        end
+
+        def ensure_redhat_products_metadata_are_in_the_library!
+          missing = @metadata_map.repositories.select do |repo|
+            repo.redhat && katello_product_for_metadata_repo(repo).nil?
+          end
+
+          if missing.any?
+            repos_in_import = generate_product_repo_i18n_string(missing)
+            fail _("The organization's manifest does not contain the subscriptions required to enable the following repositories.\n "\
+                    "%{repos}" % { repos: repos_in_import.join("")}
+                  )
+          end
+        end
+
+        def ensure_no_missing_rh_repos!
+          rh_repos = @metadata_map.repositories.select { |r| r.redhat }
+          helper = ::Katello::Pulp3::ContentViewVersion::ImportableRepositories.new(
+            organization: @organization,
+            metadata_repositories: rh_repos,
+            syncable_format: @metadata_map.syncable_format?,
+            path: path
+          )
+          helper.generate!
+          rh_repos = rh_repos.select do |repo|
+          end
+        end
+
+
+        private
+
+        def katello_product_for_metadata_repo(metadata_repo)
+          @redhat_library_products.find do |product|
+            if metadata_repo.redhat && metadata_repo.product.cp_id
+              product.cp_id == metadata_repo.product.cp_id
+            else
+              product.label == metadata_repo.product.label
+            end
+          end
+        end
+
+        def metadata_repo_for_katello_repo(repo)
+          @metadata_map.repositories.find do |metadata_repo|
+            if repo.redhat? && metadata_repo.product.cp_id
+              repo.label == metadata_repo.label && repo.product.cp_id == metadata_repo.product.cp_id
+            else
+              repo.label == metadata_repo.label && repo.product.label == metadata_repo.product.label
+            end
+          end
+        end
+
+        def redhat_library_products
+          ::Katello::Product.in_org(@organization).redhat
+        end
+
+        def generate_product_repo_i18n_string(metadata_repos)
+          metadata_repos.map do |repo|
+            _("\n* Product = '%{product}', Repository = '%{repository}'" % { product: repo.product.name, repository: repo.name })
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/content_view_version/importable_repositories.rb.bak

@@ -0,0 +1,164 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ImportableRepositories
+        attr_reader :creatable, :updatable
+
+        def initialize(organization:,
+                       metadata_repositories:,
+                       syncable_format: false,
+                       path: nil)
+          @organization = organization
+          @metadata_repositories = metadata_repositories
+          @creatable = []
+          @updatable = []
+          @syncable_format = syncable_format
+          @path = path
+        end
+
+        def find_root_repo(metadata_repo:, product:)
+          product.root_repositories.find do |r|
+            if metadata_repo.content&.id && metadata_repo.redhat
+              next if (r.content.cp_content_id != metadata_repo.content.id)
+              next if metadata_repo.arch_substituted? && r.arch != metadata_repo.arch
+              next if metadata_repo.release_version_substituted? && (r.major != metadata_repo.major ||
+                r.minor != metadata_repo.minor)
+              true
+            else
+              r.label == metadata_repo.label
+            end
+          end
+        end
+
+        def generate!
+          # For Red Hat repositories or Custom Repositories in the metadata exclusively
+          # Set up a 2 different list of importable root repositories
+          # creatable: repos that are part of the metadata but not in the library.
+          #         They are ready to be created
+          # updatable: repo that are both in the metadata and library.
+          #         These may contain updates to the repo and hence ready to be updated.
+          @metadata_repositories.each do |repo|
+            product = product_for_metadata_repo(repo)
+            fail _("Unable to find product '%s' in organization '%s'" % [repo.product.label, @organization.name]) if product.blank?
+
+            root = find_root_repo(product: product, metadata_repo: repo)
+            if root
+              updatable << { repository: root, options: update_repo_params(repo) }
+            elsif repo.redhat
+              product_content = fetch_product_content(repo.content, product)
+              substitutions = {}
+              substitutions[:basearch] = repo.arch if repo.content.arch_substituted?
+              substitutions[:releasever] = repo.minor if repo.content.release_version_substituted?
+              create_options = {
+                product: product,
+                content: product_content,
+                substitutions: substitutions,
+                override_url: fetch_feed_url(repo)
+              }
+              create_options[:override_arch] = repo.arch
+              creatable << create_options
+            else
+              creatable << { repository: product.add_repo(create_repo_params(repo, product)) }
+            end
+          end
+        end
+
+        private
+
+        def find_unique_name(metadata_repo, product)
+          name = metadata_repo.name
+          i = 1
+          while product.root_repositories.where(name: name).exists?
+            name = "#{metadata_repo.name} #{i}"
+            i += 1
+          end
+          name
+        end
+
+        def product_for_metadata_repo(repo)
+          if repo.redhat && repo.product.cp_id
+            @organization.products.includes(:root_repositories).find_by(cp_id: repo.product.cp_id)
+          else
+            @organization.products.includes(:root_repositories).find_by(label: repo.product.label)
+          end
+        end
+
+        def fetch_product_content(content_metadata, product)
+          query = ::Katello::Content.joins(:product_contents).where("#{Katello::ProductContent.table_name}.product_id": product.id)
+          table_name = Katello::Content.table_name
+          if content_metadata&.id
+            query.find_by("#{table_name}.cp_content_id": content_metadata.id)
+          else
+            query.find_by("#{table_name}.label": content_metadata.label)
+          end
+        end
+
+        def gpg_key_id(metadata_repo)
+          if metadata_repo.gpg_key
+            @organization.gpg_keys.find_by(name: metadata_repo.gpg_key.name).id
+          end
+        end
+
+        def create_repo_params(metadata_repo, product)
+          keys = [
+            :label,
+            :description,
+            :arch,
+            :unprotected,
+            :content_type,
+            :checksum_type,
+            :os_versions,
+            :major,
+            :minor,
+            :download_policy,
+            :mirroring_policy
+          ]
+
+          params = { name: find_unique_name(metadata_repo, product) }
+          params[:gpg_key_id] = gpg_key_id(metadata_repo)
+          keys.each do |key|
+            params[key] = metadata_repo.send(key)
+          end
+          url = fetch_feed_url(metadata_repo)
+          params[:url] = url if url
+          params
+        end
+
+        def fetch_feed_url(metadata_repo)
+          return unless @syncable_format
+          uri = URI(@path)
+          if uri.scheme.blank? || uri.scheme == "file"
+            "file://#{uri.path.chomp('/')}#{metadata_repo.content.url}"
+          else
+            "#{@path.chomp('/')}#{metadata_repo.content.url}"
+          end
+        end
+
+        def update_repo_params(metadata_repo)
+          keys = [
+            :description,
+            :arch,
+            :unprotected,
+            :checksum_type,
+            :os_versions,
+            :major,
+            :minor,
+            :download_policy,
+            :mirroring_policy
+          ]
+          params = {}
+          url = fetch_feed_url(metadata_repo)
+          params[:url] = url if url
+          params[:gpg_key_id] = gpg_key_id(metadata_repo)
+
+          keys.each do |key|
+            value = metadata_repo.send(key)
+            params[key] = value if value
+          end
+
+          params
+        end
+      end
+    end
+  end
+end

data/app/services/katello/repository_type.rb

@@ -17,7 +17,7 @@ module Katello
               :repositories_api_class, :api_class, :remotes_api_class, :repository_versions_api_class,
               :distributions_api_class, :remote_class, :repo_sync_url_class, :client_module_class,
               :distribution_class, :publication_class, :publications_api_class, :url_description,
-              :test_url, :test_url_root_options
+              :test_url, :test_url_root_options, :repo_discovery_class
 
     attr_accessor :metadata_publish_matching_check, :index_additional_data_proc
     attr_reader :id, :unique_content_per_repo

data/config/initializers/monkeys.rb

@@ -1,5 +1,4 @@
 #place where monkey patches are required
-require 'monkeys/anemone'
 require 'monkeys/ar_postgres_evr_t'
 require 'monkeys/fx_sqlite_skip'
 require 'monkeys/remove_hidden_distribution'

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/details/views/repository-info.html

@@ -208,6 +208,9 @@
             options="checksums"
             on-save="save(repository)">
         </dd>
+        <p ng-hide="repository.checksum_type !== 'sha1'" class="help-block" translate>
+          The sha1 checksum type is no longer supported. The checksum type will reset to 'Default' at the next edit or change in content.
+        </p>
       </span>
 
       <span ng-show="repository.content_type === 'yum' && repository.mirroring_policy === 'additive'">

data/lib/katello/repository_types/docker.rb

@@ -2,6 +2,7 @@ Katello::RepositoryTypeManager.register(::Katello::Repository::DOCKER_TYPE) do
   default_managed_content_type Katello::DockerManifest::CONTENT_TYPE
   pulp3_service_class Katello::Pulp3::Repository::Docker
   pulp3_api_class Katello::Pulp3::Api::Docker
+  repo_discovery_class ::Katello::Resources::Discovery::Container
   pulp3_skip_publication true
   pulp3_plugin 'container'
 

data/lib/katello/repository_types/yum.rb

@@ -1,6 +1,7 @@
 Katello::RepositoryTypeManager.register(::Katello::Repository::YUM_TYPE) do
   pulp3_service_class Katello::Pulp3::Repository::Yum
   pulp3_api_class Katello::Pulp3::Api::Yum
+  repo_discovery_class ::Katello::Resources::Discovery::Yum
   pulp3_plugin 'rpm'
   prevent_unneeded_metadata_publish
 

data/lib/katello/tasks/update_repository_expiry.rake

@@ -0,0 +1,114 @@
+namespace :katello do
+  def commit?
+    ENV['COMMIT'] == 'true' || ENV['FOREMAN_UPGRADE'] == '1'
+  end
+
+  desc "Regnerate metadata for all repositories. Specify CONTENT_VIEW=name and LIFECYCLE_ENVIRONMENT=name to narrow repositories."
+  task :regenerate_repo_metadata => ["dynflow:client", "check_ping"] do
+    User.current = User.anonymous_api_admin
+    repos = lookup_repositories
+
+    if repos.any?
+      task = ForemanTasks.async_task(Actions::Katello::Repository::BulkMetadataGenerate, repos.all.order_by_root(:name))
+      puts "Regenerating #{repos.count} repositories.  You can monitor these on task id #{task.id}\n"
+    else
+      puts "No repositories found for regeneration."
+    end
+  end
+
+  desc "Refresh repository metadata for all repositories. Specify CONTENT_VIEW=name and LIFECYCLE_ENVIRONMENT=name to narrow repositories."
+  task :refresh_pulp_repo_details => ["dynflow:client", "check_ping"] do
+    User.current = User.anonymous_api_admin
+    repos = lookup_repositories
+
+    if repos.any?
+      task = ForemanTasks.async_task(::Actions::BulkAction, Actions::Katello::Repository::RefreshRepository, repos.all.order_by_root(:name))
+      puts "Refreshing #{repos.count} repositories.  You can monitor these on task id #{task.id}\n"
+    else
+      puts "No repositories found for regeneration."
+    end
+  end
+
+  desc "Correct missing pulp repositories. Specify CONTENT_VIEW=name and LIFECYCLE_ENVIRONMENT=name to narrow repositories.  COMMIT=true to perform operation."
+  task :correct_repositories => ["environment", "check_ping"] do
+    puts "All operations will be skipped.  Re-run with COMMIT=true to perform corrections." unless commit?
+
+    User.current = User.anonymous_api_admin
+    repos = lookup_repositories
+
+    repos.find_each.with_index do |repo, index|
+      puts "Processing Repository #{index + 1}/#{repos.count}: #{repo.name} (#{repo.id})"
+      unless repo_exists?(repo)
+        handle_missing_repo(repo)
+      end
+    end
+
+    ::Katello::RootRepository.orphaned.each do |root_repo|
+      handle_missing_root_repo(root_repo)
+    end
+  end
+
+  desc "Change the download policy of all repos. Specify DOWNLOAD_POLICY=policy. Options are immediate or on_demand."
+  task :change_download_policy => ["environment", "check_ping"] do
+    policy = ENV['DOWNLOAD_POLICY']
+    unless ::Katello::RootRepository::DOWNLOAD_POLICIES.include?(policy)
+      puts "Invalid download policy specified: '#{policy}'. "
+      puts "Options are immediate or on_demand."
+      next
+    end
+
+    User.current = User.anonymous_api_admin
+    repos = Katello::Repository.yum_type.where(library_instance_id: nil)
+
+    repos.find_each.with_index do |repo, index|
+      puts "Processing Repository #{index + 1}/#{repos.count}: #{repo.name} (#{repo.id})"
+      begin
+        ForemanTasks.sync_task(::Actions::Katello::Repository::Update, repo.root,
+                               download_policy: policy)
+      rescue => e
+        puts "Failed to update repository #{repo.name} (#{repo.id}): #{e.message}"
+      end
+    end
+  end
+
+  def lookup_repositories
+    lifecycle_envs = Katello::KTEnvironment.where(:name => ENV['LIFECYCLE_ENVIRONMENT']) if ENV['LIFECYCLE_ENVIRONMENT']
+    content_views = Katello::ContentView.where(:name => ENV['CONTENT_VIEW']) if ENV['CONTENT_VIEW']
+
+    repos = ::Katello::Repository
+    repos = repos.in_environment(lifecycle_envs) if lifecycle_envs
+    repos = repos.in_content_views(content_views) if content_views
+    repos
+  end
+
+  def repo_exists?(repo)
+    if SmartProxy.pulp_primary!.pulp3_support?(repo)
+      backend_service = repo.backend_service(SmartProxy.pulp_primary!)
+      return false unless backend_service&.repository_reference&.repository_href
+      backend_service.api.repositories_api.read(backend_service.repository_reference.repository_href)
+    else
+      false
+    end
+    true
+  rescue StandardError => e
+    return false if e.code == 404
+  end
+
+  def handle_missing_repo(repo)
+    puts "Repository #{repo.id} Missing"
+    if repo.content_view.default?
+      puts "Recreating #{repo.id}"
+      if commit?
+        ForemanTasks.sync_task(::Actions::Katello::Repository::Create, repo, force_repo_create: true)
+        repo.reload.index_content
+      end
+    else
+      puts "Deleting #{repo.id}"
+      ForemanTasks.sync_task(::Actions::Katello::Repository::Destroy, repo) if commit?
+    end
+  end
+
+  def handle_missing_root_repo(root_repo)
+    root_repo.destroy! if commit?
+  end
+end

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "4.13.0".freeze
+  VERSION = "4.13.1".freeze
 end

data/lib/katello.rb

@@ -8,7 +8,6 @@ require "gettext_i18n_rails"
 require "foreman-tasks"
 require "foreman_remote_execution"
 require "rest_client"
-require "anemone"
 require "securerandom"
 
 require "deface"
@@ -21,7 +20,6 @@ require lib_foreman if Dir.exist?(lib_foreman)
 
 require File.expand_path("../engines/bastion/lib/bastion", File.dirname(__FILE__))
 require File.expand_path("../engines/bastion_katello/lib/bastion_katello", File.dirname(__FILE__))
-require "monkeys/anemone"
 
 module Katello
   require "katello/version"