katello 3.16.0 → 3.16.1

data/app/models/setting/content.rb

@@ -92,7 +92,7 @@ class Setting::Content < Setting
       self.set('check_services_before_actions', N_("Whether or not to check the status of backend services such as pulp and candlepin prior to performing some actions."),
                true, N_('Check services before actions')),
       self.set('foreman_proxy_content_batch_size', N_("How many repositories should be synced concurrently on the capsule.  A smaller number may lead to longer sync times.  A larger number will increase dynflow load."),
-               25, N_('Batch size to sync repositories in.')),
+               100, N_('Batch size to sync repositories in.')),
       self.set('foreman_proxy_content_auto_sync', N_("Whether or not to auto sync the Smart Proxies after a Content View promotion."),
                true, N_('Sync Smart Proxies after Content View promotion')),
       self.set('default_download_policy', N_("Default download policy for custom repositories (either 'immediate', 'on_demand', or 'background')"), "immediate",

data/app/services/cert/certs.rb

@@ -9,11 +9,19 @@ module Cert
     end
 
     def self.ssl_client_cert
-      @ssl_client_cert ||= OpenSSL::X509::Certificate.new(File.open(Setting['pulp_client_cert'], 'r').read)
+      @ssl_client_cert ||= OpenSSL::X509::Certificate.new(File.open(ssl_client_cert_filename, 'r').read)
+    end
+
+    def self.ssl_client_cert_filename
+      Setting['pulp_client_cert']
     end
 
     def self.ssl_client_key
-      @ssl_client_key ||= OpenSSL::PKey::RSA.new(File.open(Setting['pulp_client_key'], 'r').read)
+      @ssl_client_key ||= OpenSSL::PKey::RSA.new(File.open(ssl_client_key_filename, 'r').read)
+    end
+
+    def self.ssl_client_key_filename
+      Setting['pulp_client_key']
     end
 
     def self.verify_ueber_cert(organization)

data/app/services/katello/pulp3/repository.rb

@@ -193,16 +193,15 @@ module Katello
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
-      def sync_params
-        {remote: repo.remote_href, mirror: repo.root.mirror_on_sync}
+      def sync(options = {})
+        repository_sync_url_data = api.class.repository_sync_url_class.new(sync_url_params(options))
+        [api.repositories_api.sync(repository_reference.repository_href, repository_sync_url_data)]
       end
 
-      def sync
-        sync_url_params = sync_params
-        skip_type_param = skip_types
-        sync_url_params[:skip_types] = skip_type_param if skip_type_param
-        repository_sync_url_data = api.class.repository_sync_url_class.new(sync_url_params)
-        [api.repositories_api.sync(repository_reference.repository_href, repository_sync_url_data)]
+      def sync_url_params(_sync_options)
+        params = {remote: repo.remote_href, mirror: repo.root.mirror_on_sync}
+        params[:skip_types] = skip_types if skip_types
+        params
       end
 
       def create_publication

data/app/services/katello/pulp3/repository/yum.rb

@@ -3,10 +3,13 @@ require 'pulp_rpm_client'
 module Katello
   module Pulp3
     class Repository
+      # rubocop:disable Metrics/ClassLength
       class Yum < ::Katello::Pulp3::Repository
         include Katello::Util::Errata
         include Katello::Util::PulpcoreContentFilters
 
+        UNIT_LIMIT = 10_000
+
         def remote_options
           if root.url.blank?
             common_remote_options.merge(url: nil, policy: root.download_policy)
@@ -31,10 +34,6 @@ module Katello
           }
         end
 
-        def sync_params
-          {remote: repo.remote_href, mirror: repo.root.mirror_on_sync, optimize: false}
-        end
-
         def mirror_remote_options
           policy = smart_proxy.download_policy
 
@@ -63,6 +62,12 @@ module Katello
           end
         end
 
+        def sync_url_params(sync_options)
+          params = super
+          params[:optimize] = sync_options[:optimize] if sync_options.key?(:optimize)
+          params
+        end
+
         def self.distribution_bootable?(distribution)
           file_paths = distribution.results.first.images.map(&:path)
           file_paths.any? do |path|
@@ -102,14 +107,71 @@ module Katello
                 data.config << config
               end
             end
-            # FIXME: data's content being [] causes all content to be copied back
-            tasks << api.copy_api.copy_content(data)
+            tasks << copy_content_chunked(data)
           else
             tasks << remove_all_content_from_mapping(repo_id_map)
           end
           tasks.flatten
         end
 
+        def copy_api_data_dup(data)
+          data_dup = PulpRpmClient::Copy.new
+          data_dup.dependency_solving = data.dependency_solving
+          data_dup.config = []
+          data.config.each do |repo_config|
+            config_hash = {
+              source_repo_version: repo_config[:source_repo_version],
+              dest_repo: repo_config[:dest_repo],
+              content: []
+            }
+            config_hash[:dest_base_version] = repo_config[:dest_base_version] if repo_config[:dest_base_version]
+            data_dup.config << config_hash
+          end
+          data_dup
+        end
+
+        def copy_content_chunked(data)
+          tasks = []
+          # Don't chunk if there aren't enough content units
+          if data.config.sum { |repo_config| repo_config[:content].size } <= UNIT_LIMIT
+            return api.copy_api.copy_content(data)
+          end
+
+          unit_copy_counter = 0
+          i = 0
+          leftover_units = data.config.first[:content].deep_dup
+
+          # Copy data and clear its content fields
+          data_dup = copy_api_data_dup(data)
+
+          while i < data_dup.config.size
+            # Copy all units within repo or only some?
+            if leftover_units.length < UNIT_LIMIT - unit_copy_counter
+              copy_amount = leftover_units.length
+            else
+              copy_amount = UNIT_LIMIT - unit_copy_counter
+            end
+
+            data_dup.config[i][:content] = leftover_units.pop(copy_amount)
+            unit_copy_counter += copy_amount
+            # Do copy call if limit is reached or if we're under the limit but on the last repo config.
+            if unit_copy_counter >= UNIT_LIMIT || (i == data_dup.config.size - 1 && leftover_units.empty?)
+              tasks << api.copy_api.copy_content(data_dup)
+              unit_copy_counter = 0
+            end
+
+            if leftover_units.empty?
+              # Nothing more to copy -- clear current config's content
+              data_dup.config[i][:content] = []
+              i += 1
+              # Fetch unit list for next data config
+              leftover_units = data.config[i][:content].deep_dup unless i == data_dup.config.size
+            end
+          end
+
+          tasks
+        end
+
         def remove_all_content_from_mapping(repo_id_map)
           tasks = []
           repo_id_map.each do |_source_repo_ids, dest_repo_id_map|
@@ -147,6 +209,11 @@ module Katello
           api.repositories_api.modify(repo_href, data)
         end
 
+        def repair(repository_version_href)
+          data = PulpRpmClient::RepositoryVersion.new
+          api.repository_versions_api.repair(repository_version_href, data)
+        end
+
         def remove_all_content
           data = PulpRpmClient::RepositoryAddRemoveContent.new(
             remove_content_units: ['*'])

data/app/services/katello/pulp3/repository_mirror.rb

@@ -11,6 +11,11 @@ module Katello
         @repo_service = repository_service
       end
 
+      def content_guard_href
+        content_guard_api = ::Katello::Pulp3::Api::ContentGuard.new(smart_proxy)
+        content_guard_api.list&.results&.first&.pulp_href
+      end
+
       def backend_object_name
         #Create repos in pulp3 instance with the name as this repo's pulp_id
         repo.pulp_id
@@ -78,6 +83,7 @@ module Katello
           base_path: path,
           name: "#{backend_object_name}"
         }
+        ret[:content_guard] = repo.unprotected ? nil : content_guard_href
         ret[:publication] = options[:publication] if options.key? :publication
         ret[:repository_version] = options[:repository_version] if options.key? :repository_version
         ret
@@ -85,10 +91,9 @@ module Katello
 
       def remote_options
         base_options = common_remote_options
+        base_options.merge(url: remote_feed_url)
         if (type_specific_options = repo_service.try(:mirror_remote_options))
           base_options.merge(type_specific_options)
-        else
-          base_options.merge(url: remote_feed_url)
         end
       end
 

data/app/services/katello/pulp3/smart_proxy_mirror_repository.rb

@@ -10,7 +10,7 @@ module Katello
         repo_map = {}
 
         smart_proxy_helper = ::Katello::SmartProxyHelper.new(smart_proxy)
-        katello_pulp_ids = smart_proxy_helper.repos_available_to_capsule.map(&:pulp_id)
+        katello_pulp_ids = smart_proxy_helper.combined_repos_available_to_capsule.map(&:pulp_id)
         pulp3_enabled_repo_types.each do |repo_type|
           api = repo_type.pulp3_service_class.api(smart_proxy)
           repo_map[api] = api.list_all.reject { |capsule_repo| katello_pulp_ids.include? capsule_repo.name }

data/app/services/katello/smart_proxy_helper.rb

@@ -13,34 +13,31 @@ module Katello
       @smart_proxy.pulp_master?
     end
 
-    def repos_available_to_capsule(environment = nil, content_view = nil, repository = nil)
-      ret = []
-      if repository
-        environment = repository.environment
-        ret = [repository]
-      else
-        yum_repos = fetch_repos_available_to_capsule(environment, content_view) || []
-        puppet_envs = fetch_puppet_environments_available_to_capsule(environment, content_view) || []
-        ret = yum_repos + puppet_envs
-      end
+    def lifecycle_environment_check(environment = nil, repository = nil)
+      environment = repository.environment if repository
 
       if environment && !self.smart_proxy.lifecycle_environments.include?(environment)
         fail _("Lifecycle environment '%{environment}' is not attached to this capsule.") % { :environment => environment.name }
       end
-
-      ret
     end
 
-    private
+    def combined_repos_available_to_capsule(environment = nil, content_view = nil, repository = nil)
+      lifecycle_environment_check(environment, repository)
+      if repository
+        [repository]
+      else
+        repositories_available_to_capsule(environment, content_view) + puppet_environments_available_to_capsule(environment, content_view)
+      end
+    end
 
-    def fetch_repos_available_to_capsule(environments = nil, content_view = nil)
+    def repositories_available_to_capsule(environments, content_view)
       environments = @smart_proxy.lifecycle_environments if environments.nil?
       yum_repos = Katello::Repository.in_environment(environments)
       yum_repos = yum_repos.in_content_views([content_view]) if content_view
-      yum_repos.select(&:node_syncable?)
+      yum_repos.smart_proxy_syncable
     end
 
-    def fetch_puppet_environments_available_to_capsule(environments = nil, content_view = nil)
+    def puppet_environments_available_to_capsule(environments, content_view)
       environments = @smart_proxy.lifecycle_environments if environments.nil?
       puppet_environments = Katello::ContentViewPuppetEnvironment.in_environment(environments)
       puppet_environments = puppet_environments.in_content_view(content_view) if content_view

data/config/routes/api/rhsm.rb

@@ -21,6 +21,7 @@ Katello::Engine.routes.draw do
       match '/owners/:organization_id/environments' => 'candlepin_proxies#rhsm_index', :via => :get
       match '/owners/:organization_id/pools' => 'candlepin_proxies#get', :via => :get, :as => :proxy_owner_pools_path
       match '/owners/:organization_id/servicelevels' => 'candlepin_proxies#get', :via => :get, :as => :proxy_owner_servicelevels_path
+      match '/owners/:organization_id/system_purpose' => 'candlepin_proxies#get', :via => :get, :as => :proxy_owner_system_purpose_path
       match '/environments/:environment_id/consumers' => 'candlepin_proxies#consumer_create', :via => :post
       match '/consumers/:id' => 'candlepin_proxies#consumer_show', :via => :get
       match '/consumers/:id' => 'candlepin_proxies#regenerate_identity_certificates', :via => :post

data/config/routes/api/v2.rb

@@ -392,6 +392,7 @@ Katello::Engine.routes.draw do
             put :remove_docker_manifests, :action => :remove_content
             put :remove_content
             post :sync
+            post :verify_checksum
             post :export
             post :upload_content
             put :import_uploads
@@ -421,6 +422,7 @@ Katello::Engine.routes.draw do
           collection do
             match '/bulk/destroy' => 'products_bulk_actions#destroy_products', :via => :put
             match '/bulk/sync' => 'products_bulk_actions#sync_products', :via => :put
+            match '/bulk/verify_checksum' => 'products_bulk_actions#verify_checksum_products', :via => :put
             match '/bulk/sync_plan' => 'products_bulk_actions#update_sync_plans', :via => :put
             match '/bulk/http_proxy' => 'products_bulk_actions#update_http_proxy', :via => :put
           end

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/bulk/product-bulk-action.factory.js

@@ -13,6 +13,7 @@ angular.module('Bastion.products').factory('ProductBulkAction',
         return BastionResource('katello/api/v2/products/bulk/:action', {}, {
             removeProducts: {method: 'PUT', params: {action: 'destroy'}},
             syncProducts: {method: 'PUT', params: {action: 'sync'}},
+            verifyChecksumProducts: {method: 'PUT', params: {action: 'verify_checksum'}},
             updateProductSyncPlan: {method: 'PUT', params: {action: 'sync_plan'}},
             updateProductHttpProxy: {method: 'PUT', params: {action: 'http_proxy'}}
         });

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/details/repository-details.controller.js

@@ -70,6 +70,12 @@
             }, errorHandler);
         };
 
+        $scope.verifyChecksum = function (repository) {
+            Repository.verifyChecksum({id: repository.id}, function (task) {
+                $state.go('product.repository.tasks.details', {taskId: task.id});
+            }, errorHandler);
+        };
+
         $scope.republishRepository = function (repository) {
             Repository.republish({id: repository.id}, function (task) {
                 $state.go('product.repository.tasks.details', {taskId: task.id});

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/details/views/repository-details.html

@@ -32,6 +32,12 @@
           </a>
         </li>
 
+        <li role="menuitem" ng-hide="hideSyncButton(repository, true)" ng-class="{disabled: disableSyncLink()}">
+          <a ng-click="verifyChecksum(repository)" disable-link="disableSyncLink()" translate>
+            Verify Content Checksum
+          </a>
+        </li>
+
         <li role="menuitem" ng-hide="syncInProgress(repository.last_sync) || denied('edit_products', product)">
           <a ng-click="republishRepository(repository)" translate>
             Republish Repository Metadata
@@ -90,4 +96,4 @@
   <div data-block="content">
     <section ui-view></section>
   </div>
-</div>
+</div>

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/repository.factory.js

@@ -17,6 +17,7 @@ angular.module('Bastion.repositories').factory('Repository',
             {
                 update: { method: 'PUT' },
                 sync: { method: 'POST', params: { action: 'sync' } },
+                verifyChecksum: { method: 'POST', params: { action: 'verify_checksum' }},
                 removePackages: { method: 'PUT', params: { action: 'remove_packages'}},
                 removeContent: { method: 'PUT', params: { action: 'remove_content'}},
                 autocomplete: {method: 'GET', isArray: true, params: {id: 'auto_complete_search'}},

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/products.controller.js

@@ -101,6 +101,21 @@ angular.module('Bastion.products').controller('ProductsController',
             ProductBulkAction.syncProducts(getBulkParams(), success, bulkError);
         };
 
+        $scope.verifyChecksumProducts = function () {
+            var success;
+
+            success = function (task) {
+                var message = translate("Product verify checksum has been initiated in the background.");
+                Notification.setSuccessMessage(message, {
+                    link: {
+                        children: translate("Click to monitor task progress."),
+                        href: translate("/foreman_tasks/tasks/%taskId").replace('%taskId', task.id)
+                    }});
+            };
+
+            ProductBulkAction.verifyChecksumProducts(getBulkParams(), success, bulkError);
+        };
+
         $scope.goToDiscoveries = function () {
             nutupane.table.rows = [];
             nutupane.table.resource.results = [];

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/views/products.html

@@ -36,6 +36,12 @@
           </a>
         </li>
 
+        <li role="menuitem" ng-show="permitted('edit_products')" ng-class="{disabled: table.numSelected === 0}">
+          <a ng-click="verifyChecksumProducts()" disable-link="table.numSelected === 0" translate>
+            Verify Content Checksum
+          </a>
+        </li>
+
         <li role="menuitem" ng-show="permitted('edit_products')" ng-class="{disabled: table.numSelected === 0}">
           <a ng-click="openSyncPlanModal()" disable-link="table.numSelected === 0" translate>
             Manage Sync Plan

data/lib/katello/permission_creator.rb

@@ -304,8 +304,8 @@ module Katello
       @plugin.permission :edit_products,
                          {
                            'katello/api/v2/products' => [:update],
-                           'katello/api/v2/repositories' => [:update, :remove_content, :import_uploads, :upload_content, :republish],
-                           'katello/api/v2/products_bulk_actions' => [:update_sync_plans, :update_http_proxy],
+                           'katello/api/v2/repositories' => [:update, :remove_content, :import_uploads, :upload_content, :republish, :verify_checksum],
+                           'katello/api/v2/products_bulk_actions' => [:update_sync_plans, :update_http_proxy, :verify_checksum_products],
                            'katello/api/v2/content_uploads' => [:create, :update, :destroy],
                            'katello/api/v2/organizations' => [:repo_discover, :cancel_repo_discover]
                          },

data/lib/katello/plugin.rb

@@ -218,7 +218,6 @@ Foreman::Plugin.register :katello do
        :parent => :lab_features_menu,
        :turbolinks => false
 
-  allowed_template_helpers :subscription_manager_configuration_url, :repository_url
   extend_template_helpers Katello::KatelloUrlsHelper
   extend_template_helpers Katello::Concerns::BaseTemplateScopeExtensions
 

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "3.16.0".freeze
+  VERSION = "3.16.1".freeze
 end

data/webpack/redux/actions/RedHatRepositories/helpers.js

@@ -35,7 +35,7 @@ const recommendedRepositoriesSatTools = [
 const recommendedRepositoriesMisc = [
   'rhel-server-rhscl-7-rpms',
   'rhel-7-server-satellite-capsule-6.8-rpms',
-  'rhel-7-server-ansible-2.8-rpms',
+  'rhel-7-server-ansible-2.9-rpms',
 ];
 
 const recommendedRepositorySetLables = recommendedRepositoriesRHEL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: katello
 version: !ruby/object:Gem::Version
-  version: 3.16.0
+  version: 3.16.1
 platform: ruby
 authors:
 - N/A
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-10 00:00:00.000000000 Z
+date: 2020-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -87,6 +87,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.1
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -94,6 +97,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.1
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
 - !ruby/object:Gem::Dependency
   name: dynflow
   requirement: !ruby/object:Gem::Requirement
@@ -892,13 +898,13 @@ files:
 - app/lib/actions/katello/repository/sync_hook.rb
 - app/lib/actions/katello/repository/update.rb
 - app/lib/actions/katello/repository/update_content_urls.rb
-- app/lib/actions/katello/repository/update_cv_repo_cert_guard.rb
 - app/lib/actions/katello/repository/update_http_proxy_details.rb
 - app/lib/actions/katello/repository/update_metadata_sync.rb
 - app/lib/actions/katello/repository/update_redhat_repository.rb
 - app/lib/actions/katello/repository/upload_errata.rb
 - app/lib/actions/katello/repository/upload_files.rb
 - app/lib/actions/katello/repository/upload_package_group.rb
+- app/lib/actions/katello/repository/verify_checksum.rb
 - app/lib/actions/katello/repository_set/disable_repository.rb
 - app/lib/actions/katello/repository_set/enable_repository.rb
 - app/lib/actions/katello/repository_set/scan_cdn.rb
@@ -980,12 +986,13 @@ files:
 - app/lib/actions/pulp/repository_group/create.rb
 - app/lib/actions/pulp/repository_group/delete.rb
 - app/lib/actions/pulp/repository_group/export.rb
-- app/lib/actions/pulp3/ContentGuard/refresh.rb
 - app/lib/actions/pulp3/abstract.rb
 - app/lib/actions/pulp3/abstract_async_task.rb
 - app/lib/actions/pulp3/capsule_content/generate_metadata.rb
+- app/lib/actions/pulp3/capsule_content/refresh_content_guard.rb
 - app/lib/actions/pulp3/capsule_content/refresh_distribution.rb
 - app/lib/actions/pulp3/capsule_content/sync.rb
+- app/lib/actions/pulp3/content_guard/refresh.rb
 - app/lib/actions/pulp3/content_migration.rb
 - app/lib/actions/pulp3/content_view/delete_repository_references.rb
 - app/lib/actions/pulp3/import_migration.rb
@@ -1000,6 +1007,7 @@ files:
 - app/lib/actions/pulp3/orchestration/repository/refresh_repos.rb
 - app/lib/actions/pulp3/orchestration/repository/remove_units.rb
 - app/lib/actions/pulp3/orchestration/repository/sync.rb
+- app/lib/actions/pulp3/orchestration/repository/trigger_update_repo_cert_guard.rb
 - app/lib/actions/pulp3/orchestration/repository/update.rb
 - app/lib/actions/pulp3/orchestration/repository/upload_content.rb
 - app/lib/actions/pulp3/orphan_cleanup/delete_orphan_distributions.rb
@@ -1023,8 +1031,10 @@ files:
 - app/lib/actions/pulp3/repository/multi_copy_units.rb
 - app/lib/actions/pulp3/repository/presenters/abstract_sync_presenter.rb
 - app/lib/actions/pulp3/repository/presenters/content_unit_presenter.rb
+- app/lib/actions/pulp3/repository/presenters/repair_presenter.rb
 - app/lib/actions/pulp3/repository/refresh_distribution.rb
 - app/lib/actions/pulp3/repository/remove_units.rb
+- app/lib/actions/pulp3/repository/repair.rb
 - app/lib/actions/pulp3/repository/save_artifact.rb
 - app/lib/actions/pulp3/repository/save_distribution_references.rb
 - app/lib/actions/pulp3/repository/save_publication.rb