katello 3.18.1 → 3.18.3.1

data/app/services/katello/pulp3/repository/apt.rb

@@ -27,8 +27,7 @@ module Katello
           popts = super(repository_version)
           popts.merge!(
             {
-              # structured is not necessary for subscription-manager
-              #structured: true, # publish real suites (e.g. 'stable')
+              structured: true, # publish real suites (e.g. 'stable')
               simple: true # publish all into 'default'-suite
             }
           )

data/app/services/katello/pulp3/repository/yum.rb

@@ -14,7 +14,16 @@ module Katello
           url, sles_token = extract_sles_token
           options = common_remote_options
           options.merge!(sles_auth_token: sles_token) if sles_token
-          options.merge!(url: url, policy: root.download_policy)
+
+          options.merge!(url: url, policy: translated_download_policy)
+        end
+
+        def translated_download_policy
+          if root.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_BACKGROUND
+            ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE
+          else
+            root.download_policy
+          end
         end
 
         def extract_sles_token

data/app/services/katello/pulp3/rpm.rb

@@ -111,6 +111,10 @@ module Katello
         backend_data['rpm_license']
       end
 
+      def parse_filename(path)
+        File.split(path).last unless path.blank?
+      end
+
       def update_model(model)
         custom_json = {}
         custom_json['modular'] = backend_data['is_modular']
@@ -119,7 +123,7 @@ module Katello
           each { |field| custom_json[field] = backend_data[field] }
         custom_json['release_sortable'] = Util::Package.sortable_version(backend_data['release'])
         custom_json['version_sortable'] = Util::Package.sortable_version(backend_data['version'])
-        custom_json['filename'] = backend_data['location_href']
+        custom_json['filename'] = parse_filename(backend_data['location_href']) #location_href is the relative path of the rpm in the upstream repo
         custom_json['checksum'] = backend_data['pkgId']
         custom_json['sourcerpm'] = backend_data['rpm_sourcerpm']
         model.assign_attributes(custom_json)

data/app/services/katello/pulp3/task.rb

@@ -56,8 +56,10 @@ module Katello
         @pulp_data ||= tasks_api.read(@href).as_json.with_indifferent_access
       end
 
-      def tasks_api
-        ::Katello::Pulp3::Api::Core.new(@smart_proxy).tasks_api
+      delegate :tasks_api, to: :core_api
+
+      def core_api
+        ::Katello::Pulp3::Api::Core.new(@smart_proxy)
       end
 
       def task_group_href
@@ -94,10 +96,11 @@ module Katello
       end
 
       def cancel
-        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
-        tasks_api.tasks_cancel(task_data['pulp_href'], data)
+        core_api.cancel_task(task_data['pulp_href'])
         #the main task may have completed, so cancel spawned tasks too
-        task_data['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
+        task_data['spawned_tasks']&.each do |spawned|
+          core_api.cancel_task(spawned['pulp_href'])
+        end
       end
     end
   end

data/app/services/katello/pulp3/task_group.rb

@@ -71,17 +71,25 @@ module Katello
       end
 
       def error
+        return if task_group_data[WAITING] > 0 || task_group_data[RUNNING] > 0
         if task_group_data[FAILED] > 0
           "#{task_group_data[FAILED]} subtask(s) failed for task group #{@href}."
+        elsif task_group_data[CANCELLED] > 0
+          "#{task_group_data[CANCELLED]} subtask(s) cancelled for task group #{@href}."
         end
       end
 
+      def core_api
+        ::Katello::Pulp3::Api::Core.new(@smart_proxy)
+      end
+
       def cancel
-        tasks_api = ::Katello::Pulp3::Api::Core.new(@smart_proxy).tasks_api
-        tasks_response = tasks_api.list(task_group: task_group_data['pulp_href'])
-        data = PulpcoreClient::TaskResponse.new(state: 'canceled')
-        tasks_response.results.collect do |result|
-          tasks_api.tasks_cancel(result.pulp_href, data)
+        tasks_api = core_api.tasks_api
+        tasks_response = core_api.fetch_from_list do |page_opts|
+          tasks_api.list(page_opts.merge(task_group: task_group_data['pulp_href'], state__in: 'running,waiting'))
+        end
+        tasks_response.collect do |result|
+          ::Katello::Pulp3::Api::Core.new(@smart_proxy).cancel_task(result.pulp_href)
         end
       end
     end

data/app/views/katello/sync_management/_products.html.erb

@@ -42,5 +42,5 @@
     </tbody>
   </table>
 
-  <%= submit_tag _('Synchronize Now'), :class => 'button fr', :id => 'sync_button' %>
+  <%= submit_tag _('Synchronize Now'), :class => 'btn btn-default fr', :id => 'sync_button' %>
 <% end %>

data/db/migrate/20150930183738_migrate_content_hosts.rb

@@ -329,7 +329,7 @@ class MigrateContentHosts < ActiveRecord::Migration[4.2]
   # rubocop:disable Metrics/MethodLength
   # rubocop:disable Metrics/AbcSize
   def up
-    if User.where(:login => User::ANONYMOUS_API_ADMIN).first.nil?
+    unless User.unscoped.where(:login => User::ANONYMOUS_API_ADMIN).exists?
       logger.warn("Foreman anonymous admin does not exist, skipping content host migration.")
       return
     end

data/db/migrate/20200514092553_move_katello_fields_from_hostgroups.katello.rb

@@ -1,6 +1,6 @@
 class MoveKatelloFieldsFromHostgroups < ActiveRecord::Migration[6.0]
   def up
-    if User.where(login: User::ANONYMOUS_ADMIN).exists?
+    if User.unscoped.where(login: User::ANONYMOUS_ADMIN).exists?
       User.as_anonymous_admin do
         copy_data_from_hostgroup
       end
@@ -46,7 +46,10 @@ class MoveKatelloFieldsFromHostgroups < ActiveRecord::Migration[6.0]
         content_facet.kickstart_repository_id = kickstart_repository_id
         content_facet.content_view_id = content_view_id
         content_facet.lifecycle_environment_id = lifecycle_environment_id
-        content_facet.save!
+        unless content_facet.save(validate: false)
+          Rails.logger.warn("Unable to save content facet hostgroup for #{content_facet.inspect} ")
+          Rails.logger.warn(content_facet.errors.full_messages.join("\n"))
+        end
       end
     end
   end

data/db/migrate/20210201165835_add_migration_missing_content.rb

@@ -0,0 +1,12 @@
+class AddMigrationMissingContent < ActiveRecord::Migration[6.0]
+  def change
+    content_models = [Katello::Rpm, Katello::ModuleStream, Katello::Erratum, Katello::PackageGroup, Katello::YumMetadataFile,
+                      Katello::Srpm, Katello::FileUnit, Katello::DockerManifestList, Katello::DockerManifest, Katello::DockerTag,
+                      Katello::Deb]
+
+    content_models.each do |model|
+      add_column model.table_name, :missing_from_migration, :bool, :default => false, :null => false
+      add_column model.table_name, :ignore_missing_from_migration, :bool, :default => false, :null => false
+    end
+  end
+end

data/db/migrate/20210420140050_add_pulp3_hrefs_to_content_types_deb.rb

@@ -0,0 +1,5 @@
+class AddPulp3HrefsToContentTypesDeb < ActiveRecord::Migration[6.0]
+  def change
+    add_column Katello::Deb.table_name, :migrated_pulp3_href, :string, :default => nil, :null => true
+  end
+end

data/engines/bastion/app/assets/javascripts/bastion/auth/authorization.service.js

@@ -20,7 +20,7 @@ angular.module('Bastion.auth').service('Authorization', ['CurrentUser', 'Permiss
                 allowedTo = model.permissions[permissionName];
             } else {
                 angular.forEach(Permissions, function (permission) {
-                    if (permission.permission.name === permissionName) {
+                    if (permission.name === permissionName) {
                         allowedTo = true;
                     }
                 });

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/common/views/katello-agent-notice.html

@@ -1,7 +1,7 @@
 <section>
   <p bst-alert="warning" >
     <span translate>
-      Katello-agent is deprecated and will be removed in Katello 4.0.  Consider migrating to Remote Execution.
+      Katello-agent is deprecated and will be removed in a future release.
     </span>
   </p>
 </section>

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/content-hosts/bulk/views/content-hosts-bulk-system-purpose-modal.html

@@ -1,8 +1,7 @@
 <div data-extend-template="components/views/bst-modal.html">
-  <h4 data-block="modal-header" translate>Content Host System Purpose</h4>
+  <h2 data-block="modal-header" translate>Assign System Purpose</h2>
 
   <div data-block="modal-body">
-    <h4 translate>Assign System Purpose:</h4>
 
     <div class="row">
       <div class="col-sm-12">
@@ -10,67 +9,63 @@
       </div>
     </div>
 
+    <div bst-alert="info">
+      <span translate>
+        This change will be applied to <b>{{ hostCount }} systems.</b>
+      </span>
+    </div>
+    <br />
+
     <form name="contentHostContentForm" class="form" ng-hide="content.workingMode">
-      <div>
-        <label translate>Service Level:</label>
+      <div><b>
+        <dl class="dl-horizontal dl-horizontal-left">
+        <dt><label translate>Service Level (SLA):</label></dt>
 
-        <select type="select"
-                ng-options="item for item in defaultServiceLevels"
-                ng-model="selectedServiceLevels">
-        </select>
-      <br /><br />
+        <dd><select type="select"
+                    ng-options="item for item in defaultServiceLevels"
+                    ng-model="selectedServiceLevels">
+        </select></dd>
+        <br /><br />
 
-        <label translate>Usage Type:</label>
+        <dt style="margin-left: -50px;"><label translate>Usage Type:</label></dt>
 
-        <select type="select"
+        <dd><select type="select"
                 ng-options="item for item in defaultUsages"
                 ng-model="selectedRoles">
-        </select>
+        </select></dd>
         <br /><br />
 
-        <label translate>Role:</label>
+        <dt style="margin-left: -98px;"><label translate>Role:</label></dt>
 
-        <select type="select"
+        <dd><select type="select"
                 ng-options="item for item in defaultRoles"
                 ng-model="selectedUsages">
-        </select>
+        </select></dd>
         <br /><br />
 
-        <label ng-hide="!purposeAddonsList().length" translate>Add ons:</label>
-        <div class="help-block" style="text-align:left;">
-          <p translate>ctrl-click or shift-click to select multiple Add ons</p>
-        </div>
+        <dt style="margin-left: -75px;"><label ng-hide="!purposeAddonsList().length" translate>Add ons:</label></dt>
 
-        <select multiple ng-multiple="true"
+        <dd><select multiple ng-multiple="true"
                 ng-hide="!purposeAddonsList().length"
                 ng-options="item for item in purposeAddonsList()"
                 ng-model="selectedAddons">
-        </select>
-        <br /><br />
-
-      </div>
-
-      <div bst-alert="info" ng-show="showConfirm">
-        <span translate>
-          Set System Purpose values on {{ hostCount }} selected content hosts?
-        </span>
-        <div>
-          <br />
-          <button type="button" class="btn btn-primary" ng-click="showConfirm = false; performAction()" translate>Assign</button>
+        </select></dd>
+        <div class="help-block" style="text-align:center; margin-left: -180px;">
+          <p translate>ctrl-click or shift-click to select multiple Add ons</p>
         </div>
+        <br /><br />
+        </dl></b>
       </div>
-
-      <button class="btn btn-primary"
-              type="button"
-              ng-hide="showConfirm"
-              ng-click="showConfirm = true;">
-        <span translate>Assign</span>
-      </button>
-
     </form>
   </div>
 
   <div data-block="modal-footer">
+    <button class="btn btn-primary"
+            type="button"
+            ng-click="performAction()">
+      <span translate>Assign</span>
+    </button>
+
     <button type="button" class="btn btn-default" ng-click="cancel()" translate>
       Cancel
     </button>

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/content-hosts/views/register-client.html

@@ -12,7 +12,7 @@
   <section>
     <p bst-alert="warning" >
     <span translate>
-      Katello-agent is deprecated and will be removed in Katello 4.0.  Consider migrating to Remote Execution.
+      Katello-agent is deprecated and will be removed in a future release.
     </span>
     </p>
   </section>

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/i18n/bastion_katello.pot

@@ -2778,7 +2778,7 @@ msgstr ""
 
 #: app/assets/javascripts/bastion_katello/common/views/katello-agent-notice.html
 #: app/assets/javascripts/bastion_katello/content-hosts/views/register-client.html
-msgid "Katello-agent is deprecated and will be removed in Katello 4.0.  Consider migrating to Remote Execution."
+msgid "Katello-agent is deprecated and will be removed in a future release."
 msgstr ""
 
 #: app/assets/javascripts/bastion_katello/content-views/details/views/content-view-info.html

data/lib/katello/engine.rb

@@ -75,7 +75,7 @@ module Katello
 
     # make sure the Katello plugin is initialized before `after_initialize`
     # hook so that the resumed Dynflow tasks can rely on everything ready.
-    initializer 'katello.register_plugin', :before => :finisher_hook do
+    initializer 'katello.register_plugin', :before => :finisher_hook, :after => 'foreman_remote_execution.register_plugin' do
       ::Foreman::AccessControl::Permission.prepend ::Katello::Concerns::PermissionExtensions
       require 'katello/plugin'
 

data/lib/katello/tasks/check_config.rake

@@ -0,0 +1,18 @@
+namespace :katello do
+  task :check_config => ['environment'] do
+    desc "Task that can be run before a content migration to check that the configuration valid"
+    fail_msg = _("The system appears to already be using pulp3 with all content migrated.")
+
+    puts "Checking for valid Katello configuraton."
+    if SETTINGS[:katello][:use_pulp_2_for_content_type].nil?
+      fail fail_msg
+    end
+
+    if !SETTINGS[:katello][:use_pulp_2_for_content_type][:docker] &&
+      !SETTINGS[:katello][:use_pulp_2_for_content_type][:file] &&
+      !SETTINGS[:katello][:use_pulp_2_for_content_type][:yum] &&
+      !SETTINGS[:katello][:use_pulp_2_for_content_type][:deb]
+      fail fail_msg
+    end
+  end
+end

data/lib/katello/tasks/delete_orphaned_content.rake

@@ -1,9 +1,7 @@
-load "#{Katello::Engine.root}/lib/katello/tasks/common.rake"
-
 namespace :katello do
   desc "Remove orphaned and unneeded content/repos from a smart proxy.\
         Run with SMART_PROXY_ID=1 to run for a single smart proxy."
-  task :delete_orphaned_content => ["environment", "disable_dynflow"] do
+  task :delete_orphaned_content => ["dynflow:client"] do
     User.current = User.anonymous_admin
     smart_proxy_id = ENV['SMART_PROXY_ID']
     if smart_proxy_id

data/lib/katello/tasks/fix_hostgroup_facets.rake

@@ -0,0 +1,8 @@
+namespace :katello do
+  desc "This task collates hostgroup content facts that were missed during the upgrade from audit.\
+        It then updates the hostgroup content_facet accordingly."
+  task :fix_hostgroup_facets => :environment do
+    User.current = User.anonymous_admin
+    ::Katello::Util::HostgroupFacetsHelper.new.main
+  end
+end

data/lib/katello/tasks/pulp3_content_switchover.rake

@@ -3,20 +3,24 @@ require "#{Katello::Engine.root}/app/services/katello/pulp3/migration_switchover
 
 namespace :katello do
   desc "Runs a Pulp 3 migration of pulp3 hrefs to pulp ids for supported content types."
-  task :pulp3_content_switchover => :environment do
+  task :pulp3_content_switchover => ["dynflow:client", "check_config"] do
+    dryrun = ENV['DRYRUN']
+
     begin
       User.current = User.anonymous_admin
 
+      switchover_service = Katello::Pulp3::MigrationSwitchover.new(SmartProxy.pulp_primary)
+      switchover_service.remove_orphaned_content #run out of transaction for easier re-run
       ActiveRecord::Base.transaction do
-        switchover_service = Katello::Pulp3::MigrationSwitchover.new(SmartProxy.pulp_primary)
         switchover_service.run
+        fail "Dryrun completed without error, aborting and rolling back" if dryrun
       end
     rescue Katello::Pulp3::SwitchOverError => e
       $stderr.print(e.message)
       exit 1
     end
 
-    task = ForemanTasks.sync_task(Actions::Pulp3::ContentGuard::RefreshAllDistributions, SmartProxy.pulp_primary)
+    task = ForemanTasks.async_task(Actions::Pulp3::ContentGuard::RefreshAllDistributions, SmartProxy.pulp_primary)
     until !task.pending? || task.paused?
       sleep(10)
       task = ForemanTasks::Task.find(task.id)

data/lib/katello/tasks/pulp3_migration.rake

@@ -1,8 +1,9 @@
-load "#{Katello::Engine.root}/lib/katello/tasks/common.rake"
-
 namespace :katello do
   desc "Runs a Pulp 2 to 3 Content Migration for supported types.  May be run multiple times.  Use wait=false to immediately return with a task url."
-  task :pulp3_migration => ["environment", "disable_dynflow", "check_ping"] do
+  task :pulp3_migration => ["dynflow:client", 'check_config'] do
+    services = [:candlepin, :foreman_tasks, :pulp3, :pulp, :pulp_auth]
+    Katello::Ping.ping!(services: services)
+
     puts "Starting task."
     SmartProxy.pulp_primary.refresh
 
@@ -10,6 +11,7 @@ namespace :katello do
     wait = ::Foreman::Cast.to_bool(ENV['wait'] || 'true')
     preserve_output = ::Foreman::Cast.to_bool(ENV['preserve_output'])
 
+    User.current = User.anonymous_api_admin
     task = ForemanTasks.async_task(Actions::Pulp3::ContentMigration, SmartProxy.pulp_primary, reimport_all: reimport_all)
 
     if wait
@@ -30,6 +32,13 @@ namespace :katello do
         $stderr.print(msg)
         fail ForemanTasks::TaskError, task
       else
+        puts
+        Katello::Pulp3::Migration::CORRUPTABLE_CONTENT_TYPES.each do |type|
+          if type.missing_migrated_content.any?
+            puts "Some corrupted or missing content found, run 'foreman-maintain content migration-stats' for more information."
+            exit(-1)
+          end
+        end
         puts _("Content Migration completed successfully")
       end
     else

data/lib/katello/tasks/pulp3_migration_abort.rake

@@ -1,6 +1,6 @@
 namespace :katello do
   desc "Cancels all running Pulp 2 to 3 migration tasks."
-  task :pulp3_migration_abort => ["environment", "disable_dynflow"] do
+  task :pulp3_migration_abort => ["dynflow:client"] do
     migration_tasks = ForemanTasks::Task::DynflowTask.where(:label => "Actions::Pulp3::ContentMigration").where.not(:state => ["stopped", "paused"])
     cancelled_tasks_count = 0
     migration_tasks.each do |task|