katello 3.18.0.rc2 → 3.18.2

data/app/services/katello/pulp3/migration_plan.rb

@@ -21,13 +21,13 @@ module Katello
       def generate_plugins
         @repository_types.sort.map do |repository_type|
           {
-            type: pulp2_repository_type(repository_type),
+            type: self.class.pulp2_repository_type(repository_type),
             repositories: repository_migrations(repository_type)
           }
         end
       end
 
-      def pulp2_repository_type(repository_type)
+      def self.pulp2_repository_type(repository_type)
         if repository_type == 'yum'
           return 'rpm' #migration plugin uses rpm
         else

data/app/services/katello/pulp3/migration_switchover.rb

@@ -21,6 +21,7 @@ module Katello
           Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_content_types") { combine_duplicate_content_types }
           Katello::Logging.time("CONTENT_SWITCHOVER - combine_duplicate_docker_tags") { combine_duplicate_docker_tags } if docker_migration?
           Katello::Logging.time("CONTENT_SWITCHOVER - migrate_pulp3_hrefs") { migrate_pulp3_hrefs }
+          Katello::Logging.time("CONTENT_SWITCHOVER - remove_missing_content") { remove_missing_content }
         end
       end
 
@@ -106,12 +107,27 @@ module Katello
         end
       end
 
-      def migrated_content_type_check
+      def remove_missing_content
         content_types.each do |content_type|
-          if content_type.model_class == Katello::Erratum
-            migrated_errata_check
-          elsif content_type.model_class.where(migrated_pulp3_href: nil).any?
-            fail SwitchOverError, "ERROR: at least one #{content_type.model_class.table_name} record has migrated_pulp3_href NULL value\n"
+          if Migration::CORRUPTABLE_CONTENT_TYPES.include?(content_type.model_class)
+            content_type.model_class.ignored_missing_migrated_content.destroy_all
+          else
+            content_type.model_class.unmigrated_content.destroy_all
+          end
+        end
+      end
+
+      def migrated_content_type_check
+        content_classes = content_types.map(&:model_class)
+        migrated_errata_check if content_classes.include?(Katello::Erratum)
+
+        (content_classes & Migration::CORRUPTABLE_CONTENT_TYPES).each do |content_type|
+          if content_type.missing_migrated_content.any?
+            fail SwitchOverError, "ERROR: at least one #{content_type.table_name} record has been detected as corrupt or missing. Run 'foreman-rake katello:pulp3_migration_stats' for more information.\n"
+          end
+
+          if content_type.unmigrated_content.any?
+            fail SwitchOverError, "ERROR: at least one #{content_type.table_name} record was not able to be migrated\n"
           end
         end
       end

data/app/services/katello/pulp3/repository.rb

@@ -1,7 +1,7 @@
 require "pulpcore_client"
-
 module Katello
   module Pulp3
+    # rubocop:disable Metrics/ClassLength
     class Repository
       include Katello::Util::HttpProxy
       attr_accessor :repo
@@ -188,8 +188,9 @@ module Katello
         api.repositories_api.read(repository_reference.try(:repository_href))
       end
 
-      def delete(href = repository_reference.try(:repository_href))
-        repository_reference.try(:destroy)
+      def delete_repository(repo_reference = repository_reference)
+        href = repo_reference.try(:repository_href)
+        repo_reference.try(:destroy)
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
@@ -257,8 +258,12 @@ module Katello
         create_version(:base_version => from_repository.version_href)
       end
 
+      def version_zero?
+        repo.version_href.ends_with?('/versions/0/')
+      end
+
       def delete_version
-        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
+        ignore_404_exception { api.repository_versions_api.delete(repo.version_href) } unless version_zero?
       end
 
       def create_version(options = {})
@@ -300,7 +305,7 @@ module Katello
         }
         remote_options[:url] = root.url unless root.url.blank?
         remote_options[:download_concurrency] = root.download_concurrency unless root.download_concurrency.blank?
-        if root.upstream_username && root.upstream_password
+        if !root.upstream_username.blank? && !root.upstream_password.blank?
           remote_options.merge!(username: root.upstream_username,
                                password: root.upstream_password)
         end

data/app/services/katello/pulp3/repository/docker.rb

@@ -52,6 +52,11 @@ module Katello
                                       api.class.recursive_manage_class.new(content_units: options[:remove_content_units]))
         end
 
+        def tag_manifest(name, digest)
+          api.repositories_api.tag(repository_reference.repository_href,
+                                   api.class.tag_image_class.new(tag: name, digest: digest))
+        end
+
         def add_content(content_unit_href)
           content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
           api.repositories_api.add(repository_reference.repository_href, content_units: content_unit_href)

data/app/services/katello/pulp3/repository/yum.rb

@@ -14,7 +14,16 @@ module Katello
           url, sles_token = extract_sles_token
           options = common_remote_options
           options.merge!(sles_auth_token: sles_token) if sles_token
-          options.merge!(url: url, policy: root.download_policy)
+
+          options.merge!(url: url, policy: translated_download_policy)
+        end
+
+        def translated_download_policy
+          if root.download_policy == ::Runcible::Models::YumImporter::DOWNLOAD_BACKGROUND
+            ::Runcible::Models::YumImporter::DOWNLOAD_IMMEDIATE
+          else
+            root.download_policy
+          end
         end
 
         def extract_sles_token
@@ -207,14 +216,20 @@ module Katello
           tasks
         end
 
-        def copy_all(source_repository)
-          data = PulpRpmClient::Copy.new
-          data.config = [{
-            source_repo_version: source_repository.version_href,
-            dest_repo: repository_reference.repository_href
-          }]
+        def copy_all(source_repository, mirror: false)
+          if mirror
+            data = PulpRpmClient::RepositoryAddRemoveContent.new(
+                      base_version: source_repository.version_href)
 
-          [api.copy_api.copy_content(data)]
+            [api.repositories_api.modify(repository_reference.repository_href, data)]
+          else
+            data = PulpRpmClient::Copy.new
+            data.config = [{
+              source_repo_version: source_repository.version_href,
+              dest_repo: repository_reference.repository_href
+            }]
+            [api.copy_api.copy_content(data)]
+          end
         end
 
         def remove_all_content_from_repo(repo_href)

data/app/services/katello/pulp3/task.rb

@@ -68,6 +68,10 @@ module Katello
         task_data[:finish_time] || FINISHED_STATES.include?(task_data[:state])
       end
 
+      def progress_reports
+        task_data['progress_reports']
+      end
+
       def poll
         task_data(true)
         self

data/app/services/katello/pulp3/task_group.rb

@@ -53,6 +53,10 @@ module Katello
         task_group_data['all_tasks_dispatched'] == true && IN_PROGRESS_STATES.all? { |state| task_group_data[state] == 0 }
       end
 
+      def group_progress_reports
+        task_group_data['group_progress_reports']
+      end
+
       def poll
         clear_task_group_data
         task_group_data

data/app/views/katello/api/v2/content_views/show.json.rabl

@@ -4,6 +4,12 @@ extends "katello/api/v2/content_views/base"
 
 attributes :content_host_count
 
+node :errors do
+  unless @resource.valid?
+    attribute :messages => @resource.errors.full_messages
+  end
+end
+
 child :duplicate_repositories_to_publish => :duplicate_repositories_to_publish do
   attributes :id, :name
   node :components do |repo|

data/app/views/katello/layouts/react.html.erb

@@ -10,7 +10,8 @@
     <%= notifications %>
     <div id="organization-id" data-id="<%= Organization.current.id if Organization.current %>" ></div>
     <div id="user-id" data-id="<%= User.current.id if User.current %>" ></div>
-    <div id="reactRoot"></div>
-    <%= mount_react_component('katello', '#reactRoot') %>
+    <div id="reactRoot">
+      <%= react_component('katello') %>
+    </div>
 <% end %>
 <%= render file: "layouts/base" %>

data/app/views/katello/sync_management/_products.html.erb

@@ -42,5 +42,5 @@
     </tbody>
   </table>
 
-  <%= submit_tag _('Synchronize Now'), :class => 'button fr', :id => 'sync_button' %>
+  <%= submit_tag _('Synchronize Now'), :class => 'btn btn-default fr', :id => 'sync_button' %>
 <% end %>

data/app/views/overrides/activation_keys/_host_tab_pane.html.erb

@@ -9,13 +9,9 @@
 
   <%= field(f, _("Activation Keys"),
             :help_inline => _("The value will be available in templates as @host.params['#{kt_ak_label}']")) do
-    tag.div id: :kt_activation_keys
+    react_component('TypeAheadSelect', { id: 'kt_activation_keys', multiple: true, allowNew: true })
   end %>
 
-  <%= mount_react_component('TypeAheadSelect', '#kt_activation_keys',
-                            {id: 'kt_activation_keys', multiple: true, allowNew: true}.to_json,
-                            { flatten_data: true }) %>
-
   <div class="alert alert-info">
     <p><b><%= _('Subscriptions information based on selected activation keys:') %></b></p>
     <ul id="ak-subscriptions-info"></ul>

data/config/routes/api/v2.rb

@@ -113,6 +113,22 @@ Katello::Engine.routes.draw do
           end
         end
 
+        api_resources :content_exports, :only => [] do
+          collection do
+            post :version
+            post :library
+            get :index
+            get :api_status
+          end
+        end
+
+        api_resources :content_export_incrementals, :only => [] do
+          collection do
+            post :version
+            post :library
+          end
+        end
+
         api_resources :content_view_versions, :except => [:create] do
           member do
             post :promote
@@ -121,11 +137,15 @@ Katello::Engine.routes.draw do
             get :available_errata, :controller => :errata
           end
           collection do
-            get :export_histories
-            get :export_api_status
             get :auto_complete_search
             post :incremental_update
-            post :import
+          end
+        end
+
+        api_resources :content_imports, :only => [] do
+          collection do
+            post :version
+            post :library
           end
         end
 

data/db/migrate/20150930183738_migrate_content_hosts.rb

@@ -329,7 +329,7 @@ class MigrateContentHosts < ActiveRecord::Migration[4.2]
   # rubocop:disable Metrics/MethodLength
   # rubocop:disable Metrics/AbcSize
   def up
-    if User.where(:login => User::ANONYMOUS_API_ADMIN).first.nil?
+    unless User.unscoped.where(:login => User::ANONYMOUS_API_ADMIN).exists?
       logger.warn("Foreman anonymous admin does not exist, skipping content host migration.")
       return
     end

data/db/migrate/20200514092553_move_katello_fields_from_hostgroups.katello.rb

@@ -1,6 +1,6 @@
 class MoveKatelloFieldsFromHostgroups < ActiveRecord::Migration[6.0]
   def up
-    if User.where(login: User::ANONYMOUS_ADMIN).exists?
+    if User.unscoped.where(login: User::ANONYMOUS_ADMIN).exists?
       User.as_anonymous_admin do
         copy_data_from_hostgroup
       end
@@ -46,7 +46,10 @@ class MoveKatelloFieldsFromHostgroups < ActiveRecord::Migration[6.0]
         content_facet.kickstart_repository_id = kickstart_repository_id
         content_facet.content_view_id = content_view_id
         content_facet.lifecycle_environment_id = lifecycle_environment_id
-        content_facet.save!
+        unless content_facet.save
+          Rails.logger.warn("Unable to save content facet hostgroup for #{content_facet.inspect} ")
+          Rails.logger.warn(content_facet.errors.full_messages.join("\n"))
+        end
       end
     end
   end

data/db/migrate/20201119211133_pulp3_migration_progress.rb

@@ -0,0 +1,9 @@
+class Pulp3MigrationProgress < ActiveRecord::Migration[6.0]
+  def change
+    create_table :katello_content_migration_progresses do |t|
+      t.string :progress_message
+      t.boolean :canceled, null: false, default: false
+      t.string :task_id, null: false, index: {name: 'katello_content_migration_progress_task_id', unique: true }
+    end
+  end
+end

data/db/migrate/20210201165835_add_migration_missing_content.rb

@@ -0,0 +1,12 @@
+class AddMigrationMissingContent < ActiveRecord::Migration[6.0]
+  def change
+    content_models = [Katello::Rpm, Katello::ModuleStream, Katello::Erratum, Katello::PackageGroup, Katello::YumMetadataFile,
+                      Katello::Srpm, Katello::FileUnit, Katello::DockerManifestList, Katello::DockerManifest, Katello::DockerTag,
+                      Katello::Deb]
+
+    content_models.each do |model|
+      add_column model.table_name, :missing_from_migration, :bool, :default => false, :null => false
+      add_column model.table_name, :ignore_missing_from_migration, :bool, :default => false, :null => false
+    end
+  end
+end

data/engines/bastion/app/assets/javascripts/bastion/auth/authorization.service.js

@@ -20,7 +20,7 @@ angular.module('Bastion.auth').service('Authorization', ['CurrentUser', 'Permiss
                 allowedTo = model.permissions[permissionName];
             } else {
                 angular.forEach(Permissions, function (permission) {
-                    if (permission.permission.name === permissionName) {
+                    if (permission.name === permissionName) {
                         allowedTo = true;
                     }
                 });

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/common/views/katello-agent-notice.html

@@ -1,7 +1,7 @@
 <section>
   <p bst-alert="warning" >
     <span translate>
-      Katello-agent is deprecated and will be removed in Katello 4.0.  Consider migrating to Remote Execution.
+      Katello-agent is deprecated and will be removed in a future release.
     </span>
   </p>
 </section>

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/content-hosts/bulk/views/content-hosts-bulk-system-purpose-modal.html

@@ -1,8 +1,7 @@
 <div data-extend-template="components/views/bst-modal.html">
-  <h4 data-block="modal-header" translate>Content Host System Purpose</h4>
+  <h2 data-block="modal-header" translate>Assign System Purpose</h2>
 
   <div data-block="modal-body">
-    <h4 translate>Assign System Purpose:</h4>
 
     <div class="row">
       <div class="col-sm-12">
@@ -10,67 +9,63 @@
       </div>
     </div>
 
+    <div bst-alert="info">
+      <span translate>
+        This change will be applied to <b>{{ hostCount }} systems.</b>
+      </span>
+    </div>
+    <br />
+
     <form name="contentHostContentForm" class="form" ng-hide="content.workingMode">
-      <div>
-        <label translate>Service Level:</label>
+      <div><b>
+        <dl class="dl-horizontal dl-horizontal-left">
+        <dt><label translate>Service Level (SLA):</label></dt>
 
-        <select type="select"
-                ng-options="item for item in defaultServiceLevels"
-                ng-model="selectedServiceLevels">
-        </select>
-      <br /><br />
+        <dd><select type="select"
+                    ng-options="item for item in defaultServiceLevels"
+                    ng-model="selectedServiceLevels">
+        </select></dd>
+        <br /><br />
 
-        <label translate>Usage Type:</label>
+        <dt style="margin-left: -50px;"><label translate>Usage Type:</label></dt>
 
-        <select type="select"
+        <dd><select type="select"
                 ng-options="item for item in defaultUsages"
                 ng-model="selectedRoles">
-        </select>
+        </select></dd>
         <br /><br />
 
-        <label translate>Role:</label>
+        <dt style="margin-left: -98px;"><label translate>Role:</label></dt>
 
-        <select type="select"
+        <dd><select type="select"
                 ng-options="item for item in defaultRoles"
                 ng-model="selectedUsages">
-        </select>
+        </select></dd>
         <br /><br />
 
-        <label ng-hide="!purposeAddonsList().length" translate>Add ons:</label>
-        <div class="help-block" style="text-align:left;">
-          <p translate>ctrl-click or shift-click to select multiple Add ons</p>
-        </div>
+        <dt style="margin-left: -75px;"><label ng-hide="!purposeAddonsList().length" translate>Add ons:</label></dt>
 
-        <select multiple ng-multiple="true"
+        <dd><select multiple ng-multiple="true"
                 ng-hide="!purposeAddonsList().length"
                 ng-options="item for item in purposeAddonsList()"
                 ng-model="selectedAddons">
-        </select>
-        <br /><br />
-
-      </div>
-
-      <div bst-alert="info" ng-show="showConfirm">
-        <span translate>
-          Set System Purpose values on {{ hostCount }} selected content hosts?
-        </span>
-        <div>
-          <br />
-          <button type="button" class="btn btn-primary" ng-click="showConfirm = false; performAction()" translate>Assign</button>
+        </select></dd>
+        <div class="help-block" style="text-align:center; margin-left: -180px;">
+          <p translate>ctrl-click or shift-click to select multiple Add ons</p>
         </div>
+        <br /><br />
+        </dl></b>
       </div>
-
-      <button class="btn btn-primary"
-              type="button"
-              ng-hide="showConfirm"
-              ng-click="showConfirm = true;">
-        <span translate>Assign</span>
-      </button>
-
     </form>
   </div>
 
   <div data-block="modal-footer">
+    <button class="btn btn-primary"
+            type="button"
+            ng-click="performAction()">
+      <span translate>Assign</span>
+    </button>
+
     <button type="button" class="btn btn-default" ng-click="cancel()" translate>
       Cancel
     </button>