katello 3.18.1 → 3.18.3.1

data/lib/katello/tasks/pulp3_migration_approve_corrupted.rake

@@ -0,0 +1,16 @@
+namespace :katello do
+  desc "Marks corrupted or missing content as approved to be ignored during the migration"
+  task :approve_corrupted_migration_content => ["dynflow:client", "check_ping"] do
+    Katello::Pulp3::Migration::CORRUPTABLE_CONTENT_TYPES.each do |type|
+      type.missing_migrated_content.update_all(:ignore_missing_from_migration => true)
+    end
+    puts "Any missing or corrupt content will be ignored on migration to Pulp 3.  This can be undone with 'foreman-rake katello:unapprove_corrupted_migration_content'"
+  end
+
+  task :unapprove_corrupted_migration_content => ["dynflow:client", "check_ping"] do
+    Katello::Pulp3::Migration::CORRUPTABLE_CONTENT_TYPES.each do |type|
+      type.ignored_missing_migrated_content.update_all(:ignore_missing_from_migration => false)
+    end
+    puts "Resetting approval on any corrupted or missing content, you may want to re-run the 'foreman-maintain content prepare' step to attempt re-migration."
+  end
+end

data/lib/katello/tasks/pulp3_migration_reset.rake

@@ -0,0 +1,26 @@
+namespace :katello do
+  desc "Reset the Pulp 2 -> Pulp 3 migration for content types that haven't been fully switched over"
+  task :pulp3_migration_reset => ["dynflow:client", "check_ping"] do
+    puts "Starting Content Migration Reset."
+    SmartProxy.pulp_primary.refresh
+
+    task = ForemanTasks.async_task(Actions::Pulp3::ContentMigrationReset, SmartProxy.pulp_primary)
+
+    if ENV['wait'].nil? || ::Foreman::Cast.to_bool(ENV['wait'])
+      until !task.pending? || task.paused?
+        sleep(20)
+        task = ForemanTasks::Task.find(task.id)
+      end
+
+      if task.result == 'warning' || task.result == 'pending'
+        msg = _("Content Migration Reset failed, You will want to investigate: https://#{Socket.gethostname}/foreman_tasks/tasks/#{task.id}\n")
+        $stderr.print(msg)
+        fail ForemanTasks::TaskError, task
+      else
+        puts _("Content Migration Reset completed successfully")
+      end
+    else
+      puts "Content Migration Reset started, you may monitor it at: https://#{Socket.gethostname}/foreman_tasks/tasks/#{task.id}"
+    end
+  end
+end

data/lib/katello/tasks/pulp3_migration_stats.rake

@@ -10,7 +10,8 @@ namespace :katello do
     migratable_repo_count = ::Katello::Repository.count - ::Katello::Repository.puppet_type.count -
       ::Katello::Repository.ostree_type.count - ::Katello::Repository.deb_type.count
 
-    on_demand_rpm_count = Katello::RepositoryRpm.where(:repository_id => Katello::Repository.yum_type.on_demand).distinct.count
+    on_demand_rpm_count = Katello::RepositoryRpm.where(:repository_id => Katello::Repository.yum_type.on_demand).
+      select(:rpm_id).distinct.count
     on_demand_unmigrated_rpm_count = on_demand_rpm_count - migrated_rpm_count
     immediate_unmigrated_rpm_count = ::Katello::Rpm.count - migrated_rpm_count - on_demand_unmigrated_rpm_count
 
@@ -23,7 +24,7 @@ namespace :katello do
     hours = (migration_minutes / 60) % 60
     minutes = migration_minutes % 60
 
-    puts
+    puts "============Migration Summary================"
     puts "Migrated/Total RPMs: #{migrated_rpm_count}/#{::Katello::Rpm.count}"
     puts "Migrated/Total errata: #{migrated_erratum_count}/#{::Katello::RepositoryErratum.count}"
     puts "Migrated/Total repositories: #{migrated_repo_count}/#{migratable_repo_count}"
@@ -34,8 +35,41 @@ namespace :katello do
     else
       puts "Estimated migration time based on yum content: fewer than 5 minutes"
     end
+
     puts
-    puts "\e[33mNote:\e[0m ensure there is sufficient storage space for /var/lib/pulp/published to double in size before starting the migration process."
+    puts "\e[33mNote:\e[0m ensure there is sufficient storage space for /var/lib/pulp/published to triple in size before starting the migration process."
     puts "Check the size of /var/lib/pulp/published with 'du -sh /var/lib/pulp/published/'"
+
+    displayed_warning = false
+    found_missing = false
+    path = Dir.mktmpdir('unmigratable_content-')
+    Katello::Pulp3::Migration::CORRUPTABLE_CONTENT_TYPES.each do |type|
+      if type.missing_migrated_content.any?
+        unless displayed_warning
+          displayed_warning = true
+          puts
+          puts "============Missing/Corrupted Content Summary================"
+          puts "WARNING: MISSING OR CORRUPTED CONTENT DETECTED"
+        end
+
+        found_missing = true
+        name = type.name.demodulize
+        puts "Corrupted or Missing #{name}: #{type.missing_migrated_content.count}/#{type.count}"
+
+        File.open(File.join(path, name), 'w') do |file|
+          text = type.missing_migrated_content.map(&:filename).join("\n") + "\n"
+          file.write(text)
+        end
+      end
+    end
+
+    if found_missing
+      puts "Corrupted or missing content has been detected, you can examine the list of content in #{path} and take action by either:"
+      puts "1. Performing a 'Verify Checksum' sync under Advanced Sync Options, let it complete, and re-running the migration"
+      puts "2. Deleting/disabling the affected repositories and running orphan cleanup (foreman-rake katello:delete_orphaned_content) and re-running the migration"
+      puts "3. Manually correcting files on the filesystem in /var/lib/pulp/content/ and re-running the migration"
+      puts "4. Mark currently corrupted or missing content as skipped (foreman-rake katello:approve_corrupted_migration_content).  This will skip migration of missing or corrupted content."
+      puts
+    end
   end
 end

data/lib/katello/tasks/pulp3_post_migration_check.rake

@@ -1,9 +1,7 @@
 require File.expand_path("../engine", File.dirname(__FILE__))
-load "#{Katello::Engine.root}/lib/katello/tasks/common.rake"
-
 namespace :katello do
   desc "Runs a post Pulp3 migration check for supported content types."
-  task :pulp3_post_migration_check => [:environment, :disable_dynflow] do
+  task :pulp3_post_migration_check => ["dynflow:client"] do
     User.current = User.anonymous_admin
     repository_types = Katello::Pulp3::Migration.repository_types_for_migration
 

data/lib/katello/tasks/reimport.rake

@@ -2,7 +2,7 @@ require File.expand_path("../engine", File.dirname(__FILE__))
 
 namespace :katello do
   desc "Runs a katello ping and prints out the statuses of each service"
-  task :check_ping => :environment do
+  task :check_ping => [:environment, "dynflow:client"] do
     ::User.current = ::User.anonymous_admin
     ping_results = Katello::Ping.ping
     if ping_results[:status] != "ok"

data/lib/katello/tasks/reports.rake

@@ -1,4 +1,7 @@
 load "#{Rails.root}/lib/tasks/reports.rake"
 
 #Katello reports can generate a foreman task, so mark the task as a dynflow client
-["reports:daily", "reports:weekly", "reports:monthly"].each { |task| Rake::Task[task].enhance ["dynflow:client"] }
+["reports:daily", "reports:weekly", "reports:monthly"].each do |task|
+  Rake::Task[task].clear
+  Rake::Task[task].enhance ["dynflow:client"]
+end

data/lib/katello/tasks/repository.rake

@@ -1,12 +1,10 @@
-load "#{Katello::Engine.root}/lib/katello/tasks/common.rake"
-
 namespace :katello do
   def commit?
     ENV['COMMIT'] == 'true' || ENV['FOREMAN_UPGRADE'] == '1'
   end
 
   desc "Check for repositories that have not been published since their last sync, and republish if they have."
-  task :publish_unpublished_repositories => ["environment", "disable_dynflow", "check_ping"] do
+  task :publish_unpublished_repositories => ["dynflow:client", "check_ping"] do
     needing_publish = []
     Organization.find_each do |org|
       if org.default_content_view && !org.default_content_view.versions.empty?
@@ -27,7 +25,7 @@ namespace :katello do
   end
 
   desc "Regnerate metadata for all repositories. Specify CONTENT_VIEW=name and LIFECYCLE_ENVIRONMENT=name to narrow repositories."
-  task :regenerate_repo_metadata => ["environment", "disable_dynflow", "check_ping"] do
+  task :regenerate_repo_metadata => ["dynflow:client", "check_ping"] do
     User.current = User.anonymous_api_admin
     repos = lookup_repositories
 
@@ -40,7 +38,7 @@ namespace :katello do
   end
 
   desc "Refresh repository metadata for all repositories. Specify CONTENT_VIEW=name and LIFECYCLE_ENVIRONMENT=name to narrow repositories."
-  task :refresh_pulp_repo_details => ["environment", "disable_dynflow", "check_ping"] do
+  task :refresh_pulp_repo_details => ["dynflow:client", "check_ping"] do
     User.current = User.anonymous_api_admin
     repos = lookup_repositories
 

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "3.18.1".freeze
+  VERSION = "3.18.3.1".freeze
 end

data/webpack/components/TypeAhead/TypeAhead.js

@@ -1,6 +1,7 @@
 import React, { Component } from 'react';
 import Downshift from 'downshift';
 import PropTypes from 'prop-types';
+import { translate as __ } from 'foremanReact/common/I18n';
 
 import TypeAheadSearch from './pf3Search/TypeAheadSearch';
 // eslint-disable-next-line import/no-named-default
@@ -101,7 +102,7 @@ TypeAhead.propTypes = {
 };
 
 TypeAhead.defaultProps = {
-  actionText: 'Search',
+  actionText: __('Search'),
   initialInputValue: '',
   patternfly4: false,
 };

data/webpack/components/TypeAhead/pf3Search/TypeAheadSearch.js

@@ -1,6 +1,7 @@
 import React from 'react';
 import { InputGroup, Button, Icon } from 'patternfly-react';
 import PropTypes from 'prop-types';
+import { translate as __ } from 'foremanReact/common/I18n';
 
 import TypeAheadInput from './TypeAheadInput';
 import TypeAheadItems from './TypeAheadItems';
@@ -46,7 +47,7 @@ TypeAheadSearch.propTypes = {
 };
 
 TypeAheadSearch.defaultProps = {
-  actionText: 'Search',
+  actionText: __('Search'),
 };
 
 export default TypeAheadSearch;

data/webpack/scenes/Subscriptions/Manifest/ManageManifestModal.js

@@ -51,8 +51,11 @@ class ManageManifestModal extends Component {
   showDeleteManifestModal = () =>
     this.props.setModalOpen({ id: DELETE_MANIFEST_MODAL_ID });
 
-  hideDeleteManifestModal = () =>
-    this.props.setModalClosed({ id: DELETE_MANIFEST_MODAL_ID });
+  hideDeleteManifestModal = () => {
+    if (this.props.deleteManifestModalExists) {
+      this.props.setModalClosed({ id: DELETE_MANIFEST_MODAL_ID });
+    }
+  };
 
   updateRepositoryUrl = (event) => {
     this.setState({ redhat_repository_url: event.target.value });
@@ -326,6 +329,7 @@ ManageManifestModal.propTypes = {
   canImportManifest: PropTypes.bool,
   canDeleteManifest: PropTypes.bool,
   isManifestImported: PropTypes.bool,
+  deleteManifestModalExists: PropTypes.bool,
   canEditOrganizations: PropTypes.bool,
   disableManifestActions: PropTypes.bool,
   disabledReason: PropTypes.string,
@@ -351,6 +355,7 @@ ManageManifestModal.defaultProps = {
   canImportManifest: false,
   canDeleteManifest: false,
   isManifestImported: false,
+  deleteManifestModalExists: false,
   canEditOrganizations: false,
   simpleContentAccess: false,
   simpleContentAccessEligible: undefined,

data/webpack/scenes/Subscriptions/Manifest/index.js

@@ -20,6 +20,7 @@ const mapStateToProps = state => ({
   simpleContentAccess: selectSimpleContentAccessEnabled(state),
   isManifestImported: selectIsManifestImported(state),
   modalOpenState: state.foremanModals.ManageManifestModal,
+  deleteManifestModalExists: !!state.foremanModals.deleteManifestModal,
   manifestActionStarted: selectManifestActionStarted(state),
   simpleContentAccessEligible: selectSimpleContentAccessEligible(state),
 });

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: katello
 version: !ruby/object:Gem::Version
-  version: 3.18.1
+  version: 3.18.3.1
 platform: ruby
 authors:
 - N/A
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-06 00:00:00.000000000 Z
+date: 2021-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -87,6 +87,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -94,6 +97,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: foreman_remote_execution
   requirement: !ruby/object:Gem::Requirement
@@ -360,46 +366,40 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.6.2
+        version: 3.10.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.8.0
+        version: 3.11.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.6.2
+        version: 3.10.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.8.0
+        version: 3.11.0
 - !ruby/object:Gem::Dependency
   name: pulp_2to3_migration_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.8.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.6.0
-    - - "!="
-      - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.8.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.6.0
-    - - "!="
-      - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: pulp_certguard_client
   requirement: !ruby/object:Gem::Requirement
@@ -1072,6 +1072,7 @@ files:
 - app/lib/actions/pulp3/content_guard/refresh_all_distributions.rb
 - app/lib/actions/pulp3/content_migration.rb
 - app/lib/actions/pulp3/content_migration_presenter.rb
+- app/lib/actions/pulp3/content_migration_reset.rb
 - app/lib/actions/pulp3/content_view/delete_repository_references.rb
 - app/lib/actions/pulp3/content_view_version/create_exporter.rb
 - app/lib/actions/pulp3/content_view_version/create_importer.rb
@@ -1181,6 +1182,7 @@ files:
 - app/lib/katello/util/docker_manifest_clause_generator.rb
 - app/lib/katello/util/errata.rb
 - app/lib/katello/util/filter_clause_generator.rb
+- app/lib/katello/util/hostgroup_facets_helper.rb
 - app/lib/katello/util/http_proxy.rb
 - app/lib/katello/util/model.rb
 - app/lib/katello/util/module_stream_clause_generator.rb
@@ -2140,6 +2142,8 @@ files:
 - db/migrate/20201012192035_add_metadata_to_katello_content_view_version_export_history.rb
 - db/migrate/20201021150008_add_import_only_to_katello_content_view.rb
 - db/migrate/20201119211133_pulp3_migration_progress.rb
+- db/migrate/20210201165835_add_migration_missing_content.rb
+- db/migrate/20210420140050_add_pulp3_hrefs_to_content_types_deb.rb
 - db/seeds.d/101-locations.rb
 - db/seeds.d/102-organizations.rb
 - db/seeds.d/104-proxy.rb
@@ -4482,11 +4486,12 @@ files:
 - lib/katello/repository_types/puppet.rb
 - lib/katello/repository_types/yum.rb
 - lib/katello/scheduled_jobs.rb
+- lib/katello/tasks/check_config.rake
 - lib/katello/tasks/clean_backend_objects.rake
 - lib/katello/tasks/clean_old_file_repos.rake
 - lib/katello/tasks/clean_published_repo_directories.rake
-- lib/katello/tasks/common.rake
 - lib/katello/tasks/delete_orphaned_content.rake
+- lib/katello/tasks/fix_hostgroup_facets.rake
 - lib/katello/tasks/import_applicability.rake
 - lib/katello/tasks/import_subscriptions.rake
 - lib/katello/tasks/jenkins.rake
@@ -4494,6 +4499,8 @@ files:
 - lib/katello/tasks/pulp3_content_switchover.rake
 - lib/katello/tasks/pulp3_migration.rake
 - lib/katello/tasks/pulp3_migration_abort.rake
+- lib/katello/tasks/pulp3_migration_approve_corrupted.rake
+- lib/katello/tasks/pulp3_migration_reset.rake
 - lib/katello/tasks/pulp3_migration_stats.rake
 - lib/katello/tasks/pulp3_post_migration_check.rake
 - lib/katello/tasks/receptor/extract_orgs.rake
@@ -5059,7 +5066,7 @@ homepage: http://www.katello.org
 licenses:
 - GPL-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -5074,8 +5081,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Content and Subscription Management plugin for Foreman
 test_files: []

data/lib/katello/tasks/common.rake

@@ -1,7 +0,0 @@
-namespace :katello do
-  task :disable_dynflow do
-    #Ensures that we run as a dynflow 'client'
-    ::Rails.application.dynflow.initialize!
-    ForemanTasks.dynflow.config.remote = true
-  end
-end