katello 4.0.3 → 4.1.0.rc1

data/lib/katello/tasks/clean_backend_objects.rake

@@ -4,18 +4,27 @@ namespace :katello do
     class BackendCleaner
       def initialize
         @candlepin_uuids = []
+        @pulp_uuids = []
         @katello_candlepin_uuids = []
+        @katello_pulp_uuids = []
       end
 
       def populate!
         @candlepin_uuids = Katello::Resources::Candlepin::Consumer.all_uuids
         @katello_candlepin_uuids = Katello::Host::SubscriptionFacet.pluck(:uuid).compact
+
+        @pulp_uuids = ::Katello.pulp_server.extensions.consumer.retrieve_all.map { |consumer| consumer['id'] }
+        @katello_pulp_uuids = Katello::Host::ContentFacet.pluck(:uuid).compact
       end
 
       def hosts_with_no_subscriptions
         ::Host.where(:id => Katello::Host::SubscriptionFacet.where(:uuid => @katello_candlepin_uuids - @candlepin_uuids).select(:host_id))
       end
 
+      def hosts_with_no_content
+        ::Host.where(:id => Katello::Host::ContentFacet.where(:uuid => @katello_pulp_uuids - @pulp_uuids).select(:host_id))
+      end
+
       def hosts_with_nil_facets
         nil_sub = Katello::Host::SubscriptionFacet.where(:uuid => nil).select(:host_id).to_sql
         ::Host.where(" id in (#{nil_sub})")
@@ -24,6 +33,10 @@ namespace :katello do
       def cp_orphaned_host_uuids
         @candlepin_uuids - @katello_candlepin_uuids
       end
+
+      def pulp_orphaned_host_uuids
+        @pulp_uuids - @katello_pulp_uuids
+      end
     end
 
     def cleanup_hosts(cleaner)
@@ -36,6 +49,11 @@ namespace :katello do
         print "Host #{host.id} #{host.name} #{host.subscription_facet.try(:uuid)} is partially missing subscription information.  Un-registering\n"
         execute("Failed to delete host") { Katello::RegistrationManager.unregister_host(host, host_unregister_options(host)) }
       end
+
+      cleaner.hosts_with_no_content.each do |host|
+        print "Host #{host.id} #{host.name} #{host.content_facet.try(:uuid)} is partially missing content information.  Un-registering\n"
+        execute("Failed to delete host") { Katello::RegistrationManager.unregister_host(host, host_unregister_options(host)) }
+      end
     end
 
     def clean_backend_orphans(cleaner)
@@ -45,6 +63,13 @@ namespace :katello do
       cp_uuids.each do |consumer_id|
         execute("exception when destroying candlepin consumer #{consumer_id}") { Katello::Resources::Candlepin::Consumer.destroy(consumer_id) }
       end
+
+      pulp_uuids = cleaner.pulp_orphaned_host_uuids
+      print "#{pulp_uuids.count} orphaned consumer id(s) found in pulp.\n"
+      print "Pulp orphaned consumers: #{pulp_uuids}\n"
+      pulp_uuids.each do |consumer_id|
+        execute("exception when destroying pulp consumer #{consumer_id}") { Katello.pulp_server.extensions.consumer.delete(consumer_id) }
+      end
     end
 
     def host_unregister_options(host)

data/lib/katello/tasks/jenkins.rake

@@ -18,6 +18,12 @@ begin
       Rake::Task['jenkins:setup:minitest'].invoke
       Rake::Task['rake:test:katello:test'].invoke
     end
+
+    desc "Runs puplcore integration tests"
+    task 'katello:pulpcore' do
+      Rake::Task['jenkins:setup:minitest'].invoke
+      Rake::Task['rake:test:katello:test:pulpcore'].invoke
+    end
   end
 rescue LoadError
   # ci/reporter/rake/rspec not present, skipping this definition

data/lib/katello/tasks/receptor/extract_orgs.rake

@@ -17,9 +17,7 @@ namespace :katello do
         }
       end
 
-      File.open(output_file, 'w') do |f|
-        f.write(JSON.pretty_generate(data))
-      end
+      File.write(output_file, JSON.pretty_generate(data))
 
       puts "Wrote results to #{output_file}. Please delete it when finished."
     end

data/lib/katello/tasks/reimport.rake

@@ -19,7 +19,6 @@ namespace :katello do
     models = [
       Katello::Subscription,
       Katello::Pool,
-      Katello::ContentViewPuppetEnvironment,
       Katello::Content
     ]
 

data/lib/katello/tasks/repository.rake

@@ -69,21 +69,6 @@ namespace :katello do
     end
   end
 
-  desc "Correct missing pulp repositories for puppet environments. Specify CONTENT_VIEW=name and LIFECYCLE_ENVIRONMENT=name to narrow repositories.  COMMIT=true to perform operation."
-  task :correct_puppet_environments => ["environment", "check_ping"] do
-    puts "All operations will be skipped.  Re-run with COMMIT=true to perform corrections." unless commit?
-
-    User.current = User.anonymous_api_admin
-    puppet_envs = lookup_puppet_environments
-
-    puppet_envs.find_each.with_index do |puppet_env, index|
-      puts "Processing Puppet Environment #{index + 1}/#{puppet_envs.count}: #{puppet_env.pulp_id} (#{puppet_env.id})"
-      unless repo_exists?(puppet_env)
-        handle_missing_puppet_env(puppet_env)
-      end
-    end
-  end
-
   desc "Change the download policy of all repos. Specify DOWNLOAD_POLICY=policy. Options are #{::Runcible::Models::YumImporter::DOWNLOAD_POLICIES.join(', ')}."
   task :change_download_policy => ["environment", "check_ping"] do
     policy = ENV['DOWNLOAD_POLICY']
@@ -126,16 +111,6 @@ namespace :katello do
     repos
   end
 
-  def lookup_puppet_environments
-    lifecycle_envs = Katello::KTEnvironment.where(:name => ENV['LIFECYCLE_ENVIRONMENT']) if ENV['LIFECYCLE_ENVIRONMENT']
-    content_views = Katello::ContentView.where(:name => ENV['CONTENT_VIEW']) if ENV['CONTENT_VIEW']
-
-    repos = ::Katello::ContentViewPuppetEnvironment
-    repos = repos.in_environment(lifecycle_envs) if lifecycle_envs
-    repos = repos.in_content_view(content_views) if content_views
-    repos
-  end
-
   def repo_exists?(repo)
     if SmartProxy.pulp_primary!.pulp3_support?(repo)
       backend_service = repo.backend_service(SmartProxy.pulp_primary!)
@@ -162,9 +137,4 @@ namespace :katello do
   def handle_missing_root_repo(root_repo)
     root_repo.destroy! if commit?
   end
-
-  def handle_missing_puppet_env(puppet_env)
-    puts "Content View Puppet Environment #{puppet_env.id} Missing, Creating."
-    ForemanTasks.sync_task(::Actions::Katello::ContentViewPuppetEnvironment::Create, puppet_env) if commit?
-  end
 end

data/lib/katello/tasks/update_subscription_facet_backend_data.rake

@@ -12,9 +12,9 @@ namespace :katello do
         filename = "subscription_facet_upgrade-#{Time.now.to_i}.log"
         path = File.join(path, filename)
 
-        file = File.open(path, 'w')
-        @errors.each { |error| file.write(error) }
-        file.close
+        File.open(path, 'w') do |file|
+          @errors.each { |error| file.write(error) }
+        end
         $stderr.print "***********************************\n"
         $stderr.print "*************WARNING***************\n"
         $stderr.print "Errors detected during upgrade step.\n"

data/lib/katello/tasks/upgrades/4.0/remove_ostree_puppet_content.rake

@@ -5,13 +5,11 @@ namespace :katello do
       task :remove_ostree_puppet_content => ["environment", "check_ping"] do
         contents = Katello::Content.where(content_type: ['ostree', 'puppet'])
         contents.each do |content|
-          unless content.products.any?(&:redhat?)
-            Katello::Resources::Candlepin::Content.destroy(content.organization.label, content.cp_content_id)
-            content.destroy
-          end
+          Katello::Resources::Candlepin::Content.destroy(content.organization.label, content.cp_content_id)
         rescue RestClient::NotFound
           #skip content not found
         end
+        contents.destroy_all
       end
     end
   end

data/lib/katello/tasks/upgrades/4.1/fix_invalid_pools.rake

@@ -0,0 +1,62 @@
+namespace :katello do
+  namespace :upgrades do
+    namespace '4.1' do
+      desc "Removed orphaned pools and correct org+subscription mismatch"
+      task :fix_invalid_pools => ["environment", "check_ping"] do
+        logger = Logger.new(STDOUT)
+        invalid_pools = Katello::Pool.all.select(&:invalid?)
+
+        # Make sure Subscriptions are up to date for any org that has invalid pools
+        org_ids = invalid_pools.map(&:organization_id).uniq.compact
+        orgs = ::Organization.where(id: org_ids)
+        orgs.each do |org|
+          Katello::Subscription.import_all(org)
+        end
+
+        orphaned_pools = []
+
+        invalid_pools.each do |pool|
+          pool_json = Katello::Resources::Candlepin::Pool.find(pool.cp_id)
+
+          correct_org = Organization.find_by(:label => pool_json['owner']['key'])
+          unless correct_org
+            orphaned_pools << pool
+            logger.info("Pool id=#{pool.id} cp_id=#{pool.cp_id} will be removed because its organization does not exist")
+            next
+          end
+
+          # authoritatively set the org id for this pool
+          pool.organization = correct_org
+
+          subscription = Katello::Pool.determine_subscription(
+            product_id: pool_json['productId'],
+            source_stack_id: pool_json['sourceStackId'],
+            organization: correct_org
+          )
+
+          unless subscription
+            orphaned_pools << pool
+            logger.info("Pool id=#{pool.id} cp_id=#{pool.cp_id} will be removed because it has no matching subscription")
+            next
+          end
+
+          pool.subscription = subscription
+
+          # the matching subscription should have been imported above
+          # with the right org assigned, import the pool and reindex associated hosts+activationkeys
+          pool.import_data(true)
+          logger.info("Pool id=#{pool.id} cp_id=#{pool.cp_id} has been reimported")
+        rescue ::Katello::Errors::CandlepinPoolGone
+          logger.info("Pool id=#{pool.id} cp_id=#{pool.cp_id} will be removed because it's not in Candlepin")
+          orphaned_pools << pool
+        end
+
+        imported_count = invalid_pools.count - orphaned_pools.count
+        logger.info("Corrected #{imported_count} invalid pools")
+
+        orphaned_pools.each(&:destroy)
+        logger.info("Removed #{orphaned_pools.count} orphaned pools")
+      end
+    end
+  end
+end

data/lib/katello/tasks/upgrades/4.1/sync_noarch_content.rake

@@ -0,0 +1,15 @@
+namespace :katello do
+  namespace :upgrades do
+    namespace '4.1' do
+      desc "Synchronize arches field of content in candlepin which should be unset."
+      task :sync_noarch_content => ['environment'] do
+        roots = Katello::RootRepository.joins(:product).merge(Katello::Product.custom).where(arch: 'noarch')
+        roots.each do |root|
+          Katello::Resources::Candlepin::Content.update(root.library_instance.organization.label, id: root.content_id, arches: '')
+        rescue RestClient::NotFound
+          #skip content not found
+        end
+      end
+    end
+  end
+end

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "4.0.3".freeze
+  VERSION = "4.1.0.rc1".freeze
 end

data/lib/katello.rb

@@ -28,6 +28,6 @@ module Katello
   require "katello/engine"
 
   def self.pulp_server
-    Katello::Pulp::Server.config(::SmartProxy.pulp_primary.url + '/pulp/api/v2/', User.remote_user)
+    Katello::Pulp::Server.config(SETTINGS[:katello][:pulp][:url], User.remote_user)
   end
 end

data/locale/action_names.rb

@@ -1,73 +1,75 @@
 # Autogenerated!
-_("Generate repository applicability")
-_("Synchronize smart proxy")
-_("Sync capsule")
-_("Delete")
+_("Synchronize")
+_("Remove Content")
+_("Instance update")
+_("Index package groups")
+_("Index module streams")
+_("Index errata")
+_("Index content")
+_("Upload into")
+_("Filtered index content")
+_("Fetch pxe files")
+_("Export")
 _("Errata mail")
-_("Incremental Update of  Content View Version(s) ")
-_("Update")
-_("Publish")
+_("Discover")
+_("Delete Package Group")
+_("Delete")
 _("Create")
-_("Import Default Content View")
-_("Incremental Update")
-_("Export")
-_("Import Content View Version")
+_("Update")
+_("Update http proxy")
+_("Reindex subscriptions")
+_("Delete Product")
+_("Product Create")
+_("Destroy")
+_("Combined Profile Update")
+_("Package Profile Update")
+_("Updating System Purpose for host")
+_("Update release version for host")
+_("Update for host")
+_("Update Content Overrides")
+_("Remove subscriptions")
+_("Hypervisors update")
+_("Hypervisors")
+_("Install Applicable Errata")
+_("Destroy Content Host")
+_("Auto attach subscriptions")
+_("Attach subscriptions")
+_("Publish Lifecycle Environment Repositories")
 _("Delete Lifecycle Environment")
-_("Promote")
 _("Republish Version Repositories")
-_("Publish Lifecycle Environment Repositories")
-_("Attach subscriptions")
-_("Promotion to Environment")
-_("Auto attach subscriptions")
-_("Destroy Content Host")
-_("Remove from Environment")
+_("Incremental Update")
+_("Import Default Content View")
+_("Import Content View Version")
 _("Remove Version")
-_("Install Applicable Errata")
-_("Hypervisors")
-_("Hypervisors update")
+_("Remove from Environment")
 _("Remove Versions and Associations")
-_("Update Content Overrides")
-_("Update for host")
-_("Update release version for host")
-_("Updating System Purpose for host")
-_("Remove subscriptions")
-_("Combined Profile Update")
-_("Destroy")
-_("Product Create")
-_("Delete Product")
-_("Reindex subscriptions")
-_("Update http proxy")
-_("Delete Package Group")
-_("Discover")
-_("Fetch pxe files")
-_("Upload into")
-_("Filtered index content")
-_("Index content")
-_("Index errata")
-_("Index module streams")
-_("Index package groups")
-_("Instance update")
-_("Remove Content")
-_("Synchronize")
-_("Update content urls")
-_("Update http proxy details")
-_("Update redhat repository")
-_("Upload errata into")
-_("Create Package Group")
-_("Verify checksum")
-_("Disable")
-_("Enable")
-_("Run Sync Plan:")
-_("Package Profile Update")
-_("Copy version units to library")
-_("Export Library")
-_("Import")
+_("Publish")
+_("Promotion to Environment")
+_("Promote")
 _("Remote action:")
-_("Import Puppet classes")
-_("Action with sub plans")
-_("Abstract async task")
-_("Delete Activation Key")
-_("Agent action")
+_("Incremental Update of  Content View Version(s) ")
+_("Sync capsule")
+_("Synchronize smart proxy")
+_("Generate repository applicability")
+_("Bulk generate applicability for hosts")
 _("Generate host applicability")
+_("Agent action")
+_("Delete Activation Key")
+_("Abstract async task")
+_("Import Puppet classes")
 _("Import facts")
-_("Bulk generate applicability for hosts")
+_("Action with sub plans")
+_("Import")
+_("Export Library")
+_("Copy version units to library")
+_("Create Import History")
+_("Create Export History")
+_("Run Sync Plan:")
+_("Create Package Group")
+_("Disable")
+_("Verify checksum")
+_("Enable")
+_("Upload errata into")
+_("Update redhat repository")
+_("Update http proxy details")
+_("Update content urls")