katello 3.14.0.rc1 → 3.14.0.rc2

data/app/services/katello/event_daemon.rb

@@ -0,0 +1,91 @@
+module Katello
+  class EventDaemon
+    class << self
+      def initialize
+        FileUtils.touch(lock_file)
+        @cache = ActiveSupport::Cache::MemoryStore.new
+      end
+
+      def settings
+        SETTINGS[:katello][:event_daemon]
+      end
+
+      def pid
+        return unless pid_file && File.exist?(pid_file)
+
+        File.open(pid_file) { |f| f.read.to_i }
+      end
+
+      def pid_file
+        pid_dir.join('katello_event_daemon.pid')
+      end
+
+      def pid_dir
+        Rails.root.join('tmp', 'pids')
+      end
+
+      def lock_file
+        Rails.root.join('tmp', 'katello_event_daemon.lock')
+      end
+
+      def write_pid_file
+        return unless pid_file
+
+        FileUtils.mkdir_p(pid_dir)
+        File.open(pid_file, 'w') { |f| f.puts Process.pid }
+      end
+
+      def stop
+        return unless pid == Process.pid
+        services.values.each(&:close)
+        File.unlink(pid_file) if pid_file && File.exist?(pid_file)
+      end
+
+      def start
+        return unless runnable?
+
+        lockfile = File.open(lock_file, 'r')
+        begin
+          lockfile.flock(File::LOCK_EX)
+          return if started? # ensure it wasn't started while we waited for the lock
+
+          start_services
+          write_pid_file
+
+          at_exit do
+            stop
+          end
+
+          Rails.logger.info("Katello event daemon started process=#{Process.pid}")
+        ensure
+          lockfile.flock(File::LOCK_UN)
+        end
+      end
+
+      def started?
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH, TypeError # process no longer exists or we had no PID cached
+        false
+      end
+
+      def start_services
+        services.values.each(&:run)
+      end
+
+      def runnable?
+        # avoid accessing the disk on each request
+        @cache.fetch('katello_event_daemon_runnable', expires_in: 1.minute) do
+          !started? && settings[:enabled] && !::Foreman.in_rake? && !Rails.env.test?
+        end
+      end
+
+      def services
+        {
+          candlepin_events: ::Katello::CandlepinEventListener,
+          katello_events: ::Katello::EventMonitor::PollerThread
+        }
+      end
+    end
+  end
+end

data/app/services/katello/event_monitor/poller_thread.rb

@@ -0,0 +1,108 @@
+module Katello
+  module EventMonitor
+    class PollerThread
+      SLEEP_INTERVAL = 3
+      PROCESSED_COUNT_CACHE_KEY = 'katello_events_processed'.freeze
+      FAILED_COUNT_CACHE_KEY = 'katello_events_failed'.freeze
+
+      cattr_accessor :instance
+
+      def self.initialize(logger = nil)
+        self.instance ||= self.new(logger)
+      end
+
+      def self.close
+        if self.instance
+          self.instance.close
+          self.instance = nil
+        end
+        reset_status
+      end
+
+      def self.run
+        initialize
+        ::Katello::EventQueue.reset_in_progress
+        instance.poll_for_events
+      end
+
+      def self.status
+        {
+          processed_count: Rails.cache.fetch(PROCESSED_COUNT_CACHE_KEY) { 0 },
+          failed_count: Rails.cache.fetch(FAILED_COUNT_CACHE_KEY) { 0 },
+          queue_depth: Katello::EventQueue.queue_depth
+        }
+      end
+
+      def self.reset_status
+        Rails.cache.write(PROCESSED_COUNT_CACHE_KEY, 0)
+        Rails.cache.write(FAILED_COUNT_CACHE_KEY, 0)
+      end
+
+      def initialize(logger = nil)
+        @logger = logger || ::Foreman::Logging.logger('katello/katello_events')
+        @failed_count = 0
+        @processed_count = 0
+      end
+
+      def close
+        @logger.info("Stopping Katello Event Monitor")
+        @thread.kill if @thread
+      end
+
+      def run_event(event)
+        @logger.debug("event_queue_event: type=#{event.event_type}, object_id=#{event.object_id}")
+
+        event_instance = nil
+        begin
+          ::User.as_anonymous_admin do
+            event_instance = ::Katello::EventQueue.create_instance(event)
+            event_instance.run
+          end
+        rescue => e
+          @failed_count += 1
+          Rails.cache.write(FAILED_COUNT_CACHE_KEY, @failed_count, expires_in: 24.hours)
+
+          @logger.error("event_queue_error: type=#{event.event_type}, object_id=#{event.object_id}")
+          @logger.error(e.message)
+          @logger.error(e.backtrace.join("\n"))
+        ensure
+          if event_instance.try(:retry)
+            result = ::Katello::EventQueue.reschedule_event(event)
+            if result == :expired
+              @logger.warn("event_queue_event_expired: type=#{event.event_type} object_id=#{event.object_id}")
+            elsif !result.nil?
+              @logger.warn("event_queue_rescheduled: type=#{event.event_type} object_id=#{event.object_id}")
+            end
+          end
+          ::Katello::EventQueue.clear_events(event.event_type, event.object_id, event.created_at)
+        end
+      end
+
+      def poll_for_events
+        @thread.kill if @thread
+        @thread = Thread.new do
+          begin
+            @logger.info("Polling Katello Event Queue")
+            loop do
+              Rails.application.executor.wrap do
+                Katello::Util::Support.with_db_connection(@logger) do
+                  until (event = ::Katello::EventQueue.next_event).nil?
+                    run_event(event)
+                    @processed_count += 1
+                    Rails.cache.write(PROCESSED_COUNT_CACHE_KEY, @processed_count, expires_in: 24.hours)
+                  end
+                end
+              end
+
+              sleep SLEEP_INTERVAL
+            end
+          rescue => e
+            @logger.error(e.message)
+            @logger.error("Fatal error in Katello Event Monitor")
+            self.class.close
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/katello/event_queue.rb

@@ -12,6 +12,10 @@ module Katello
       end
     end
 
+    def self.queue_depth
+      ::Katello::Event.all.size
+    end
+
     def self.runnable_events
       Katello::Event.where(process_after: nil).or(Katello::Event.where(process_after: Date.new..Time.zone.now))
     end

data/app/services/katello/pulp/repository.rb

@@ -11,8 +11,10 @@ module Katello
       end
 
       def backend_data(force = false)
-        return smart_proxy.pulp_api.extensions.repository.retrieve_with_details(repo.pulp_id) if (repo.pulp_id && force)
-        @backend_data ||= smart_proxy.pulp_api.extensions.repository.retrieve_with_details(repo.pulp_id) if repo.pulp_id
+        if (repo.pulp_id && (force || @backend_data.nil?))
+          @backend_data = smart_proxy.pulp_api.extensions.repository.retrieve_with_details(repo.pulp_id)
+        end
+        @backend_data
       rescue RestClient::ResourceNotFound
         nil
       end
@@ -84,6 +86,10 @@ module Katello
       end
 
       def mirror_needs_updates?
+        needs_update?
+      end
+
+      def needs_update?
         needs_importer_updates? || needs_distributor_updates?
       end
 
@@ -195,8 +201,8 @@ module Katello
         distributor['id']
       end
 
-      def proxy_host_importer_value
-        root.ignore_global_proxy ? "" : nil
+      def refresh_if_needed
+        refresh if needs_update?
       end
 
       def refresh
@@ -259,10 +265,11 @@ module Katello
         tasks
       end
 
-      def copy_units(destination_repo, units, override_config = {})
+      def copy_units(destination_repo, units, incremental_update: false)
+        override_config = {}
+        override_config = build_override_config(destination_repo, incremental_update: incremental_update) if respond_to? :build_override_config
         content_type = units.first.class::CONTENT_TYPE
         unit_ids = units.pluck(:pulp_id)
-
         smart_proxy.pulp_api.extensions.send(content_type).copy(repo.pulp_id, destination_repo.pulp_id, ids: unit_ids, override_config: override_config)
       end
 

data/app/services/katello/pulp/repository/yum.rb

@@ -75,11 +75,45 @@ module Katello
           smart_proxy.pulp_api.extensions.repository.regenerate_applicability_by_ids([repo.pulp_id], true)
         end
 
-        def copy_contents(destination_repo, options = {})
-          override_config = Katello::Repository.build_override_config(options)
-          rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(options[:filters]&.yum(false), options[:rpm_filenames])
-          tasks = [smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id,
-                   rpm_copy_clauses.merge(:override_config => override_config))]
+        def generate_mapping(destination_repo)
+          source_repo_map = {}
+          repo.siblings.yum_type.each do |sibling|
+            key = sibling.library_instance? ? sibling.id : sibling.library_instance_id
+            source_repo_map[key] = sibling.pulp_id
+          end
+
+          Hash[destination_repo.siblings.yum_type.map { |sibling| [source_repo_map[sibling.library_instance_id], sibling.pulp_id] }]
+        end
+
+        def build_override_config(destination_repo, incremental_update: false,
+                                                    solve_dependencies: false,
+                                                    filters: [])
+          config = {}
+          if incremental_update ||
+              (filters.present? && solve_dependencies)
+            if Setting[:dependency_solving_algorithm] == 'greedy'
+              config[:recursive] = true
+            else
+              config[:recursive_conservative] = true
+            end
+            config[:additional_repos] = generate_mapping(destination_repo)
+          end
+          config
+        end
+
+        def copy_contents(destination_repo, filters: nil,
+                                            solve_dependencies: false,
+                                            rpm_filenames: [])
+          rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(filters&.yum(false), rpm_filenames)
+          tasks = []
+          if rpm_copy_clauses
+            override_config = build_override_config(destination_repo,
+                                                    filters: filters,
+                                                    solve_dependencies: solve_dependencies)
+
+            tasks << smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id,
+                      rpm_copy_clauses.merge(:override_config => override_config))
+          end
 
           if rpm_remove_clauses
             tasks << smart_proxy.pulp_api.extensions.repository.unassociate_units(destination_repo.pulp_id,
@@ -87,7 +121,10 @@ module Katello
                                                                           filters: {unit: rpm_remove_clauses})
           end
 
-          tasks.concat(copy_module_contents(destination_repo, options[:filters], override_config))
+          tasks.concat(copy_module_contents(destination_repo,
+                                              filters: filters,
+                                              solve_dependencies: solve_dependencies))
+
           [:srpm, :errata, :package_group, :package_environment,
            :yum_repo_metadata_file, :distribution, :module_default].each do |type|
             tasks << smart_proxy.pulp_api.extensions.send(type).copy(repo.pulp_id, destination_repo.pulp_id)
@@ -155,10 +192,12 @@ module Katello
           smart_proxy.pulp_api.extensions.repository.unassociate_units(repo.pulp_id, :filters => criteria)
         end
 
-        def copy_module_contents(destination_repo, filters, override_config)
-          if filters
-            filters = filters.module_stream.or(filters.errata)
-          end
+        def copy_module_contents(destination_repo, filters:, solve_dependencies:)
+          override_config = build_override_config(destination_repo,
+                                                  filters: filters,
+                                                  solve_dependencies: solve_dependencies)
+
+          filters = filters.module_stream.or(filters.errata) if filters
 
           copy_clauses, remove_clauses = generate_module_stream_copy_clauses(filters)
           tasks = [smart_proxy.pulp_api.extensions.module.copy(repo.pulp_id, destination_repo.pulp_id,
@@ -203,11 +242,12 @@ module Katello
             remove = clause_gen.remove_clause
             remove_clauses = {filters: {unit: remove}} if remove
           else
-            copy_clauses = {filters: {unit: ContentViewPackageFilter.generate_rpm_clauses(::Katello::Rpm.in_repositories(repo).non_modular.pluck(:filename))}}
+            non_modular_rpms = ::Katello::Rpm.in_repositories(repo).non_modular.pluck(:filename)
+            copy_clauses = non_modular_rpms.blank? ? nil : {filters: {unit: ContentViewPackageFilter.generate_rpm_clauses(non_modular_rpms)}}
             remove_clauses = nil
           end
 
-          copy_clauses.merge!(fields: ::Katello::Pulp::Rpm::PULP_SELECT_FIELDS)
+          copy_clauses.merge!(fields: ::Katello::Pulp::Rpm::PULP_SELECT_FIELDS) if copy_clauses
           [copy_clauses, remove_clauses]
         end
       end

data/lib/katello/engine.rb

@@ -4,13 +4,16 @@ module Katello
   class Engine < ::Rails::Engine
     isolate_namespace Katello
 
-    initializer 'katello.selective_params_parser', :before => :build_middleware_stack do |app|
+    initializer 'katello.middleware', :before => :build_middleware_stack do |app|
       require 'katello/prevent_json_parsing'
       app.middleware.insert_after(
         Rack::MethodOverride,
         Katello::PreventJsonParsing,
         ->(env) { env['PATH_INFO'] =~ /consumers/ && env['PATH_INFO'] =~ /profile|packages/ }
       )
+
+      require 'katello/middleware/event_daemon'
+      app.middleware.use(Katello::Middleware::EventDaemon)
     end
 
     initializer 'katello.mount_engine', :before => :sooner_routes_load, :after => :build_middleware_stack do |app|
@@ -26,6 +29,9 @@ module Katello
         :redhat_repository_url => 'https://cdn.redhat.com',
         :consumer_cert_rpm => 'katello-ca-consumer-latest.noarch.rpm',
         :consumer_cert_sh => 'katello-rhsm-consumer',
+        :event_daemon => {
+          enabled: true
+        },
         :pulp => {
           :default_login => 'admin',
           :url => 'https://localhost/pulp/api/v2/',
@@ -85,15 +91,6 @@ module Katello
       end
     end
 
-    initializer "katello.initialize_cp_listener", :before => :finisher_hook do
-      unless Rails.env.test?
-        ForemanTasks.dynflow.config.post_executor_init do |world|
-          ::Actions::Candlepin::ListenOnCandlepinEvents.ensure_running(world)
-          ::Actions::Katello::EventQueue::Monitor.ensure_running(world)
-        end
-      end
-    end
-
     initializer "katello.load_app_instance_data" do |app|
       app.config.filter_parameters += [:_json] #package profile parameter
       Katello::Engine.paths['db/migrate'].existent.each do |path|
@@ -196,10 +193,6 @@ module Katello
       ::Api::V2::HostgroupsController.send :include, Katello::Concerns::Api::V2::HostgroupsControllerExtensions
       ::Api::V2::SmartProxiesController.send :include, Katello::Concerns::Api::V2::SmartProxiesControllerExtensions
 
-      Katello::EventQueue.register_event(Katello::Events::ImportHostApplicability::EVENT_TYPE, Katello::Events::ImportHostApplicability)
-      Katello::EventQueue.register_event(Katello::Events::ImportPool::EVENT_TYPE, Katello::Events::ImportPool)
-      Katello::EventQueue.register_event(Katello::Events::AutoPublishCompositeView::EVENT_TYPE, Katello::Events::AutoPublishCompositeView)
-
       ::HostsController.class_eval do
         helper Katello::Concerns::HostsAndHostgroupsHelperExtensions
       end
@@ -212,6 +205,12 @@ module Katello
 
       load 'katello/repository_types.rb'
       load 'katello/scheduled_jobs.rb'
+
+      Katello::EventQueue.register_event(Katello::Events::ImportHostApplicability::EVENT_TYPE, Katello::Events::ImportHostApplicability)
+      Katello::EventQueue.register_event(Katello::Events::ImportPool::EVENT_TYPE, Katello::Events::ImportPool)
+      Katello::EventQueue.register_event(Katello::Events::AutoPublishCompositeView::EVENT_TYPE, Katello::Events::AutoPublishCompositeView)
+
+      Katello::EventDaemon.initialize
     end
 
     rake_tasks do

data/lib/katello/middleware/event_daemon.rb

@@ -0,0 +1,14 @@
+module Katello
+  module Middleware
+    class EventDaemon
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        Katello::EventDaemon.start
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/katello/plugin.rb

@@ -236,6 +236,8 @@ Foreman::Plugin.register :katello do
   logger :action, :enabled => true
   logger :manifest_import_logger, :enabled => true
   logger :registry_proxy, :enabled => true
+  logger :katello_events, :enabled => true
+  logger :candlepin_events, :enabled => true
 
   widget 'errata_widget', :name => 'Latest Errata', :sizey => 1, :sizex => 6
   widget 'content_views_widget', :name => 'Content Views', :sizey => 1, :sizex => 6

data/lib/katello/tasks/upgrade_check.rake

@@ -1,18 +1,13 @@
 namespace :katello do
   task :upgrade_check => ['environment'] do
     desc "Task that can be run before upgrading Katello to check if system is upgrade ready"
-    CP_LISTEN_ACTION = Actions::Candlepin::ListenOnCandlepinEvents.to_s
-    EVENT_QUEUE_ACTION = Actions::Katello::EventQueue::Monitor.to_s
-
-    success = "PASS"
-    fail = "FAIL"
 
     puts "This script makes no modifications and can be re-run multiple times for the most up to date results."
     puts "Checking upgradeability...\n\n"
 
     # check for any running tasks
-    task_count = ::ForemanTasks::Task.active.where("label NOT IN (?)", [CP_LISTEN_ACTION, EVENT_QUEUE_ACTION]).where("state != 'scheduled'").count
-    task_status = task_count > 0 ? fail : success
+    task_count = ::ForemanTasks::Task.active.where("state != 'scheduled'").count
+    task_status = task_count > 0 ? "FAIL" : "SUCCESS"
     puts "Checking for running tasks..."
     puts "[#{task_status}] - There are #{task_count} active tasks. "
     if task_count > 0