katello 4.0.0.rc1 → 4.0.0.rc2

data/app/lib/katello/agent/remove_package_message.rb

@@ -0,0 +1,28 @@
+module Katello
+  module Agent
+    class RemovePackageMessage < BaseMessage
+      def initialize(content:, consumer_id:)
+        @packages = content
+        @consumer_id = consumer_id
+        @content_type = 'rpm'
+        @method = 'uninstall'
+      end
+
+      protected
+
+      def units
+        @packages.map do |package|
+          nvra = ::Katello::Util::Package.parse_nvrea_nvre(package)
+          unit_key = nvra || {
+            name: package
+          }
+
+          {
+            type_id: @content_type,
+            unit_key: unit_key
+          }
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/agent/update_package_message.rb

@@ -0,0 +1,25 @@
+module Katello
+  module Agent
+    class UpdatePackageMessage < BaseMessage
+      def initialize(content:, consumer_id:)
+        @packages = content
+        @consumer_id = consumer_id
+        @content_type = 'rpm'
+        @method = 'update'
+      end
+
+      protected
+
+      def units
+        @packages.map do |package|
+          {
+            type_id: @content_type,
+            unit_key: {
+              name: package
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/event_daemon/services/agent_event_receiver.rb

@@ -0,0 +1,62 @@
+module Katello
+  module EventDaemon
+    module Services
+      class AgentEventReceiver
+        STATUS_CACHE_KEY = 'katello_agent_events'.freeze
+
+        class Handler
+          attr_accessor :processed, :failed
+
+          def initialize
+            @processed = 0
+            @failed = 0
+          end
+
+          def handle(message)
+            ::Katello::Util::Support.with_db_connection do
+              ::Katello::Agent::ClientMessageHandler.new(message).handle
+              @processed += 1
+            rescue => e
+              @failed += 1
+              Rails.logger.error("Error handling Katello Agent client message")
+              Rails.logger.error(e.message)
+              Rails.logger.error(e.backtrace.join("\n"))
+            end
+          end
+        end
+
+        def self.logger
+          ::Foreman::Logging.logger('katello/agent')
+        end
+
+        def self.run
+          fail("Katello agent event receiver already started") if running?
+
+          @thread = Thread.new do
+            @handler = Handler.new
+            agent_connection = ::Katello::Agent::Connection.new
+            agent_connection.fetch_agent_messages(@handler)
+          end
+        end
+
+        def self.close
+          @thread&.kill
+        end
+
+        def self.running?
+          @thread&.status.present?
+        end
+
+        def self.status(refresh: true)
+          Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
+            {
+              running: running?,
+              processed_count: @handler&.processed || 0,
+              failed_count: @handler&.failed || 0
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/messaging/received_message.rb

@@ -3,7 +3,7 @@ module Katello
     class ReceivedMessage
       attr_reader :headers, :body
 
-      def initialize(body:, headers:)
+      def initialize(body:, headers: {})
         @body = body
         @headers = headers
       end

data/app/lib/katello/qpid/connection.rb

@@ -0,0 +1,130 @@
+require 'qpid_proton'
+
+module Katello
+  module Qpid
+    class Connection
+      class Sender < ::Qpid::Proton::MessagingHandler
+        def initialize(url, address, messages)
+          super()
+          @url = url
+          @address = address
+          @messages = messages
+          @sent = 0
+          @confirmed = 0
+        end
+
+        def on_container_start(container)
+          c = container.connect(@url)
+          c.open_sender
+          @receiver = c.open_receiver(@address) if @address
+        end
+
+        def on_sendable(sender)
+          @messages.each do |msg|
+            msg.reply_to = @receiver.remote_source.address if @receiver
+            sender.send(msg)
+            @sent += 1
+          end
+        ensure
+          sender.close
+        end
+
+        def on_tracker_accept(tracker)
+          @confirmed += 1
+          if @confirmed == @sent
+            tracker.connection.close
+          end
+        end
+
+        def on_message(_delivery, message)
+          opcode = message.properties['qmf.opcode']
+          if opcode == '_exception'
+            error_code = message.body.dig('_values', 'error_code')
+            if error_code != '7' # not found
+              error_message = message.body.dig('_values', 'error_text')
+              fail(error_message)
+            end
+          end
+        end
+      end
+
+      class Receiver < ::Qpid::Proton::MessagingHandler
+        def initialize(url, address, handler)
+          super()
+          @url = url
+          @address = address
+          @handler = handler
+        end
+
+        def on_container_start(container)
+          c = container.connect(@url,
+            idle_timeout: 4
+          )
+          c.open_receiver(@address)
+        end
+
+        def on_message(_delivery, message)
+          received = Katello::Messaging::ReceivedMessage.new(body: message.body)
+          @handler.handle(received)
+        end
+      end
+
+      def initialize(url)
+        @url = url
+      end
+
+      def delete_queue(queue_name)
+        address = "qmf.default.direct"
+        message = ::Qpid::Proton::Message.new
+        message.subject = 'broker'
+        message.address = address
+        message.body = {
+          '_object_id' => {
+            '_object_name' => 'org.apache.qpid.broker:broker:amqp-broker'
+          },
+          '_method_name' => 'delete',
+          '_arguments' => {
+            'strict' => true,
+            'name' => queue_name,
+            'type' => 'queue',
+            'properties' => {}
+          }
+        }
+
+        message.properties = {
+          'qmf.opcode' => '_method_request',
+          'x-amqp-0-10.app-id' => 'qmf2',
+          'method' => 'request'
+        }
+
+        sender = Sender.new(@url, address, [message])
+        with_connection(sender)
+      end
+
+      def send_messages(messages)
+        qpid_messages = messages.map do |message|
+          msg = ::Qpid::Proton::Message.new
+          msg.body = message.to_s
+          msg.address = message.recipient_address
+          msg
+        end
+        sender = Sender.new(@url, nil, qpid_messages)
+        with_connection(sender)
+      end
+
+      def receive_messages(address:, handler:)
+        receiver = Receiver.new(@url, address, handler)
+        with_connection(receiver)
+      end
+
+      private
+
+      def with_connection(handler)
+        container = ::Qpid::Proton::Container.new(handler)
+        container.run
+      ensure
+        container&.stop
+      end
+    end
+  end
+end

data/app/models/katello/agent/dispatch_history.rb

@@ -0,0 +1,17 @@
+module Katello
+  module Agent
+    class DispatchHistory < Katello::Model
+      self.table_name = 'katello_agent_dispatch_histories'
+
+      serialize :result, Hash
+
+      def accepted?
+        accepted_at.present?
+      end
+
+      def finished?
+        result.present?
+      end
+    end
+  end
+end

data/app/models/katello/concerns/host_managed_extensions.rb

@@ -33,6 +33,8 @@ module Katello
       included do
         prepend Overrides
 
+        has_many :dispatch_histories, :class_name => "::Katello::Agent::DispatchHistory", :foreign_key => :host_id, :dependent => :delete_all
+
         has_many :host_installed_packages, :class_name => "::Katello::HostInstalledPackage", :foreign_key => :host_id, :dependent => :delete_all
         has_many :installed_packages, :class_name => "::Katello::InstalledPackage", :through => :host_installed_packages
 
@@ -212,6 +214,7 @@ module Katello
         streams = {}
         module_streams.each do |module_stream|
           stream = AvailableModuleStream.where(name: module_stream["name"],
+                                               context: module_stream["context"],
                                                stream: module_stream["stream"]).first_or_create!
           streams[stream.id] = module_stream
         end

data/app/models/katello/events/delete_host_agent_queue.rb

@@ -0,0 +1,19 @@
+module Katello
+  module Events
+    class DeleteHostAgentQueue
+      EVENT_TYPE = 'delete_host_agent_queue'.freeze
+
+      attr_accessor :metadata
+
+      def initialize(_host_id)
+        yield(self) if block_given?
+      end
+
+      def run
+        if metadata[:queue_name]
+          Katello::Agent::Dispatcher.delete_client_queue(queue_name: metadata[:queue_name])
+        end
+      end
+    end
+  end
+end

data/app/models/katello/ping.rb

@@ -14,7 +14,7 @@ module Katello
 
       def services(capsule_id = nil)
         proxy = fetch_proxy(capsule_id)
-        services = [:candlepin, :candlepin_auth, :foreman_tasks, :katello_events, :candlepin_events]
+        services = [:candlepin, :candlepin_auth, :foreman_tasks, :katello_events, :candlepin_events, :katello_agent]
         services += [:pulp3] if proxy&.pulp3_enabled? && pulpcore_enabled
         if proxy.nil? || proxy.has_feature?(SmartProxy::PULP_NODE_FEATURE) || proxy.has_feature?(SmartProxy::PULP_FEATURE)
           services += [:pulp, :pulp_auth]
@@ -38,6 +38,7 @@ module Katello
         ping_foreman_tasks(result[:foreman_tasks]) if result.include?(:foreman_tasks)
         ping_katello_events(result[:katello_events]) if result.include?(:katello_events)
         ping_candlepin_events(result[:candlepin_events]) if result.include?(:candlepin_events)
+        ping_katello_agent(result[:katello_agent]) if result.include?(:katello_agent)
 
         # set overall status result code
         result = {:services => result}
@@ -77,6 +78,13 @@ module Katello
         end
       end
 
+      def ping_katello_agent(result)
+        exception_watch(result) do
+          status = Katello::EventDaemon::Services::AgentEventReceiver.status(refresh: false)
+          event_daemon_status(status, result)
+        end
+      end
+
       def ping_pulp3_without_auth(service_result, capsule_id)
         exception_watch(service_result) do
           Katello::Ping.pulp3_without_auth(fetch_proxy(capsule_id).pulp3_url)

data/app/models/setting/content.rb

@@ -120,8 +120,6 @@ class Setting::Content < Setting
                '', N_('Subscription manager name registration fact'), nil),
       self.set('register_hostname_fact_strict_match', N_('If true, and register_hostname_fact is set and provided, registration will look for a new host by name only '\
                 'using that fact, and will skip all hostname matching'), false, N_('Subscription manager name registration fact strict matching'), nil),
-      self.set('erratum_install_batch_size', N_("Errata installed via katello-agent will be triggered in batches of this size. Set to 0 to install all errata in one batch."),
-               0, N_('Erratum Install Batch Size')),
       self.set('default_location_subscribed_hosts',
                N_('Default Location where new subscribed hosts will put upon registration'),
                nil, N_('Default Location subscribed hosts'), nil,

data/app/services/katello/agent/dispatcher.rb

@@ -0,0 +1,66 @@
+module Katello
+  module Agent
+    class Dispatcher
+      @supported_messages = {}
+
+      def self.register_message(name, klass)
+        @supported_messages[name] = klass
+      end
+
+      register_message(:install_package, Katello::Agent::InstallPackageMessage)
+      register_message(:remove_package, Katello::Agent::RemovePackageMessage)
+      register_message(:update_package, Katello::Agent::UpdatePackageMessage)
+      register_message(:install_errata, Katello::Agent::InstallErrataMessage)
+      register_message(:install_package_group, Katello::Agent::InstallPackageGroupMessage)
+      register_message(:remove_package_group, Katello::Agent::RemovePackageGroupMessage)
+
+      def self.dispatch(message_type, host_ids, args)
+        message_class = @supported_messages[message_type]
+
+        fail("Unsupported message type") unless message_class
+
+        uuid_data = ::Katello::Host::ContentFacet.where(host_id: host_ids).pluck(:host_id, :uuid)
+        fail("Couldn't find all hosts specified") unless host_ids.size == uuid_data.size
+
+        host_data = uuid_data.map do |host_id, consumer_id|
+          {
+            host_id: host_id,
+            consumer_id: consumer_id,
+            history: Katello::Agent::DispatchHistory.new(host_id: host_id),
+            message: message_class.new(**args.merge(consumer_id: consumer_id))
+          }
+        end
+
+        histories = host_data.map { |attrs| attrs[:history] }
+        ActiveRecord::Base.transaction do
+          Katello::Agent::DispatchHistory.import(histories)
+
+          host_data.each do |d|
+            d[:message].dispatch_history_id = d[:history].id
+            d[:message].recipient_address = settings[:client_queue_format] % [d[:consumer_id]]
+            d[:message].reply_to = settings[:event_queue_name]
+          end
+
+          connection = Connection.new
+          connection.send_messages(host_data.map { |d| d[:message] })
+        end
+
+        histories
+      end
+
+      def self.delete_client_queue(queue_name:)
+        connection = Connection.new
+        connection.delete_client_queue(queue_name)
+      end
+
+      def self.host_queue_name(host)
+        uuid = host.content_facet.uuid
+        settings[:client_queue_format] % uuid
+      end
+
+      def self.settings
+        SETTINGS[:katello][:agent]
+      end
+    end
+  end
+end

data/app/services/katello/candlepin_event_listener.rb

@@ -1,4 +1,5 @@
 module Katello
+  # TODO: Move this class to app/lib/katello/event_daemon/services with other service definitions
   class CandlepinEventListener
     Event = Struct.new(:subject, :content)
 

data/app/services/katello/event_monitor/poller_thread.rb

@@ -1,5 +1,6 @@
 module Katello
   module EventMonitor
+    # TODO: Move this class to app/lib/katello/event_daemon/services with other service definitions
     class PollerThread
       SLEEP_INTERVAL = 3
 

data/app/services/katello/registration_manager.rb

@@ -115,6 +115,8 @@ module Katello
         # This can be cleaned up later via clean_backend_objects.
         pulp_consumer_destory(host.content_facet.uuid) if host.content_facet.try(:uuid) && pulp2_supported?
 
+        delete_agent_queue(host) if host.content_facet.try(:uuid)
+
         host.subscription_facet.try(:destroy!)
 
         if unregistering
@@ -266,6 +268,14 @@ module Katello
         Rails.logger.warn(_("Pulp Consumer %s has already been removed") % host_uuid)
       end
 
+      def delete_agent_queue(host)
+        queue_name = Katello::Agent::Dispatcher.host_queue_name(host)
+        Katello::EventQueue.push_event(::Katello::Events::DeleteHostAgentQueue::EVENT_TYPE, host.id) do |attrs|
+          attrs[:metadata] = { queue_name: queue_name }
+          attrs[:process_after] = 10.minutes.from_now
+        end
+      end
+
       def populate_content_facet(host, content_view_environment, uuid)
         content_facet = host.content_facet || ::Katello::Host::ContentFacet.new(:host => host)
         content_facet.content_view = content_view_environment.content_view