manageiq-messaging 0.1.0

data/examples/common.rb

@@ -0,0 +1,40 @@
+require 'optparse'
+
+class Common
+  def initialize
+    @options = {}
+  end
+
+  def parse
+    options = {}
+
+    OptionParser.new do |opt|
+      opt.on("--hostname HOSTNAME", String,  "Hostname") { |v| options[:hostname]  = v }
+      opt.on("--port PORT",         Integer, "Port"    ) { |v| options[:port]      = v }
+      opt.on("--username USERNAME", String,  "Username") { |v| options[:user]      = v }
+      opt.on("--password PASSWORD", String,  "Password") { |v| options[:password]  = v }
+      opt.on("--debug") { ManageIQ::Messaging.logger = Logger.new(STDOUT) }
+      opt.parse!
+    end
+
+    options[:hostname]   ||= ENV["QUEUE_HOSTNAME"] || "localhost"
+    options[:port]       ||= ENV["QUEUE_PORT"]     || 61616
+    options[:user]       ||= ENV["QUEUE_USER"]     || "admin"
+    options[:password]   ||= ENV["QUEUE_PASSWORD"] || "smartvm"
+
+    options[:port] = options[:port].to_i
+
+    @options = options
+    self
+  end
+
+  def q_options
+    {
+      :host       => @options[:hostname],
+      :port       => @options[:port].to_i,
+      :username   => @options[:user],
+      :password   => @options[:password],
+      :client_ref => "background_example",
+    }
+  end
+end

data/examples/message.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'manageiq-messaging'
+require_relative "common"
+
+Thread::abort_on_exception = true
+
+class ProducerConsumer < Common
+  def run
+    ManageIQ::Messaging::Client.open(q_options) do |client|
+      puts "producer"
+      5.times do |i|
+        client.publish_message(
+          :service  => 'ems_operation',
+          :affinity => 'ems_amazon1',
+          :message  => 'power_on',
+          :payload  => {
+            :ems_ref => 'u987',
+            :id      => i.to_s,
+          }
+        )
+      end
+      puts "produced 5 messages"
+
+      puts "consumer"
+      client.subscribe_messages(:service => 'ems_operation', :affinity => 'ems_amazon1') do |messages|
+        messages.each do |msg|
+          do_stuff(msg)
+          client.ack(msg.ack_ref)
+        end
+      end
+      sleep(5)
+      puts "consumed"
+    end
+  end
+
+  def do_stuff(msg)
+    puts "GOT MESSAGE: #{msg.message}: #{msg.payload}"
+  end
+end
+
+ProducerConsumer.new.parse.run

data/lib/manageiq-messaging.rb

@@ -0,0 +1 @@
+require 'manageiq/messaging'

data/lib/manageiq/messaging.rb

@@ -0,0 +1,24 @@
+require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/hash'
+require 'yaml'
+
+require 'manageiq/messaging/null_logger'
+
+module ManageIQ
+  module Messaging
+    autoload :Stomp, 'manageiq/messaging/stomp'
+    autoload :Kafka, 'manageiq/messaging/kafka'
+
+    class << self
+      attr_writer :logger
+    end
+
+    def self.logger
+      @logger ||= NullLogger.new
+    end
+  end
+end
+
+require 'manageiq/messaging/version'
+require 'manageiq/messaging/client'
+require 'manageiq/messaging/received_message'

data/lib/manageiq/messaging/client.rb

@@ -0,0 +1,205 @@
+module ManageIQ
+  module Messaging
+    # The abstract client class. It defines methods needed to publish or subscribe messages.
+    # It is not recommended to directly create a solid subclass instance. The proper way is
+    # to call class method +Client.open+ with desired protocol. For example:
+    #
+    #   client = ManageIQ::Messaging::Client.open(
+    #     :protocol  => 'Stomp',
+    #     :host       => 'localhost',
+    #     :port       => 61616,
+    #     :password   => 'smartvm',
+    #     :username   => 'admin',
+    #     :client_ref => 'generic_1',
+    #     :encoding   => 'json'
+    #   )
+    #
+    # To close the connection one needs to explicitly call +client.close+.
+    # Alternatively if a block is given for the +open+ method, the connection will be closed
+    # automatically before existing the block. For example:
+    #
+    #   ManageIQ::Messaging::Client.open(
+    #     :protocol   => 'Stomp'
+    #     :host       => 'localhost',
+    #     :port       => 61616,
+    #     :password   => 'smartvm',
+    #     :username   => 'admin',
+    #     :client_ref => 'generic_1'
+    #   ) do |client|
+    #     # do stuff with the client
+    #     end
+    #   end
+    class Client
+      # Open or create a connection to the message broker.
+      # Expected +options+ keys are:
+      # * :protocol (Implemented: 'Stomp', 'Kafka'. Default 'Stomp')
+      # * :encoding ('yaml' or 'json'. Default 'yaml')
+      # Other connection options are underlying messaging system specific.
+      #
+      # Returns a +Client+ instance if no block is given.
+      def self.open(options)
+        protocol = options[:protocol] || :Stomp
+        client = Object.const_get("ManageIQ::Messaging::#{protocol}::Client").new(options)
+        return client unless block_given?
+
+        begin
+          yield client
+        ensure
+          client.close
+        end
+        nil
+      end
+
+      # Publish a message to a queue. The message will be delivered to only one subscriber.
+      # Expected keys in +options+ are:
+      # * :service    (service and affinity are used to determine the queue name)
+      # * :affinity   (optional)
+      # * :class_name (optional)
+      # * :message    (e.g. method name or message type)
+      # * :payload    (message body, a string or an user object that can be serialized)
+      # * :sender     (optional, identify the sender)
+      # Other options are underlying messaging system specific.
+      #
+      # Optionally a call back block can be provided to wait on the consumer to send
+      # an acknowledgment. Not every underlying messaging system supports callback.
+      # Example:
+      #
+      #   client.publish_message(
+      #     :service  => 'ems_operation',
+      #     :affinity => 'ems_amazon1',
+      #     :message  => 'power_on',
+      #     :payload  => {
+      #       :ems_ref => 'u987',
+      #       :id      => '123'
+      #     }
+      #   ) do |result|
+      #     ansible_install_pkg(vm1) if result == 'running'
+      #   end
+      def publish_message(options, &block)
+        assert_options(options, [:message, :service])
+
+        publish_message_impl(options, &block)
+      end
+
+      # Publish multiple messages to a queue.
+      # An aggregate version of +#publish_message+ but for better performance.
+      # All messages are sent in a batch. Every element in +messages+ array is
+      # an +options+ hash.
+      #
+      def publish_messages(messages)
+        publish_messages_impl(messages)
+      end
+
+      # Subscribe to receive messages from a queue.
+      # Expected keys in +options+ are:
+      # * :service  (service and affinity are used to determine the queue)
+      # * :affinity (optional)
+      # Other options are underlying messaging system specific.
+      #
+      # A callback block is needed to consume the messages:
+      #
+      #   client.subscribe_message(options) do |messages|
+      #     messages.each do |msg|
+      #       # msg is a type of ManageIQ::Messaging::ReceivedMessage
+      #       # attributes in msg
+      #       msg.sender
+      #       msg.message
+      #       msg.payload
+      #       msg.ack_ref #used to ack the message
+      #
+      #       client.ack(msg.ack_ref)
+      #       # process the message
+      #     end
+      #   end
+      #
+      # Some messaging systems require the subscriber to ack each message in the
+      # callback block. The code in the block can decide when to ack according
+      # to whether a message can be retried. Ack the message in the beginning of
+      # processing if the message is not re-triable; otherwise ack it after the
+      # message is done. Any un-acked message will be redelivered to next subscriber
+      # AFTER the current subscriber disconnects normally or abnormally (e.g. crashed).
+      #
+      # To ack a message call +ack+(+msg.ack_ref+)
+      def subscribe_messages(options, &block)
+        raise "A block is required" unless block_given?
+        assert_options(options, [:service])
+
+        subscribe_messages_impl(options, &block)
+      end
+
+      # Subscribe to receive from a queue and run each message as a background job.
+      # Expected keys in +options+ are:
+      # * :service  (service and affinity are used to determine the queue)
+      # * :affinity (optional)
+      # Other options are underlying messaging system specific.
+      #
+      # This subscriber consumes messages sent through +publish_message+ with required
+      # +options+ keys, for example:
+      #
+      #     client.publish_message(
+      #       :service    => 'generic',
+      #       :class_name => 'MiqTask',
+      #       :message    => 'update_attributes', # method name, for instance method :instance_id is required
+      #       :payload    => {
+      #         :instance_id => 2, # database id of class instance stored in rails DB
+      #         :args        => [{:status => 'Timeout'}] # argument list expected by the method
+      #       }
+      #     )
+      #
+      # Background job assumes each job is not re-triable. It will ack as soon as a request
+      # is received
+      def subscribe_background_job(options)
+        assert_options(options, [:service])
+
+        subscribe_background_job_impl(options)
+      end
+
+      # Publish a message as a topic. All subscribers will receive a copy of the message.
+      # Expected keys in +options+ are:
+      # * :service (service is used to determine the topic address)
+      # * :event   (event name)
+      # * :payload (message body, a string or an user object that can be serialized)
+      # * :sender  (optional, identify the sender)
+      # Other options are underlying messaging system specific.
+      #
+      def publish_topic(options)
+        assert_options(options, [:event, :service])
+
+        publish_topic_impl(options)
+      end
+
+      # Subscribe to receive topic type messages.
+      # Expected keys in +options+ are:
+      # * :service (service is used to determine the topic address)
+      # Other options are underlying messaging system specific.
+      #
+      # Some messaging systems allow subscribers to consume events missed during the period when
+      # the client is offline when they reconnect. Additional options are needed to turn on
+      # this feature.
+      #
+      # A callback block is needed to consume the topic:
+      #
+      #   client.subcribe_topic(:service => 'provider_events') do |sender, event, payload|
+      #     # sender, event, and payload are from publish_topic
+      #   end
+      #
+      def subscribe_topic(options, &block)
+        raise "A block is required" unless block_given?
+        assert_options(options, [:service])
+
+        subscribe_topic_impl(options, &block)
+      end
+
+      private
+
+      def logger
+        ManageIQ::Messaging.logger
+      end
+
+      def assert_options(options, keys)
+        missing = keys - options.keys
+        raise ArgumentError, "options must contain keys #{missing}" unless missing.empty?
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/common.rb

@@ -0,0 +1,36 @@
+module ManageIQ
+  module Messaging
+    module Common
+      private
+
+      def encode_body(headers, body)
+        return body if body.kind_of?(String)
+        headers[:encoding] = encoding
+        case encoding
+        when "json"
+          JSON.generate(body)
+        when "yaml"
+          body.to_yaml
+        else
+          raise "unknown message encoding: #{encoding}"
+        end
+      end
+
+      def decode_body(headers, raw_body)
+        return raw_body unless headers.kind_of?(Hash)
+        case headers["encoding"]
+        when "json"
+          JSON.parse(raw_body)
+        when "yaml"
+          YAML.safe_load(raw_body)
+        else
+          raw_body
+        end
+      end
+
+      def payload_log(payload)
+        payload.to_s[0..100]
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/kafka.rb

@@ -0,0 +1,7 @@
+module ManageIQ
+  module Messaging
+    module Kafka
+      autoload :Client, 'manageiq/messaging/kafka/client'
+    end
+  end
+end

data/lib/manageiq/messaging/kafka/background_job.rb

@@ -0,0 +1,13 @@
+module ManageIQ
+  module Messaging
+    module Kafka
+      module BackgroundJob
+        private
+
+        def subscribe_background_job_impl(_options)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/manageiq/messaging/kafka/client.rb

@@ -0,0 +1,91 @@
+module ManageIQ
+  module Messaging
+    module Kafka
+      # Messaging client implementation with Kafka being the underlying supporting system.
+      # Do not directly instantiate an instance from this class. Use
+      # +ManageIQ::Messaging::Client.open+ method.
+      #
+      # Kafka specific connection options accepted by +open+ method:
+      # * :client_ref (A reference string to identify the client)
+      # * :hosts (Array of Kafka cluster hosts, or)
+      # * :host (Single host name)
+      # * :port (host port number)
+      # * :ssl_ca_cert (security options)
+      # * :ssl_client_cert
+      # * :ssl_client_cert_key
+      # * :sasl_gssapi_principal
+      # * :sasl_gssapi_keytab
+      # * :sasl_plain_username
+      # * :sasl_plain_password
+      # * :sasl_scram_username
+      # * :sasl_scram_password
+      # * :sasl_scram_mechanism
+      #
+      # Kafka specific +publish_message+ options:
+      # * :group_name (Used as Kafka partition_key)
+      #
+      # Kafka specific +subscribe_topic+ options:
+      # * :persist_ref (Used as Kafka group_id)
+      #
+      # Without +:persist_ref+ every topic subscriber receives a copy of each message
+      # only when they are active. If multiple topic subscribers join with the same
+      # +:persist_ref+, each message is consumed by only one of the subscribers. This
+      # allows a load balancing among the subscribers. Also any messages sent when
+      # all members of the +:persist_ref+ group are offline will be persisted and delivered
+      # when any member in the group is back online. Each message is still copied and
+      # delivered to other subscribers that belongs to other +:persist_ref+ groups or no group.
+      #
+      # +subscribe_background_job+ is currently not implemented.
+      class Client < ManageIQ::Messaging::Client
+        require 'kafka'
+        require 'manageiq/messaging/kafka/common'
+        require 'manageiq/messaging/kafka/queue'
+        require 'manageiq/messaging/kafka/background_job'
+        require 'manageiq/messaging/kafka/topic'
+
+        include Common
+        include Queue
+        include BackgroundJob
+        include Topic
+
+        private *delegate(:subscribe, :unsubscribe, :publish, :to => :kafka_client)
+        delegate :close, :to => :kafka_client
+
+        attr_accessor :encoding
+
+        def ack(*_args)
+        end
+
+        def close
+          @consumer.try(:stop)
+          @consumer = nil
+
+          @producer.try(:shutdown)
+          @producer = nil
+
+          kafka_client.close
+          @kafka_client = nil
+        end
+
+        private
+
+        attr_reader :kafka_client
+
+        def initialize(options)
+          hosts = Array(options[:hosts] || options[:host])
+          hosts.collect! { |host| "#{host}:#{options[:port]}" }
+
+          @encoding = options[:encoding] || 'yaml'
+          require "json" if @encoding == "json"
+
+          connection_opts = {}
+          connection_opts[:client_id] = options[:client_ref] if options[:client_ref]
+
+          connection_opts.merge!(options.slice(:ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :sasl_gssapi_principal, :sasl_gssapi_keytab, :sasl_plain_username, :sasl_plain_password, :sasl_scram_username, :sasl_scram_password, :sasl_scram_mechanism))
+
+          @kafka_client = ::Kafka.new(hosts, connection_opts)
+        end
+      end
+    end
+  end
+end