pipeline_toolkit 1.0.4 → 1.0.6

data/lib/pipeline_toolkit/message_generator.rb

@@ -0,0 +1,21 @@
+require 'json'
+
+class MessageGenerator
+  
+  def initialize(opts)
+    @delay = opts.delay
+    @msg = JSON.parse(opts.msg)
+  end
+
+  def start
+    loop do
+      self.send_msg(@msg)
+      sleep(@delay) unless @delay.nil?
+    end
+  end
+
+  def send_msg(msg)
+    # puts MessageCoder.encode(@msg)
+    $stdout.flush
+  end
+end

data/lib/pipeline_toolkit/message_probe.rb

@@ -13,16 +13,16 @@ class MessageProbe
     self.init_dnssd if opts.dnssd
   end
 
-  def init_loop
-    EM.start_server('0.0.0.0', @http_port, ProbeHttpRequest, self)
-    EM.add_periodic_timer(@interval) { self.tick }
-  end
-
   def init_dnssd
     require 'dnssd'
     DNSSD.register!("#{@name} probe", "_http._tcp", nil, @http_port)
   end
 
+  def initialize_machine
+    EM.start_server('0.0.0.0', @http_port, ProbeHttpRequest, self)
+    EM.add_periodic_timer(@interval) { self.tick }
+  end
+
   def tick
     @time_delta = Time.now - @prev_time
     @mps = @count / @time_delta
@@ -40,7 +40,7 @@ class MessageProbe
   
   # OPTIMIZE. can improve performance by overriding base process_line method
   # saving us the unnecessary marshal step.
-  def process_message(msg)
+  def process_standard(msg)
     @count += 1
     msg
   end

data/lib/pipeline_toolkit/message_pusher.rb

@@ -1,61 +1,58 @@
 require "mq"
 
+##
+# A Message Queue machine to handle messages that has to be published 
+# back into a message queue.
+#
 class MessagePusher
   include MessageCommand
-
-  def initialize(opts)
-    @key_eval = opts.key_eval
-    if opts.key_file
-      @key_file = opts.key_file
-      load_route(@key_file)
-      self.init_route
-    end
-    @exchange_names = opts.exchanges.map { |str| str.split(":") }
-    @msg_server_config = opts.select_keys(:host, :port, :user, :pass, :vhost)
-  end
-
-  def init_loop
-    @msg_server = MQ.new(AMQP.connect(@msg_server_config))
-    self.setup_exchanges
-  end
-
-  def setup_exchanges
-    @exchanges = []
-    @exchange_names.each do |name, type|
-      type ||= :fanout
-      @exchanges << MQ::Exchange.new(@msg_server, type.to_sym, name, :durable => true, :passive => false)
-    end
-  end
-
-  def syspipe_open
-    # Send back details of where messages are going
-    msg = {:msg_type => :pipe_desc, :exchanges => @exchange_names.join(",")}
-    self.sys_pipe.syswrite(MessageCoder.encode(msg) << "\n")
-  end
-
-  def load_route(key_file)
-    require key_file
-    self.extend eval(classify(key_file.gsub(".rb", "")))
-  end
-  
-  # Turn path Class or Module name (i.e. strip directories and turn into camel-case)
-  def classify(str)
-    str.gsub(/^.*\//, '').gsub(".rb","").gsub(/(?:^|_)(.)/) { $1.upcase }
-  end
-  
-  # is overriden by included fork_file if specified
-  def route_key(msg)
-    @key_eval ? eval(@key_eval) : nil
-  end
-
-  def process_message(msg)
-    @exchanges.each do |exchange|
-      key = route_key(msg)
-      exchange.publish(msg.to_yaml)
-      # OPTIMIZE. Using MessageCoder.encode(msg) instead of to_yaml is 2x faster. But won't be easy to
-      # debug. Worth it?
-    end
-    :ack
+  include Amqp::Writer
+
+  ##
+  # Initializes a new intance
+  #
+  # @param options<Hash> Options hash for the message pusher.
+  # 
+  def initialize(options = {})
+    super(options)
+    DefaultLogger.debug("MessagePusher#initialize(options = {})") if options[:env] == "development"
+
+    queues_string = options[:queues].map { |name, type| "#{name} (key:#{type})" }.join(",")
+    DefaultLogger.info "================================================================"
+    DefaultLogger.info "Booting #{self.class.name}  (#{options[:env]})"
+    DefaultLogger.info "Exchange: #{options[:exchange]} (#{options[:type]} passive:#{options[:passive]} durable:#{options[:durable]})"
+    DefaultLogger.info "Queues: #{queues_string}"
+    DefaultLogger.info "amqp://#{options[:user]}:#{options[:pass]}@#{options[:host]}:#{options[:port]}#{options[:vhost]}"
+    DefaultLogger.info ""
+  end
+
+  ##
+  # Initialize the AMQP server connection and exchange so we can write messages to the queue.
+  #
+  def initialize_machine
+    DefaultLogger.debug("MessagePusher#initialize_machine") if options[:env] == "development"
+    initialize_writer
+    initialize_queues    
+  end
+
+  ##
+  # Notify back down the pipeline information about this machine.
+  #
+  def report_back
+    DefaultLogger.debug("MessagePusher#open_acknowledgement_pipe") if options[:env] == "development"
+    message = { :msg_type => :pipe_desc, :queues => queue_names.join(",") }
+    write_to_pipe(message, @ack_pipe)
+  end
+
+  ##
+  # Handle the messages by writing them into the AMQP server exchange.
+  #
+  # @param message<Hash> The message to write to the exchange.
+  # 
+  def process_standard(message)
+    DefaultLogger.debug("MessagePusher#process_standard(message)") if options[:env] == "development"
+    publish(message)
+    acknowledge(message) if options[:ack]
   end
 
 end

data/lib/pipeline_toolkit/message_sink.rb

@@ -1,7 +1,7 @@
 class MessageSink
   include MessageCommand
   
-  def process_message(msg)
+  def process_standard(msg)
     :ack
   end
     

data/lib/pipeline_toolkit/message_subscriber.rb

@@ -3,246 +3,227 @@ require "mq"
 require "time"
 require "socket"
 
+##
+# The message subscriber is used to subscribe to a AMQP server queue 
+# and handle a single message at a time.
+#
 class MessageSubscriber
-  include DefaultLogger
   
-  attr_reader :name, :start_time, :mps, :queue_name
-  attr_accessor :exchanges_out
+  include Amqp::Reader
+
+  attr_reader :start_time
+  attr_reader :mps
+  attr_reader :options
+  attr_accessor :queues_out
 
   PIPE_PATH = "/tmp"
   
-  def initialize(opts)
-    @exchange_name, @exchange_type = opts[:exchange_bind].split(":")
-    @queue_name = opts[:queue]
-    @use_ack = opts[:ack]
-    @topic = opts[:topic]
-    @msg_server_opts = opts.select_keys(:host, :port, :user, :pass, :vhost)
-    @unackd_msgs = {}
-    @max_unackd = opts[:max_unackd]
-
-    @http_port = opts.http_port || Socket.select_random_port(10_000, 11_000)
-    @name = opts.name || (Socket.gethostname + '_' + Process.pid.to_s)
-    self.init_dnssd if opts.dnssd
-    self.reset
-  end
-
-  def init_dnssd
-    # FIXME. DNS-SD breaks when under load. Not sure what problem is yet, have raised it with
-    # gem author:
-    # http://github.com/tenderlove/dnssd/issues#issue/3
-    require 'dnssd'
-    DNSSD.register!("#{@name}_pipe", "_http._tcp", nil, @http_port)
+  ##
+  # Initializes a new instance
+  #
+  # @param options<Hash> An options hash, see command line interface.
+  # 
+  def initialize(options = {})
+    
+    DefaultLogger.init_logger(options)
+    
+    @options = options
+    
+    @options[:http_port] ||= Socket.select_random_port(10_000, 11_000)
+    @options[:name] ||= (Socket.gethostname + '_' + Process.pid.to_s)
+
+    DefaultLogger.info "================================================================"
+    DefaultLogger.info "Booting #{self.class.name}  (#{options[:env]})"
+    DefaultLogger.info "Exchange: #{options[:exchange]} (#{options[:type]} passive:#{options[:passive]} durable:#{options[:durable]})"
+    DefaultLogger.info "Queue: #{options[:queue]} (ack:#{options[:ack]})"
+    DefaultLogger.info "amqp://#{options[:user]}:#{options[:pass]}@#{options[:host]}:#{options[:port]}#{options[:vhost]}"
+    DefaultLogger.info "Monitoring: http://localhost:#{options[:http_port]}/ (#{options[:content_type]} name:#{options[:name]} DNS-SD:#{options[:dnssd]})"
+    DefaultLogger.info ""
+
+    @queues_out = ''
+    
+    initialize_dnssd
+    
+    reset_message_statistics
   end
 
+  ##
+  # Starts the subscriber reactor loop
+  #
   def start
-    # FIXME. Need to do more investigation on if/when messages are lost with shutdowns
-    Signal.trap('INT')  { AMQP.stop{ EM.stop } }
-    Signal.trap('TERM') { AMQP.stop{ EM.stop } }
 
+    Signal.trap('INT') { EM.stop  }
+    Signal.trap('TERM') { EM.stop  }
+    
+    DefaultLogger.debug("MessageSubscriber#start")
     begin
-      self.create_sys_pipe
-      AMQP.start(@msg_server_opts) do
+      create_ack_pipe
+
+      EM.run do
         @start_time = Time.now
-        self.setup_queue
-        # NB. prefetch limits the amount of unknowledged messages that come down the pipe.
-        MQ.prefetch(@max_unackd)
-        @queue.subscribe(:ack => @use_ack) do |header, body|
-          self.process_message(header, body)
-        end
-        
-        conn = EM.watch(@sys_pipe, HandleCtlMessages, @sys_pipe, self)
+
+        initialize_reader
+        queue_subscribe
+     
+        conn = EM.watch(@ack_pipe, Handlers::MessageHandler, self, options)
+        # must call this to setup callback to notify_readable
         conn.notify_readable = true
-        
-        EM.start_server('0.0.0.0', @http_port, PipeHttpRequest, self)
-        EM.add_periodic_timer(5) { self.calc_stats }
+
+        EM.start_server('0.0.0.0', options[:http_port], Monitoring::MonitorServer, self, options)
+        EM.add_periodic_timer(5) { calculate_message_statistics }
       end
     rescue StandardError => e
-      log.error e.message + " " + e.backtrace.join("\n")
+      DefaultLogger.error "#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n")
       raise e
     ensure
-      self.shutdown
+      shutdown
     end
   end
-
-  def calc_stats
-    @time_delta = Time.now - @prev_time
-    @mps  = @count  / @time_delta
-    self.reset
-  end
-
-  def reset
-    @count = 0
-    @prev_time = Time.now
-  end
-
+  
+  ##
+  # Stop the subscriber
+  #
   def shutdown
-    log.info "Shutting down"
-    self.destroy_sys_pipe
-  end
-
-  def setup_queue
-    # If a queue_name is given then we treat the queue as fixed, otherwise as temporary
-    @queue = @queue_name ? MQ.queue(@queue_name, :durable => true) : 
-                           self.generate_temporary_queue
-    if @exchange_name
-      create_exchange(@exchange_name, (@exchange_type || :fanout))
-      log.info("Binding to exchange:#{@exchange_str} #{@topic ? "using topic:" + @topic : ""}")
-      @queue.bind(@exchange_str, :key => @topic)
+    DefaultLogger.info("Shutting down #{self.class.name}")
+    DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+    destroy_ack_pipe
+  end
+
+  ##
+  # Callback for the Handlers::MessageHandler when it receives a message
+  #
+  # @param message<Hash> The decoded message
+  # 
+  def process(message)
+    case message[:msg_type]
+      when :ack
+        perform_acknowledgement(message[:ack_id])
+      when :pipe_desc
+        DefaultLogger.debug("AcknowledgementHandler#queues_out")
+        @queues_out = message[:queues]
+      else
+        raise Exception.new("Unknown control message received: #{message}")
     end
   end
-
-  def create_exchange(name, type)
-    MQ::Exchange.new(MQ.default, type.to_sym, @exchange_name, :durable => true, :passive => false)
-  end
-
-  def create_sys_pipe
-    log.debug("creating sys-pipe")
-    name = File.join(PIPE_PATH, "sys_pipe_#{self.generate_guid}")
-    `mkfifo #{name}`
-    @sys_pipe = File.new(name, "r+")
-    $stdout.puts(MessageCoder.encode({:msg_type => :system, 
-                                      :sys_pipe => name,
-                                      :use_ack => @use_ack,
-                                      :max_unackd => @max_unackd}))
-    $stdout.flush
-  end
-
-  def destroy_sys_pipe
-    `rm #{@sys_pipe.path}`
-  end
   
-  def generate_temporary_queue
-    qname = "#{Socket.gethostname}_#{self.generate_guid}"
-    log.debug("Binding temporary queue #{qname}")
-    MQ.queue(qname, :auto_delete => true, :durable => false)
+  ##
+  # Support templating of member data.
+  #
+  def get_binding
+    binding
   end
   
-  # Generates a guid. Stolen from EM.
-  def generate_guid
-    # Cache uuidgen seed for better performance
-    if @ix and @ix >= 10_000
-      @ix = nil
-      @seed = nil
-    end
-    
-    # NB. This will only work on *nix platforms
-    @seed ||= `uuidgen`.chomp.gsub(/-/,"")
-    @ix ||= 0
-
-    "#{@seed}#{@ix += 1}"
-  end
-    
-  def process_message(header, body)
-    msg = YAML.load(body)
-    store_ack(msg, header) if @use_ack
-    write_msg(msg)
+  # :nodoc:
+  def name
+    options[:name] || ''
   end
   
-  def write_msg(msg)
-    @count += 1
-    $stdout.syswrite(MessageCoder.encode(msg) << "\n")
+  # :nodoc:
+  def queue_name
+    options[:name] || ''
   end
   
-  def perform_ack(ack_id)    
-    header = @unackd_msgs.delete(ack_id)
-    header.ack
-  end
+  protected
   
-  def store_ack(msg, header)
-    msg.ack_id = header.delivery_tag.to_s
-    @unackd_msgs[msg.ack_id] = header
+  ##
+  # Process the message received from AMQP server
+  #
+  # @param header<MQ::Header> The header of the message
+  # @param body Description
+  # 
+  def process_queue_message(header, body)
+    DefaultLogger.debug("MessageSubscriber#process_queue_message(header, body)")
+    unless body.is_a?(Hash)
+      message = { :msg_type => :standard, :raw => body }
+    else
+      message = { :msg_type => :standard }.merge(body)
+    end
+    store_acknowledgement(message, header) if options[:ack]
+    write(message)
   end
+  
+  private
+  
+    ##
+    # Reset the instance variables used to store message statistics
+    #
+    def reset_message_statistics
+      @count = 0
+      @prev_time = Time.now
+    end
+    
+    ##
+    # Calculate the statistics on message handling
+    #
+    def calculate_message_statistics
+      @time_delta = Time.now - @prev_time
+      @mps  = @count  / @time_delta
+      reset_message_statistics
+    end
 
-  # Handles msg acks
-  module HandleCtlMessages
-    include DefaultLogger
-
-    def initialize(sys_pipe, msg_sub)
-      @sys_pipe = sys_pipe
-      @msg_sub = msg_sub
+    ##
+    # Create the acknowledgement named pipe
+    #
+    def create_ack_pipe
+      name = File.join(PIPE_PATH, "sys_pipe_#{generate_guid}")
+      `mkfifo #{name}`
+      @ack_pipe = File.new(name, "r+")
+      message = { :msg_type => :system, 
+              :sys_pipe => name, 
+              :ack => options[:ack] }
+      write_to_pipe(message)
+    end
+    
+    ##
+    # Destroy the acknowledgement named pipe
+    #
+    def destroy_ack_pipe
+      `rm #{@ack_pipe.path}`
     end
 
-    def notify_readable
-      msg = MessageCoder.decode(@sys_pipe.gets.chomp!)
-      case msg[:msg_type]
-      when :ack
-        @msg_sub.perform_ack(msg.ack_id)
-      when :pipe_desc
-        @msg_sub.exchanges_out = msg.exchanges
-      else
-        raise Exception.new("Unknown control message received: #{message}")
+    ##
+    # Generates a unique guid to use as part of the acknowledgement named pipe name. 
+    # Stolen from EM.
+    #
+    def generate_guid
+      # Cache uuidgen seed for better performance
+      if @ix and @ix >= 10_000
+        @ix = nil
+        @seed = nil
       end
-    end
-  end
 
-  class PipeHttpRequest < EM::Connection
-    include EM::HttpServer
+      # NB. This will only work on *nix platforms
+      @seed ||= `uuidgen`.chomp.gsub(/-/,"")
+      @ix ||= 0
 
-    def initialize(msg_sub)
-      @msg_sub = msg_sub
+      "#{@seed}#{@ix += 1}"
     end
 
-    def post_init
-      super
-      no_environment_strings
+    ##
+    # Write the message to STDOUT
+    #
+    # @param message<Hash> The message to write
+    # 
+    def write(message)
+      DefaultLogger.debug("MessageSubscriber#write(message)")
+      @count += 1
+      write_to_pipe(message)
     end
 
-    def process_http_request
-      response = EM::DelegatedHttpResponse.new(self)
-      response.status = 200
-      response.content_type 'text/html'
-      response.content = <<-EOL
-      <html>
-        <head>
-          <title>Message Probe</title>
-          <style type="text/css">
-            body {
-              background: black;
-              color: #80c0c0;
-            }
-            h1 {
-              font: 12pt Monospace;
-              text-align:center;
-            }
-            table {
-              font: 10pt Monospace;
-              margin-left:auto;
-              margin-right:auto;
-              text-align:right;
-            }
-            .page {
-              position:relative;
-              top: 20%;
-              # border-style:solid;
-              # border-width:5px;
-              width: 30%;
-              margin-left:auto;
-              margin-right:auto;
-            }
-          </style>
-        </head>
-        <body>
-          <div class=page>
-            <h1><span class="name">#{@msg_sub.name}</span></h1>
-            <table>
-              <tr>
-                <td>Structure:</td><td><span class="queue_in">#{@msg_sub.queue_name}</span> -> <span class="exchanges_out">#{@msg_sub.exchanges_out}</span></td>
-              </tr>
-              <tr>
-                <td>Throughput:</td><td><span class="mps">#{@msg_sub.mps.to_i}</span></td>
-              </tr>
-              <tr>
-                <td>Uptime:</td><td><span class="uptime">#{(Time.now - @msg_sub.start_time).to_i / 60}mins</span></td>
-              </tr>
-            </table>
-          </div>
-        </body>
-      </html>
-      EOL
-
-      response.send_response
+    ##
+    # Initialize the DNS Service Discovery (aka Bonjour, MDNS).
+    #
+    def initialize_dnssd
+      return unless options[:dnssd]
+      # FIXME. DNS-SD breaks when under load. Not sure what problem is yet, have raised it with
+      # gem author:
+      # http://github.com/tenderlove/dnssd/issues#issue/3
+      require 'dnssd'
+      DNSSD.register!("#{options[:name]}_pipe", "_http._tcp", nil, options[:http_port])
     end
-  end
-    
-end
 
+    def write_to_pipe(message, pipe=$stdout)
+      pipe.syswrite(MessageCoder.encode(message) << "\n")
+    end
+
+end