pipeline_toolkit 1.1.0 → 1.2.0

data/lib/pipeline_toolkit/message_generator.rb

@@ -1,27 +1,31 @@
 require 'json'
 
-class MessageGenerator
+module PipelineToolkit
   
-  def initialize(options)
-    @delay = options.delay
-    @msg = JSON.parse(options.msg)    
-    @limit = options.limit
-  end
+  class MessageGenerator
+  
+    def initialize(options)
+      @delay = options.delay
+      @msg = JSON.parse(options.msg)    
+      @limit = options.limit
+    end
   
-  def start
-    @count = 0
-    loop do
-      @count += 1
-      break if @limit && @count > @limit
+    def start
+      @count = 0
+      loop do
+        @count += 1
+        break if @limit && @count > @limit
 
-      self.send_msg(@msg)
-      sleep(@delay) unless @delay.nil?
+        self.send_msg(@msg)
+        sleep(@delay) unless @delay.nil?
+      end
     end
-  end
 
-  def send_msg(msg)
-    puts MessageCoder.encode(@msg)
-    $stdout.flush
-  end
+    def send_msg(msg)
+      puts MessageCoder.encode(@msg)
+      $stdout.flush
+    end
   
+  end
+
 end

data/lib/pipeline_toolkit/message_pusher.rb

@@ -1,58 +1,57 @@
 require "mq"
 
-##
-# A Message Queue machine to handle messages that has to be published 
-# back into a message queue.
-#
-class MessagePusher
-  include MessageCommand
-  include Amqp::Writer
+module PipelineToolkit
 
   ##
-  # Initializes a new intance
+  # A Message Queue machine to handle messages that has to be published 
+  # back into a message queue.
   #
-  # @param options<Hash> Options hash for the message pusher.
-  # 
-  def initialize(options = {})
-    super(options)
-    DefaultLogger.debug("MessagePusher#initialize(options = {})") if options[:env] == "development"
-
-    queues_string = options[:queues].map { |name, type| "#{name} (key:#{type})" }.join(",")
-    DefaultLogger.info "================================================================"
-    DefaultLogger.info "Booting #{self.class.name}  (#{options[:env]})"
-    DefaultLogger.info "Exchange: #{options[:exchange]} (#{options[:type]} passive:#{options[:passive]} durable:#{options[:durable]})"
-    DefaultLogger.info "Queues: #{queues_string}"
-    DefaultLogger.info "amqp://#{options[:user]}:#{options[:pass]}@#{options[:host]}:#{options[:port]}#{options[:vhost]}"
-    DefaultLogger.info ""
-  end
+  class MessagePusher
+    include MessageCommand
+    include Amqp::Writer
 
-  ##
-  # Initialize the AMQP server connection and exchange so we can write messages to the queue.
-  #
-  def initialize_machine
-    DefaultLogger.debug("MessagePusher#initialize_machine") if options[:env] == "development"
-    initialize_writer
-    initialize_queues    
-  end
+    ##
+    # Initializes a new intance
+    #
+    # @param options<Hash> Options hash for the message pusher.
+    # 
+    def initialize(options = {})
+      super(options)
+      DefaultLogger.debug("MessagePusher#initialize(options = {})") if options[:env] == "development"
 
-  ##
-  # Notify back down the pipeline information about this machine.
-  #
-  def report_back
-    DefaultLogger.debug("MessagePusher#open_acknowledgement_pipe") if options[:env] == "development"
-    message = { :msg_type => :pipe_desc, :queues => queue_names.join(",") }
-    write_to_pipe(message, @ack_pipe)
-  end
+      queues_string = options[:queues].map { |name, type| "#{name} (key:#{type})" }.join(",")
+      DefaultLogger.info "================================================================"
+      DefaultLogger.info "Booting #{self.class.name}  (#{options[:env]})"
+      DefaultLogger.info "Exchange: #{options[:exchange]} (#{options[:type]} passive:#{options[:passive]} durable:#{options[:durable]})"
+      DefaultLogger.info "Queues: #{queues_string}"
+      DefaultLogger.info "amqp://#{options[:user]}:#{options[:pass]}@#{options[:host]}:#{options[:port]}#{options[:vhost]}"
+      DefaultLogger.info ""
+    end
+
+    ##
+    # Initialize the AMQP server connection and exchange so we can write messages to the queue.
+    #
+    def initialize_machine
+      DefaultLogger.debug("MessagePusher#initialize_machine") if options[:env] == "development"
+      initialize_writer
+      initialize_queues    
+    end
+
+    def description
+      "exchange:#{options[:exchange]}"
+    end
+
+    ##
+    # Handle the messages by writing them into the AMQP server exchange.
+    #
+    # @param message<Hash> The message to write to the exchange.
+    # 
+    def process_standard(message)
+      DefaultLogger.debug("MessagePusher#process_standard(message)") if options[:env] == "development"
+      publish(message)
+      acknowledge(message) if options[:ack]
+    end
 
-  ##
-  # Handle the messages by writing them into the AMQP server exchange.
-  #
-  # @param message<Hash> The message to write to the exchange.
-  # 
-  def process_standard(message)
-    DefaultLogger.debug("MessagePusher#process_standard(message)") if options[:env] == "development"
-    publish(message)
-    acknowledge(message) if options[:ack]
   end
 
 end

data/lib/pipeline_toolkit/message_sink.rb

@@ -1,8 +1,12 @@
-class MessageSink
-  include MessageCommand
+module PipelineToolkit
+
+  class MessageSink
+    include MessageCommand
   
-  def process_standard(msg)
-    :ack
-  end
+    def process_standard(message)
+      acknowledge(message)
+    end
     
+  end
+  
 end

data/lib/pipeline_toolkit/message_subscriber.rb

@@ -3,228 +3,235 @@ require "mq"
 require "time"
 require "socket"
 
-##
-# The message subscriber is used to subscribe to a AMQP server queue 
-# and handle a single message at a time.
-#
-class MessageSubscriber
+module PipelineToolkit
+  ##
+  # The message subscriber is used to subscribe to a AMQP server queue 
+  # and handle a single message at a time.
+  #
+  class MessageSubscriber
   
-  include Amqp::Reader
+    include Amqp::Reader
 
-  attr_reader :start_time
-  attr_reader :mps
-  attr_reader :options
-  attr_accessor :queues_out
+    attr_reader :start_time
+    attr_reader :mps
+    attr_reader :options
+    attr_accessor :structure
 
-  PIPE_PATH = "/tmp"
+    PIPE_PATH = "/tmp"
   
-  ##
-  # Initializes a new instance
-  #
-  # @param options<Hash> An options hash, see command line interface.
-  # 
-  def initialize(options = {})
-    
-    DefaultLogger.init_logger(options)
-    
-    @options = options
+    ##
+    # Initializes a new instance
+    #
+    # @param options<Hash> An options hash, see command line interface.
+    # 
+    def initialize(options = {})
     
-    @options[:http_port] ||= Socket.select_random_port(10_000, 11_000)
-    @options[:name] ||= (Socket.gethostname + '_' + Process.pid.to_s)
-
-    DefaultLogger.info "================================================================"
-    DefaultLogger.info "Booting #{self.class.name}  (#{options[:env]})"
-    DefaultLogger.info "Exchange: #{options[:exchange]} (#{options[:type]} passive:#{options[:passive]} durable:#{options[:durable]})"
-    DefaultLogger.info "Queue: #{options[:queue]} (ack:#{options[:ack]})"
-    DefaultLogger.info "amqp://#{options[:user]}:#{options[:pass]}@#{options[:host]}:#{options[:port]}#{options[:vhost]}"
-    DefaultLogger.info "Monitoring: http://localhost:#{options[:http_port]}/ (#{options[:content_type]} name:#{options[:name]} DNS-SD:#{options[:dnssd]})"
-    DefaultLogger.info ""
-
-    @queues_out = ''
+      DefaultLogger.init_logger(options)
     
-    initialize_dnssd
+      @options = options
+      options[:ack] = !options[:no_acks]
     
-    reset_message_statistics
-  end
+      DefaultLogger.info "================================================================"
+      DefaultLogger.info "Booting #{self.class.name}  (#{options[:env]})"
+      DefaultLogger.info "Exchange: #{options[:exchange]} (#{options[:type]} passive:#{options[:passive]} durable:#{options[:durable]})"
+      DefaultLogger.info "Queue: #{options[:queue]} (ack:#{options[:ack]})"
+      DefaultLogger.info "amqp://#{options[:user]}:#{options[:pass]}@#{options[:host]}:#{options[:port]}#{options[:vhost]}"
+      DefaultLogger.info "Monitoring: http://localhost:#{options[:http_port]}/ (#{options[:content_type]})"
+      DefaultLogger.info ""
+
+      @structure = [description]
+        
+      reset_message_statistics
+    end
 
-  ##
-  # Starts the subscriber reactor loop
-  #
-  def start
+    ##
+    # Starts the subscriber reactor loop
+    #
+    def start
 
-    Signal.trap('INT') { EM.stop  }
-    Signal.trap('TERM') { EM.stop  }
+      Signal.trap('INT') { EM.stop  }
+      Signal.trap('TERM') { EM.stop  }
     
-    DefaultLogger.debug("MessageSubscriber#start")
-    begin
-      create_ack_pipe
+      DefaultLogger.debug("MessageSubscriber#start")
+      begin
+        create_sys_pipe
 
-      EM.run do
-        @start_time = Time.now
+        EM.run do
+          @start_time = Time.now
 
-        initialize_reader
-        queue_subscribe
+          initialize_reader
+          queue_subscribe
      
-        conn = EM.watch(@ack_pipe, Handlers::MessageHandler, self, options)
-        # must call this to setup callback to notify_readable
-        conn.notify_readable = true
-
-        EM.start_server('0.0.0.0', options[:http_port], Monitoring::MonitorServer, self, options)
-        EM.add_periodic_timer(5) { calculate_message_statistics }
+          conn = EM.watch(@sys_pipe, Handlers::MessageHandler, self, options)
+          # must call this to setup callback to notify_readable
+          conn.notify_readable = true
+
+          if monitoring_enabled?
+            EM.start_server('0.0.0.0', options[:http_port], Monitoring::MonitorServer, self, options)
+            EM.add_periodic_timer(5) { calculate_message_statistics }
+          end
+        end
+      rescue StandardError => e
+        DefaultLogger.error "#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n")
+        raise e
+      ensure
+        shutdown
       end
-    rescue StandardError => e
-      DefaultLogger.error "#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n")
-      raise e
-    ensure
-      shutdown
     end
-  end
   
-  ##
-  # Stop the subscriber
-  #
-  def shutdown
-    DefaultLogger.info("Shutting down #{self.class.name}")
-    DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
-    destroy_ack_pipe
-  end
-
-  ##
-  # Callback for the Handlers::MessageHandler when it receives a message
-  #
-  # @param message<Hash> The decoded message
-  # 
-  def process(message)
-    case message[:msg_type]
-      when :ack
-        perform_acknowledgement(message[:ack_id])
-      when :pipe_desc
-        DefaultLogger.debug("AcknowledgementHandler#queues_out")
-        @queues_out = message[:queues]
-      else
-        raise Exception.new("Unknown control message received: #{message}")
+    ##
+    # Stop the subscriber
+    #
+    def shutdown
+      DefaultLogger.info("Shutting down #{self.class.name}")
+      DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+      destroy_sys_pipe
     end
-  end
-  
-  ##
-  # Support templating of member data.
-  #
-  def get_binding
-    binding
-  end
-  
-  # :nodoc:
-  def name
-    options[:name] || ''
-  end
-  
-  # :nodoc:
-  def queue_name
-    options[:name] || ''
-  end
-  
-  protected
   
-  ##
-  # Process the message received from AMQP server
-  #
-  # @param header<MQ::Header> The header of the message
-  # @param body Description
-  # 
-  def process_queue_message(header, body)
-    DefaultLogger.debug("MessageSubscriber#process_queue_message(header, body)")
-    body = MessageCoder.decode(body)
-    unless body.is_a?(Hash)
-      message = { :msg_type => :standard, :raw => body }
-    else
-      message = { :msg_type => :standard }.merge(body)
+    def monitoring_enabled?
+      !options[:http_port].nil?
     end
-    store_acknowledgement(message, header) if options[:ack]
-    write(message)
-  end
-  
-  private
-  
+
     ##
-    # Reset the instance variables used to store message statistics
+    # Callback for the Handlers::MessageHandler when it receives a message
     #
-    def reset_message_statistics
-      @count = 0
-      @prev_time = Time.now
+    # @param message<Hash> The decoded message
+    # 
+    def process(message)
+      case message[:msg_type]
+        when "ack"
+          DefaultLogger.debug("Acknowledging message #{message[:ack_id]}")
+          perform_acknowledgement(message[:ack_id])
+        when "pipe_desc"
+          @structure << message[:description].to_s
+        else
+          raise "Unknown control message received: #{message.inspect}"
+      end
     end
-    
+  
     ##
-    # Calculate the statistics on message handling
+    # Support templating of member data.
     #
-    def calculate_message_statistics
-      @time_delta = Time.now - @prev_time
-      @mps  = @count  / @time_delta
-      reset_message_statistics
+    def get_binding
+      binding
     end
-
+  
+    # :nodoc:
+    def process_id
+      Process.pid.to_s
+    end
+    
     ##
-    # Create the acknowledgement named pipe
+    # The uptime of the subscriber in mins
     #
-    def create_ack_pipe
-      name = File.join(PIPE_PATH, "sys_pipe_#{generate_guid}")
-      `mkfifo #{name}`
-      @ack_pipe = File.new(name, "r+")
-      message = { :msg_type => :system, 
-              :sys_pipe => name, 
-              :ack => options[:ack] }
-      write_to_pipe(message)
+    def uptime
+      (Time.now - self.start_time).to_i
     end
     
     ##
-    # Destroy the acknowledgement named pipe
+    # The hostname of the server the subscriber is running on
     #
-    def destroy_ack_pipe
-      `rm #{@ack_pipe.path}`
+    def hostname
+      Socket.gethostname
     end
 
+    def description
+      "queue:#{options[:queue]}"
+    end
+  
+    protected
+  
     ##
-    # Generates a unique guid to use as part of the acknowledgement named pipe name. 
-    # Stolen from EM.
+    # Process the message received from AMQP server
     #
-    def generate_guid
-      # Cache uuidgen seed for better performance
-      if @ix and @ix >= 10_000
-        @ix = nil
-        @seed = nil
+    # @param header<MQ::Header> The header of the message
+    # @param body Description
+    # 
+    def process_queue_message(header, body)
+      DefaultLogger.debug("MessageSubscriber#process_queue_message(header, body)")
+      body = MessageCoder.decode(body)
+      DefaultLogger.debug("---------------  #{body.inspect}")
+      unless body.is_a?(Hash)
+        message = { :msg_type => 'standard', :raw => body }
+      else
+        message = { :msg_type => 'standard' }.merge(body)
+      end
+      store_acknowledgement(message, header) if options[:ack]
+      write(message)
+    end
+  
+    private
+  
+      ##
+      # Reset the instance variables used to store message statistics
+      #
+      def reset_message_statistics
+        @count = 0
+        @prev_time = Time.now
+      end
+    
+      ##
+      # Calculate the statistics on message handling
+      #
+      def calculate_message_statistics
+        @time_delta = Time.now - @prev_time
+        @mps  = @count  / @time_delta
+        reset_message_statistics
       end
 
-      # NB. This will only work on *nix platforms
-      @seed ||= `uuidgen`.chomp.gsub(/-/,"")
-      @ix ||= 0
+      ##
+      # Create the acknowledgement named pipe
+      #
+      def create_sys_pipe
+        name = File.join(PIPE_PATH, "sys_pipe_#{generate_guid}")
+        `mkfifo #{name}`
+        @sys_pipe = File.new(name, "r+")
+        message = { :msg_type => "system", 
+                    :sys_pipe => name, 
+                    :ack => options[:ack] }
+        write_to_pipe(message)
+      end
+    
+      ##
+      # Destroy the acknowledgement named pipe
+      #
+      def destroy_sys_pipe
+        `rm #{@sys_pipe.path}`
+      end
 
-      "#{@seed}#{@ix += 1}"
-    end
+      ##
+      # Generates a unique guid to use as part of the acknowledgement named pipe name. 
+      # Stolen from EM.
+      #
+      def generate_guid
+        # Cache uuidgen seed for better performance
+        if @ix and @ix >= 10_000
+          @ix = nil
+          @seed = nil
+        end
+
+        # NB. This will only work on *nix platforms
+        @seed ||= `uuidgen`.chomp.gsub(/-/,"")
+        @ix ||= 0
+
+        "#{@seed}#{@ix += 1}"
+      end
 
-    ##
-    # Write the message to STDOUT
-    #
-    # @param message<Hash> The message to write
-    # 
-    def write(message)
-      DefaultLogger.debug("MessageSubscriber#write(message)")
-      @count += 1
-      write_to_pipe(message)
-    end
+      ##
+      # Write the message to STDOUT
+      #
+      # @param message<Hash> The message to write
+      # 
+      def write(message)
+        DefaultLogger.debug("MessageSubscriber#write(#{message.inspect})")
+        @count += 1
+        write_to_pipe(message)
+      end
 
-    ##
-    # Initialize the DNS Service Discovery (aka Bonjour, MDNS).
-    #
-    def initialize_dnssd
-      return unless options[:dnssd]
-      # FIXME. DNS-SD breaks when under load. Not sure what problem is yet, have raised it with
-      # gem author:
-      # http://github.com/tenderlove/dnssd/issues#issue/3
-      require 'dnssd'
-      DNSSD.register!("#{options[:name]}_pipe", "_http._tcp", nil, options[:http_port])
-    end
+      def write_to_pipe(message, pipe=$stdout)
+        pipe.syswrite(MessageCoder.encode(message) << "\n")
+      end
 
-    def write_to_pipe(message, pipe=$stdout)
-      pipe.syswrite(MessageCoder.encode(message) << "\n")
-    end
+  end
 
-end
+end