pipeline_toolkit 1.0.4 → 1.0.6

data/lib/pipeline_toolkit/commands/msg_probe/cli.rb

@@ -0,0 +1,46 @@
+require 'pp'
+require 'trollop'
+require 'eventmachine'
+
+Signal.trap('INT') do
+  # puts "\nStopping"
+  # TODO: complete signal trap interrupt
+end
+
+Signal.trap('TERM') do
+  # puts "\nStopping"
+  # TODO: complete signal trap terminate
+end
+
+module Commands # :nodoc:
+  module MsgProbe
+    class CLI
+      def self.execute(stdout, arguments=[])
+
+        opts = Trollop::options do
+          banner <<-EOL
+Message Probe
+------------------
+  TODO: description
+
+  Usage:
+  msg_probe [options]
+
+Examples:
+  msg_probe --name octane_message_formatter
+  msg_probe --name octane_message_formatter --http_port 10001 --interval 1
+
+Options:
+            EOL
+          opt :interval, "Time in seconds between updates", :short => "i", :default => 2
+          opt :http_port, "The port the HTTP server runs on. Default is a random port between 10000-11000", :type => :integer
+          opt :name, "The name of the probe. Used in monitoring", :type => :string, :short => 'n'
+          opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :short => 'd'
+          opt :env, "The environment to run (development, production)", :default => "development", :short => :e
+        end
+
+        MessageProbe.new(opts).start
+      end
+    end
+  end
+end

data/lib/pipeline_toolkit/commands/msg_push/cli.rb

@@ -0,0 +1,58 @@
+require 'pp'
+require 'trollop'
+require 'eventmachine'
+
+module Commands # :nodoc:
+  module MsgPush
+    class CLI
+      def self.execute(stdout, arguments=[])
+
+        opts = Trollop::options do
+          banner <<-EOL
+Message Publisher
+------------------
+  Publish to a Message Queue server, the received message from the 
+  last worker in the pipeline.
+
+  Usage:
+  msg_push [options]
+
+Examples:
+  msg_push --exchange holding_tank
+  msg_push --exchange holding_tank --type topic \\
+            --queues us_stocks:stock.us.*, dax:stock.de.dax
+
+Options:
+            EOL
+
+          # Msg exchange  
+          opt :exchange, "Exchange name", :type => :string, :required => true, :short => :x
+          opt :type, "Exchange type (direct, fanout, topic)", :default => "fanout", :short => :t
+          opt :passive, "If true, the exchange will not be created if it does not already exist.", :default => false, :short => :s
+          opt :durable, "If true, the exchange will be marked as durable.", :default => false, :short => :d
+          
+          # Msg queues
+          opt :queues, "Destination queue(s) (e.g. queue1:routing_key, queue2:routing_key). Routing keys will be ignored if exchange type is not topic.", :type => :strings, :required => false, :short => :q
+          
+          # Msg server
+          opt :host, "AMQP message server host", :default => "localhost", :short => :h
+          opt :port, "AMQP message server port", :default => 5672, :short => :p
+          opt :user, "AMQP message server username", :default => "guest", :short => :u
+          opt :pass, "AMQP message server password", :default => "guest", :short => :w
+          opt :vhost, "AMQP message server vhost", :default => "/", :short => :v
+
+          opt :env, "Environment (development, production)", :default => "development", :short => :e
+        end
+                
+        # Change ['queue1:routing_key', ...] to [['queue1','routing_key'], ...]
+        queues = []
+        opts.delete(:queues).each do |queue| # trollop gives an array of strings
+          queues << queue.strip.split(":")
+        end
+        opts[:queues] = queues
+        
+        MessagePusher.new(opts).start
+      end
+    end
+  end
+end

data/lib/pipeline_toolkit/commands/msg_sink/cli.rb

@@ -0,0 +1,41 @@
+require 'rubygems'
+require 'trollop'
+
+Signal.trap('INT') do
+  # puts "\nStopping"
+  # TODO: complete signal trap interrupt
+end
+
+Signal.trap('TERM') do
+  # puts "\nStopping"
+  # TODO: complete signal trap terminate
+end
+
+module Commands # :nodoc:
+  module MsgSink
+    class CLI
+      def self.execute(stdout, arguments=[])
+
+        opts = Trollop::options do
+          banner <<-EOL
+Message Sink
+------------------
+  TODO: description
+
+  Usage:
+  msg_sink [options]
+
+Examples:
+  msg_sink 
+
+Options:
+            EOL
+
+          opt :env, "The environment to run (development, production)", :default => "development", :short => :e
+        end
+
+        MessageSink.new(opts).start
+      end
+    end
+  end
+end

data/lib/pipeline_toolkit/commands/msg_subscribe/cli.rb

@@ -0,0 +1,58 @@
+require 'trollop'
+require 'eventmachine'
+
+module Commands # :nodoc:
+  module MsgSubscribe # :nodoc:
+    
+    class CLI
+      def self.execute(stdout, arguments=[])
+
+        opts = Trollop::options do
+          banner <<-EOL
+Message Subscriber
+------------------
+  Subscribe to a Message Queue server using AMQP and pass messages to the next worker.
+
+  Usage:
+  msg_subscribe [options]
+
+Examples:
+  msg_subscribe --name octane_message_formatter --port 8000  \\
+                --queue us_stocks
+  msg_subscribe --exchange octane --type topic --queue us_stocks \\
+                --host www.domain.com --port 7000
+
+Options:
+            EOL
+
+          # Msg exchange  
+          opt :exchange, "The exchange name", :type => :string, :required => true, :short => :x
+          opt :type, "The exchange type (direct, fanout or topic)", :default => "fanout", :short => :t
+          opt :passive, "If set to true, the server will not create the exchange if it does not already exist.", :default => false, :short => :s
+          opt :durable, "If set to true, the exchange will be marked as durable.", :default => false, :short => :d
+
+          # Messages
+          opt :queue, "The destination queue (queue:routing_key). Routing keys will be ignored if exchange type is not topic.", :short => :q, :type => :string, :required => true
+          opt :ack, "If this field is set to false the server does not expect acknowledgments for messages.", :short => :a, :type => :boolean, :default => true
+
+          # Msg server
+          opt :host, "The AMQP message server host", :default => "localhost", :short => :h
+          opt :port, "The AMQP message server port", :default => 5672, :short => :p
+          opt :user, "The AMQP message server username", :default => "guest", :short => :u
+          opt :pass, "The AMQP message server username", :default => "guest", :short => :w
+          opt :vhost, "The AMQP message server vhost", :default => "/", :short => :v
+            
+          # Monitoring
+          opt :name, "The name used to describe the entire process chain", :type => :string, :short => :n  
+          opt :http_port, "The port the HTTP monitoring server runs on. Default is a random port between 10000-11000", :type => :integer, :short => :o
+          opt :content_type, "The type of response we are expecting from the HTTP monitoring server. (http, xml and json supported)", :type => :string, :default => "html", :short => :c
+          opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :type => :boolean, :default => false, :short => :b
+                      
+          opt :env, "The environment to run (development, production)", :default => "development", :short => :e
+        end
+
+        MessageSubscriber.new(opts).start
+      end
+    end
+  end
+end

data/lib/pipeline_toolkit/default_logger.rb

@@ -1,16 +1,131 @@
-require "logger"
-require 'syslog_logger'
+begin
+  require "logger"
+  require "fileutils"
+  require 'syslog_logger'
+rescue LoadError
+end
 
-# OPTIMIZE. Is there a better way to handle default/common/shared logging?
-module DefaultLogger
-  
-  # Return the logger for this class
-  def log
-    @logger ||= SyslogLogger.new(self.syslog_name)
+
+# replace for tidy timestamped logs
+class Logger
+  remove_method :format_message
+  def format_message(severity, timestamp, progname, msg)
+    "#{timestamp.strftime('%b%d %H:%M:%S')}  #{msg}\n" 
+  end
+end
+
+##
+# Default logger supports logging to both syslogger and logger. When running the 
+# logger in 'development' mode will cause the logger to log to the 
+# $HOME/pipeline_toolkit/logs directory. Alternativaly, in 'production' the logger 
+# logs to syslog. You can set the environment by passing the :env option with either
+# 'development' or 'production' when calling the {.init_logger} method. 
+#
+class DefaultLogger
+
+  ##
+  # Initialize a new logger
+  #
+  # @param options<Hash> Options hash of settings
+  # @option options [String] :env The environment to we initialize the logger for. 
+  # @option options [String] :log_name The name to start the log name with. ie. [log_name]_[env].log 
+  #
+  def self.init_logger(options = {})
+    options[:env] ||= "development"
+    options[:log_name] ||= "pipeline"
+    
+    if options[:env] == "development"
+      @@logger ||= Logger.new("#{log_directories}/#{options[:log_name]}_#{options[:env]}.log", 'daily')
+      @@logger.level = Logger::DEBUG
+    else
+      @@logger ||= SyslogLogger.new("#{options[:log_name]}_#{options[:env]}")
+      @@logger.level = Logger::INFO
+    end
+    
+    @@logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+  end
+
+  ##
+  # Setter for setting the logger to use
+  #
+  # @param log<Logger> The new logger to use
+  # 
+  def self.logger=(log)
+    @@logger = log
   end
   
-  def syslog_name
-    "pipeline"
+  ##
+  # Getter to return the logger 
+  #
+  def self.logger
+    @@logger
+  end
+
+  ##
+  # Setter for setting the log level to use
+  #
+  # @param new_level<Integer> The new log level to use
+  # 
+  def self.level=(new_level)
+    logger.level = new_level
+  end
+
+  ##
+  # Getter for the log level
+  #
+  def self.level
+    logger.level
+  end
+
+  ##
+  # Publish a debug log message
+  #
+  # @param message<String> The log message to publish
+  # 
+  def self.debug(message)
+    logger.debug(message)
   end
   
-end
+  ##
+  # Publish a info log message
+  #
+  # @param message<String> The log message to publish
+  #
+  def self.info(message)
+    logger.info(message)
+  end
+
+  ##
+  # Publish a warning log message
+  #
+  # @param message<String> The log message to publish
+  #
+  def self.warn(message)
+    logger.warn(message)
+  end
+
+  ##
+  # Publish a error log message
+  #
+  # @param message<String> The log message to publish
+  #
+  def self.error(message)
+    logger.error(message)
+  end
+
+  private
+
+    ##
+    # Ensure we have the log directories in the home path
+    #
+    def self.log_directories
+      # Ensure the log directory exist
+      dir = "#{ENV["HOME"]}/pipeline_toolkit/logs"
+      unless File.directory?(dir)
+        # Create log directory
+        FileUtils.makedirs(dir)
+      end
+      dir
+    end
+
+end

data/lib/pipeline_toolkit/handlers/message_handler.rb

@@ -0,0 +1,38 @@
+module Handlers # :nodoc:
+  
+  ##
+  # The Message Process handler that is called by Event Machine reactor loop 
+  # when it receives a new message from IO watcher
+  #
+  module MessageHandler
+
+    attr_reader :options
+    
+    ##
+    # Initialize a new instance
+    #
+    # @param msg_command<MessageCommand> An instance of the object to delegate the message handling to
+    # @param options<Hash> An options hash
+    # 
+    def initialize(target, options = {})
+      @options = options
+      @target = target
+    end
+    
+    ##
+    # The callback method that EventMachine calls as soon as a new message arrives
+    #
+    def notify_readable
+      DefaultLogger.debug("Handlers::MessageHandler#notify_readable") if options[:env] == "development"
+      message_coded = @io.gets
+      unless message_coded.nil?
+        DefaultLogger.debug("Raw message: #{message_coded}") if options[:env] == "development"
+        message = MessageCoder.decode(message_coded.chomp)
+        DefaultLogger.debug("Message: #{message}") if options[:env] == "development"
+        @target.process(message)
+      end
+    rescue StandardError => e # rescued here because main thread does not seem to see it
+      DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
+    end
+  end
+end

data/lib/pipeline_toolkit/message_coder.rb

@@ -1,18 +1,28 @@
-
-# Encodes messages
-# 
+##
+# Encode and decode messages using Marshal
+#
 class MessageCoder
-
-  def self.encode(msg)
+  
+  ##
+  # Encode the hash message
+  #
+  # @param message<Hash> The message as a hash 
+  # 
+  def self.encode(message)
     # NB: Using Marshal here because it's 9-10x faster than to_yaml
     # See http://gist.github.com/190849
-    str = Marshal.dump(msg)
+    str = Marshal.dump(message)
     str.gsub!("\n", '--\\n')
-    str
+    str 
   end
 
+  ##
+  # Decode the binary string into a hash
+  #
+  # @param str<String> Binary string
+  # 
   def self.decode(str)
-    str.gsub!('--\\n', "\n")
+    str.gsub!('--\\n', "\n") 
     Marshal.load(str)
   end
   

data/lib/pipeline_toolkit/message_command.rb

@@ -1,104 +1,181 @@
 require "rubygems"
 require "eventmachine"
 
+##
+# Provide abstract base functionality for pipeline machines.
+#
+# Required, To implement a your own machine, override the following methods:
+# 
+# * include MessageCommand
+# * {#process_standard} Handle the messages 
+#
+# Optionally, your machine can implement the following:
+#
+# * {#initialize_machine} Setup your machine
+# * {#report_back} Notify on the acknowledgement pipe about your worker
+#
+# @example An example of a pipeline machine
+#
+#    class DefaultMachine
+#      include MessageCommand
+#    
+#      def report_back
+#        
+#      end
+#    
+#      def process_standard(message)
+#        # do some work ie. transform the message or apply some business rules
+#        # raise exception if processing or business rule fails
+#        pass_on(message)
+#      rescue
+#        # handle the storage of the message that could not be processed
+#        # and acknowledge the AMQP server server
+#        acknowledge(message)
+#      end
+#    end  
+#    
+#    DefaultMachine.new.start
+#
+#
+# @abstract 
+#
 module MessageCommand
-  include DefaultLogger
   
-  attr_reader :sys_pipe
-
+  ##
+  # The options the machine should use
+  #
+  attr_reader :options
+  
+  ##
+  # Initialize a new instance. A message command can only 
+  # be initialized through the implementation instance.
+  #
+  # @param options<Hash> An options hash.
+  # 
+  def initialize(options = {})
+    DefaultLogger.init_logger(options)
+    @options = options
+  end
+  
+  ##
+  # Starts the machine up, ie. creates a new eventmachine reactor loop 
+  # and starts watching the STDIN for new messages
+  #
   def start
-    log.info("starting")
-
-    Signal.trap('INT') { EM.stop }
-    Signal.trap('TERM'){ EM.stop }
+    DefaultLogger.debug("MessageCommand#start") if options[:env] == "development"
 
-    @ack_buffer ||= ""
+    Signal.trap('INT')  { EM.stop }
+    Signal.trap('TERM') { EM.stop }
 
     begin
       EM.run do
-        conn = EM.watch($stdin, ProcessLine, self)
+        conn = EM.watch($stdin, Handlers::MessageHandler, self, options)
+        # must call this to setup callback to notify_readable
         conn.notify_readable = true
-        self.init_loop
+
+        initialize_machine
       end
     rescue StandardError => e
-      log.info e
+      DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
       raise e
     ensure
-      self.shutdown
+      shutdown
     end
   end
   
+  ##
+  # Stops the machine
+  #
   def shutdown
-    log.info("shutting down")
-    @sys_pipe && @sys_pipe.close
+    DefaultLogger.info("Shutting down #{self.class.name}")
+    DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+    @ack_pipe.close if @ack_pipe
   end
   
-  def process_line(line)
-    msg = MessageCoder.decode(line)
-    
-    case msg[:msg_type]
-    when :system
-      result = process_system(msg)
-    else
-      result = process_message(msg)
-    end
-    
-    case result
-    when :ack
-      self.ack_msg(msg)
+  ##
+  # Callback for the Handlers::MessageHandler when it receives a message
+  #
+  # @param message<Hash> The decoded message
+  # 
+  def process(message)
+    DefaultLogger.debug("MessageCommand#process(message)") if options[:env] == "development"
+    if message[:msg_type] == :system
+      process_system(message)
     else
-      self.pass_on_msg(result)
+      process_standard(message)
     end
   end
   
-  def ack_msg(msg)
-    return unless @use_ack
-    msg = {:msg_type => :ack, :ack_id => msg.ack_id}
-    @sys_pipe.syswrite(MessageCoder.encode(msg) << "\n")
+  ##
+  # Notify the AMQP server server that we've handled the message
+  #
+  # @param message<Hash> The message to acknowledge
+  #
+  def acknowledge(message)
+    DefaultLogger.debug("MessageCommand#acknowledge(message)") if options[:env] == "development"
+    ack_message = {:msg_type => :ack, :ack_id => message[:ack_id]}
+    write_to_pipe(ack_message, @ack_pipe)
   end
 
-  def pass_on_msg(msg)
-    $stdout.syswrite(MessageCoder.encode(msg) << "\n")
+  ##
+  # Pass the message to the next machine in the pipeline by writing the message to the STDOUT.
+  #
+  # @param message<Hash> The message to pass on
+  # 
+  def pass_on(message)
+    DefaultLogger.debug("MessageCommand#pass_on(message)") if options[:env] == "development"
+    write_to_pipe(message)
   end
 
-  # Override in included class. Provides a chance to initialize any
-  # code that needs to take place once the EM loop has started.
-  def init_loop
+  ##
+  # @abstract Override in the class the MessageCommand is included into. Provides a 
+  # chance to initialize any code that needs to take place once the EventMachine loop 
+  # has started.
+  #
+  def initialize_machine
+    DefaultLogger.debug("MessageCommand#initialize_machine") if options[:env] == "development"
     # Implemented in class that includes me
   end
 
-  # Can be overriden in included class. Called when the sys_pipe has been established. 
-  def syspipe_open
+  ##
+  # @abstract Can be overriden in the class the MessageCommand is included into. Called 
+  # when the acknowledgement named pipe has been established.
+  # 
+  def report_back
+    DefaultLogger.debug("MessageCommand#report_back") if options[:env] == "development"
     # Implemented in class that includes me
   end
 
-  # Override in included class. Processes a message. This method
-  # must return either a msg object -- which may or may not have been modified -- or the symbol :ack.
-  # Returning :ack mean that the message has been dealt with and can be acknowledged back to the queue 
-  # server. All messages must be acknowledged by at least one message_command.
-  def process_message(msg)
+  ##
+  # @abstract In your machine implementation you need to override the {#process_standard} method. 
+  # Processes a message. This method must call {#pass_on} if the message was handled successfully, 
+  # or if the handling of the message fails for whatever reason, you need to decide how you are going
+  # to handle the failure of the message (write message to error log | queue | etc.) and then 
+  # acknowledge the message by calling {acknowledge}. Acknowledging the message mean that the 
+  # message has been dealt with. All messages must be acknowledged by at least one worker in a pipeline.
+  # 
+  def process_standard(message)
+    DefaultLogger.debug("MessageCommand#process_standard(message)") if options[:env] == "development"
     # Implemented in class that includes me
-    msg
+    pass_on(message)
   end
 
-  def process_system(msg)
-    @sys_pipe = File.open(msg.sys_pipe, "w")
-    @use_ack = msg.use_ack
-    @max_unackd = msg.max_unackd
-    msg
-    self.syspipe_open
+  ##
+  # Process the first system message to notify machines down the pipeline which named pipe to 
+  # use for acknowledgements.
+  #
+  # @param message<Hash> The system message
+  # 
+  def process_system(message)
+    DefaultLogger.debug("MessageCommand#process_system(message)") if options[:env] == "development"
+    options[:ack] = message[:ack] # inherit setting from upstream
+    @ack_pipe = File.open(message[:sys_pipe], "w") # open ack pipe (needs to already exist)
+    report_back
+    pass_on(message)
   end
 
-  module ProcessLine
-    include DefaultLogger
-    
-    def initialize(msg_command)
-      @msg_command = msg_command
-    end
-    
-    def notify_readable
-      @msg_command.process_line(@io.gets)
-    end
+  def write_to_pipe(message, pipe=$stdout)
+    pipe.syswrite(MessageCoder.encode(message) << "\n")
   end
 
 end