pipeline_toolkit 1.0.1 → 1.0.2

data/README.rdoc

@@ -1,7 +1,7 @@
 = Pipeline Toolkit
 by VisFleet
 
-Command line tools for processing messages by constructing a pipeline of workers. AMQP and Unix pipes are used to construct the pipeline. Messages are simple Hashs (serialized as YAML) so they can hold any values and change throughout the processing.
+Command line tools for processing messages by constructing a pipeline of workers. AMQP and Unix pipes are used to construct the pipeline. Messages are simple Hashes (serialized as YAML) so they can hold any values and change throughout the processing.
 
 Provides:
 - Processing acknowledgments, ensuring the a message is only disposed of once it has been successful processed
@@ -14,8 +14,7 @@ Provides:
 
 == Install
 
-  > gem sources -a http://gems.github.com
-  > sudo gem install visfleet-pipeline_toolkit
+  > sudo gem install pipeline_toolkit
 
 === Dependancies
 
@@ -37,7 +36,7 @@ It is assumed that you have:
   end  
   MyWorker.new.start
 
-2. Hook it up to the main pipe (i.e. the AMQP server)
+2. Hook it up to a AMQP message source
 
   > msg_subscribe.rb -q source | my_worker.rb | msg_push.rb -x dest
 
@@ -48,7 +47,24 @@ You can learn more about the command line tools and what options are available b
   > msg_sink --help
   > msg_probe --help
 
+== Examples
 
-  
+Generate a bunch of sample messages and push them to an AMQP exchange 'test-exchange'. Hook up a pipe to consume and push results to /dev/null. Goto http://localhost:9090 to see message throughput.
+
+  > msg_generator.rb | msg_push.rb -x test-exchange
+  > msg_subscribe.rb --http-port 9090 -x test-exchange -q test_queue > /dev/null
+
+Same as above, but this time with acknowledgements switched on. This guarantees that a message isn't removed from the message server until handled. msg_sink.rb acknowledges all messages.
+
+  > msg_generator.rb | msg_push.rb -x test-exchange
+  > msg_subscribe.rb -a -x test-exchange -q test_queue | msg_sink.rb
+
+== Todo
+
+- Logging. For the moment we're using SysLogLogger (which requires Syslog). This needs to be switched to the default ruby logging, and we need to find out a way to plug in alternatives if required.
+
+- DNS-SD is broken. See http://github.com/tenderlove/dnssd/issues#issue/3
+
+- Decide if we want to support other exchange, queue and binding configurations. For example, exchanges are always fanout and durable at the moment.
 
 

data/Rakefile

@@ -10,9 +10,11 @@ begin
     gem.homepage = "http://github.com/visfleet/pipeline_toolkit"
     gem.authors = ["Aisha Fenton"]
     gem.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_sink.rb", "msg_generator.rb"]
-    gem.add_runtime_dependency('amqp', ">=0.6.4")
+    gem.add_runtime_dependency('amqp', ">=0.6.5")
     gem.add_runtime_dependency('trollop', ">=1.14")
-    gem.add_runtime_dependency('eventmachine', ">=0.12.8")
+    gem.add_runtime_dependency('eventmachine', ">=0.12.10")
+    gem.add_runtime_dependency('eventmachine_httpserver', ">=0.2.0")
+    gem.add_runtime_dependency('SyslogLogger', ">=1.4.0")
   end
   Jeweler::GemcutterTasks.new  
 

data/VERSION

@@ -1 +1 @@
-1.0.1
+1.0.2

data/bin/msg_generator.rb

@@ -1,36 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'trollop'
-require 'pipeline_toolkit'
-
-opts = Trollop::options do
-  banner <<-EOS
-  Generates messages for testing the pipeline
-  Usage:
-        msg_generator.rb -m "My message"
-  EOS
-  opt :msg, "The message to send", :short => "m", :default => "Test message"
-  opt :delay, "Sleep time between sends in seconds", :short => "d", :type => :float
-end
-
-class MsgGenerator
-
-  def initialize(opts)
-    @delay = opts.delay
-    @msg = opts.msg
-  end
-
-  def start
-    loop do
-      self.send_msg(@msg)
-      sleep(@delay) unless @delay.nil?
-    end
-  end
-
-  def send_msg(msg)
-    puts MessageCoder.encode(@msg)
-    $stdout.flush
-  end
-end
-
-MsgGenerator.new(opts).start

data/bin/msg_probe.rb

@@ -6,8 +6,10 @@ require 'pipeline_toolkit'
 
 opts = Trollop::options do
   opt :interval, "Time in seconds between updates", :short => "i", :default => 2
-  opt :http_port, "The HTTP server port for accessing the stats", :default => 9070
+  opt :http_port, "The port the HTTP server runs on. Default is a random port between 10000-11000", :type => :integer
   opt :name, "The name of the probe. Used in monitoring", :type => :string, :short => 'n'
+  opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :short => 'd'
 end
 
-MessageProbe.new(opts).start
+MessageProbe.new(opts).start
+

data/bin/msg_subscribe.rb

@@ -4,11 +4,17 @@ require 'trollop'
 require 'pipeline_toolkit'
 
 opts = Trollop::options do
-  opt :exchange, "The exchange to subscribe to", :short => "x", :type => :string
-  opt :queue, "The source queue", :short => "q", :type => :string
+  # Monitoring
+  opt :name, "The name used to describe the entire process chain", :short => "n", :type => :string  
+  opt :http_port, "The port the HTTP monitoring server runs on. Default is a random port between 10000-11000", :type => :integer
+  opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :short => 'd'
+
+  # Messages
+  opt :queue, "The source queue", :short => "q", :type => :string, :required => true
   opt :ack, "Switch that requires that messages are successfully processed before continuing with the next message", :short => "a"
   opt :topic, "The queue topic to subscribe to", :short => "t", :type => :string
   opt :max_unackd, "The maximum number of unaknowledged messages to buffer before waiting for them to be acknowledged. Defaults to 100", :default => 100
+  opt :exchange_bind, "Binds the the queue to the specified exchange. The created exchange is (for now) fanout, durable, and not passive", :short => "x", :type => :string
   
   # Msg server
   opt :host, "The AMQP message server host", :default => "localhost"

data/lib/pipeline_toolkit.rb

@@ -6,3 +6,4 @@ require "pipeline_toolkit/message_pusher"
 require "pipeline_toolkit/message_subscriber"
 require "pipeline_toolkit/message_sink"
 require "pipeline_toolkit/open_hash"
+require "pipeline_toolkit/socket_util"

data/lib/pipeline_toolkit/default_logger.rb

@@ -10,9 +10,7 @@ module DefaultLogger
   end
   
   def syslog_name
-    "pipeline-" + Process.ppid.to_s
+    "pipeline"
   end
   
-end
-
-
+end

data/lib/pipeline_toolkit/message_command.rb

@@ -16,9 +16,9 @@ module MessageCommand
 
     begin
       EM.run do
-        self.init_loop
         conn = EM.watch($stdin, ProcessLine, self)
         conn.notify_readable = true
+        self.init_loop
       end
     rescue StandardError => e
       log.info e
@@ -53,7 +53,8 @@ module MessageCommand
   
   def ack_msg(msg)
     return unless @use_ack
-    @sys_pipe.syswrite(msg.ack_id + "\n")
+    msg = {:msg_type => :ack, :ack_id => msg.ack_id}
+    @sys_pipe.syswrite(MessageCoder.encode(msg) << "\n")
   end
 
   def pass_on_msg(msg)
@@ -66,6 +67,11 @@ module MessageCommand
     # Implemented in class that includes me
   end
 
+  # Can be overriden in included class. Called when the sys_pipe has been established. 
+  def syspipe_open
+    # Implemented in class that includes me
+  end
+
   # Override in included class. Processes a message. This method
   # must return either a msg object -- which may or may not have been modified -- or the symbol :ack.
   # Returning :ack mean that the message has been dealt with and can be acknowledged back to the queue 
@@ -80,6 +86,7 @@ module MessageCommand
     @use_ack = msg.use_ack
     @max_unackd = msg.max_unackd
     msg
+    self.syspipe_open
   end
 
   module ProcessLine

data/lib/pipeline_toolkit/message_probe.rb

@@ -6,10 +6,11 @@ class MessageProbe
 
   def initialize(opts)
     @interval = opts.interval
-    @http_port= opts.http_port
+    @http_port= opts.http_port || Socket.select_random_port(10_000, 11_000)
     @start_time = Time.now
     @name = opts.name
     self.reset
+    self.init_dnssd if opts.dnssd
   end
 
   def init_loop
@@ -17,10 +18,14 @@ class MessageProbe
     EM.add_periodic_timer(@interval) { self.tick }
   end
 
+  def init_dnssd
+    require 'dnssd'
+    DNSSD.register!("#{@name} probe", "_http._tcp", nil, @http_port)
+  end
+
   def tick
     @time_delta = Time.now - @prev_time
     @mps = @count / @time_delta
-    
     self.reset
   end
 
@@ -33,6 +38,8 @@ class MessageProbe
     Time.now - @start_time
   end
   
+  # OPTIMIZE. can improve performance by overriding base process_line method
+  # saving us the unnecessary marshal step.
   def process_message(msg)
     @count += 1
     msg
@@ -94,7 +101,7 @@ class ProbeHttpRequest < EM::Connection
               <td>messages per second:</td><td><span class="mps">#{@probe.mps}</span></td><td></td>
             </tr>
             <tr>
-              <td>uptime:</td><td><span class="uptime">#{@probe.uptime.to_i / 60}</span></td><td>mins</td
+              <td>uptime:</td><td><span class="uptime">#{@probe.uptime.to_i / 60}</span></td><td>mins</td>
             </tr>
           </table>
         </div>

data/lib/pipeline_toolkit/message_pusher.rb

@@ -1,6 +1,6 @@
 require "mq"
 
-class MessagePusher 
+class MessagePusher
   include MessageCommand
 
   def initialize(opts)
@@ -27,6 +27,12 @@ class MessagePusher
     end
   end
 
+  def syspipe_open
+    # Send back details of where messages are going
+    msg = {:msg_type => :pipe_desc, :exchanges => @exchange_names.join(",")}
+    self.sys_pipe.syswrite(MessageCoder.encode(msg) << "\n")
+  end
+
   def load_route(key_file)
     require key_file
     self.extend eval(classify(key_file.gsub(".rb", "")))
@@ -45,8 +51,9 @@ class MessagePusher
   def process_message(msg)
     @exchanges.each do |exchange|
       key = route_key(msg)
-      exchange.publish(msg.to_yaml, :routing_key => key)
-      # OPTIMIZE. Using MessageCoder.encode(msg) instead of to_yaml is 2x faster. But won't be easy to debug. Worth it?
+      exchange.publish(msg.to_yaml)
+      # OPTIMIZE. Using MessageCoder.encode(msg) instead of to_yaml is 2x faster. But won't be easy to
+      # debug. Worth it?
     end
     :ack
   end

data/lib/pipeline_toolkit/message_subscriber.rb

@@ -5,47 +5,76 @@ require "socket"
 
 class MessageSubscriber
   include DefaultLogger
-   
+  
+  attr_reader :name, :start_time, :mps, :queue_name
+  attr_accessor :exchanges_out
+
   PIPE_PATH = "/tmp"
   
   def initialize(opts)
-    @exchange_name, @exchange_type = opts[:exchange].split(":")
+    @exchange_name, @exchange_type = opts[:exchange_bind].split(":")
     @queue_name = opts[:queue]
     @use_ack = opts[:ack]
     @topic = opts[:topic]
     @msg_server_opts = opts.select_keys(:host, :port, :user, :pass, :vhost)
     @unackd_msgs = {}
     @max_unackd = opts[:max_unackd]
+
+    @http_port = opts.http_port || Socket.select_random_port(10_000, 11_000)
+    @name = opts.name || (Socket.gethostname + '_' + Process.pid.to_s)
+    self.init_dnssd if opts.dnssd
+    self.reset
+  end
+
+  def init_dnssd
+    # FIXME. DNS-SD breaks when under load. Not sure what problem is yet, have raised it with
+    # gem author:
+    # http://github.com/tenderlove/dnssd/issues#issue/3
+    require 'dnssd'
+    DNSSD.register!("#{@name}_pipe", "_http._tcp", nil, @http_port)
   end
 
   def start
-    Signal.trap('INT') { AMQP.stop{ EM.stop } }
-    Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+    # FIXME. Need to do more investigation on if/when messages are lost with shutdowns
+    Signal.trap('INT')  { AMQP.stop{ EM.stop } }
+    Signal.trap('TERM') { AMQP.stop{ EM.stop } }
 
     begin
       self.create_sys_pipe
       AMQP.start(@msg_server_opts) do
-        # For ack to work appropriatly you must shutdown AMQP gracefully,
-        # otherwise all items in your queue will be returned
-        # FIXME. Doesn't shut down cleanly with these commands included. Why?
-
+        @start_time = Time.now
         self.setup_queue
         # NB. prefetch limits the amount of unknowledged messages that come down the pipe.
         MQ.prefetch(@max_unackd)
         @queue.subscribe(:ack => @use_ack) do |header, body|
-            self.process_msg(header, body)
+          self.process_message(header, body)
         end
         
-        EM.attach(@sys_pipe, HandleAcks, @sys_pipe, @unackd_msgs)
+        conn = EM.watch(@sys_pipe, HandleCtlMessages, @sys_pipe, self)
+        conn.notify_readable = true
+        
+        EM.start_server('0.0.0.0', @http_port, PipeHttpRequest, self)
+        EM.add_periodic_timer(5) { self.calc_stats }
       end
     rescue StandardError => e
-      log.info e
+      log.error e.message + " " + e.backtrace.join("\n")
       raise e
     ensure
       self.shutdown
     end
   end
 
+  def calc_stats
+    @time_delta = Time.now - @prev_time
+    @mps  = @count  / @time_delta
+    self.reset
+  end
+
+  def reset
+    @count = 0
+    @prev_time = Time.now
+  end
+
   def shutdown
     log.info "Shutting down"
     self.destroy_sys_pipe
@@ -63,7 +92,7 @@ class MessageSubscriber
   end
 
   def create_exchange(name, type)
-    MQ::Exchange.new(MQ.default, type.to_sym, @exchange_name, :durable => true, :passive => false)    
+    MQ::Exchange.new(MQ.default, type.to_sym, @exchange_name, :durable => true, :passive => false)
   end
 
   def create_sys_pipe
@@ -103,34 +132,115 @@ class MessageSubscriber
     "#{@seed}#{@ix += 1}"
   end
     
-  def process_msg(header, body)
+  def process_message(header, body)
     msg = YAML.load(body)
     store_ack(msg, header) if @use_ack
     write_msg(msg)
   end
   
   def write_msg(msg)
+    @count += 1
     $stdout.syswrite(MessageCoder.encode(msg) << "\n")
   end
   
+  def perform_ack(ack_id)    
+    header = @unackd_msgs.delete(ack_id)
+    header.ack
+  end
+  
   def store_ack(msg, header)
     msg.ack_id = header.delivery_tag.to_s
     @unackd_msgs[msg.ack_id] = header
   end
 
   # Handles msg acks
-  module HandleAcks
+  module HandleCtlMessages
     include DefaultLogger
 
-    def initialize(sys_pipe, unackd_msgs)
+    def initialize(sys_pipe, msg_sub)
       @sys_pipe = sys_pipe
-      @unackd_msgs = unackd_msgs
+      @msg_sub = msg_sub
     end
 
     def notify_readable
-      ack_id = @sys_pipe.gets.chomp!
-      header = @unackd_msgs.delete(ack_id)
-      header.ack
+      msg = MessageCoder.decode(@sys_pipe.gets.chomp!)
+      case msg[:msg_type]
+      when :ack
+        @msg_sub.perform_ack(msg.ack_id)
+      when :pipe_desc
+        @msg_sub.exchanges_out = msg.exchanges
+      else
+        raise Exception.new("Unknown control message received: #{message}")
+      end
+    end
+  end
+
+  class PipeHttpRequest < EM::Connection
+    include EM::HttpServer
+
+    def initialize(msg_sub)
+      @msg_sub = msg_sub
+    end
+
+    def post_init
+      super
+      no_environment_strings
+    end
+
+    def process_http_request
+      response = EM::DelegatedHttpResponse.new(self)
+      response.status = 200
+      response.content_type 'text/html'
+      response.content = <<-EOL
+      <html>
+        <head>
+          <title>Message Probe</title>
+          <style type="text/css">
+            body {
+              background: black;
+              color: #80c0c0;
+            }
+            h1 {
+              font: 12pt Monospace;
+              text-align:center;
+            }
+            table {
+              font: 10pt Monospace;
+              margin-left:auto;
+              margin-right:auto;
+              text-align:right;
+            }
+            .page {
+              position:relative;
+              top: 20%;
+              # border-style:solid;
+              # border-width:5px;
+              width: 30%;
+              margin-left:auto;
+              margin-right:auto;
+            }
+          </style>
+        </head>
+        <body>
+          <div class=page>
+            <h1><span class="name">#{@msg_sub.name}</span></h1>
+            <table>
+              <tr>
+                <td>Structure:</td><td><span class="queue_in">#{@msg_sub.queue_name}</span> -> <span class="exchanges_out">#{@msg_sub.exchanges_out}</span></td>
+              </tr>
+              <tr>
+                <td>Throughput:</td><td><span class="mps">#{@msg_sub.mps.to_i}</span></td>
+              </tr>
+              <tr>
+                <td>Uptime:</td><td><span class="uptime">#{(Time.now - @msg_sub.start_time).to_i / 60}mins</span></td>
+              </tr>
+            </table>
+          </div>
+        </body>
+      </html>
+      EOL
+
+      response.send_response
     end
   end
     

data/lib/pipeline_toolkit/socket_util.rb

@@ -0,0 +1,24 @@
+
+# Extend socket class
+class Socket
+  
+  def self.select_random_port(low, high, host = "localhost")
+    raise Exception.new("'low' must be lower than 'high'") unless low < high
+    port = nil
+    begin
+      port = low + rand(high - low)
+    end while Socket.socket_in_use?(host, port)
+    port
+  end
+
+  # OPTIMIZE. Is there a better way to do this? Feels ugly to me.
+  def self.socket_in_use?(host, port)
+    begin
+      TCPSocket.new(host, port)
+    rescue Errno::ECONNREFUSED => e
+      return false
+    end
+    return true
+  end
+  
+end

data/pipeline_toolkit.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pipeline_toolkit}
-  s.version = "1.0.1"
+  s.version = "1.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aisha Fenton"]
-  s.date = %q{2009-12-06}
+  s.date = %q{2010-01-15}
   s.email = %q{labs@visfleet.com}
   s.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_sink.rb", "msg_generator.rb"]
   s.extra_rdoc_files = [
@@ -34,6 +34,7 @@ Gem::Specification.new do |s|
      "lib/pipeline_toolkit/message_sink.rb",
      "lib/pipeline_toolkit/message_subscriber.rb",
      "lib/pipeline_toolkit/open_hash.rb",
+     "lib/pipeline_toolkit/socket_util.rb",
      "monitor/munin.rb",
      "pipeline_toolkit.gemspec"
   ]
@@ -48,18 +49,24 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<amqp>, [">= 0.6.4"])
+      s.add_runtime_dependency(%q<amqp>, [">= 0.6.5"])
       s.add_runtime_dependency(%q<trollop>, [">= 1.14"])
-      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8"])
+      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.10"])
+      s.add_runtime_dependency(%q<eventmachine_httpserver>, [">= 0.2.0"])
+      s.add_runtime_dependency(%q<SyslogLogger>, [">= 1.4.0"])
     else
-      s.add_dependency(%q<amqp>, [">= 0.6.4"])
+      s.add_dependency(%q<amqp>, [">= 0.6.5"])
       s.add_dependency(%q<trollop>, [">= 1.14"])
-      s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
+      s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
+      s.add_dependency(%q<eventmachine_httpserver>, [">= 0.2.0"])
+      s.add_dependency(%q<SyslogLogger>, [">= 1.4.0"])
     end
   else
-    s.add_dependency(%q<amqp>, [">= 0.6.4"])
+    s.add_dependency(%q<amqp>, [">= 0.6.5"])
     s.add_dependency(%q<trollop>, [">= 1.14"])
-    s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
+    s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
+    s.add_dependency(%q<eventmachine_httpserver>, [">= 0.2.0"])
+    s.add_dependency(%q<SyslogLogger>, [">= 1.4.0"])
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: pipeline_toolkit
 version: !ruby/object:Gem::Version 
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors: 
 - Aisha Fenton
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-06 00:00:00 +13:00
+date: 2010-01-15 00:00:00 +13:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.6.4
+        version: 0.6.5
     version: 
 - !ruby/object:Gem::Dependency 
   name: trollop
@@ -40,7 +40,27 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.12.8
+        version: 0.12.10
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine_httpserver
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.2.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: SyslogLogger
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.0
     version: 
 description: 
 email: labs@visfleet.com
@@ -73,6 +93,7 @@ files:
 - lib/pipeline_toolkit/message_sink.rb
 - lib/pipeline_toolkit/message_subscriber.rb
 - lib/pipeline_toolkit/open_hash.rb
+- lib/pipeline_toolkit/socket_util.rb
 - monitor/munin.rb
 - pipeline_toolkit.gemspec
 has_rdoc: true