visfleet-pipeline_toolkit 0.0.1 → 0.1.0

data/Rakefile

@@ -5,11 +5,11 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name = "pipeline_toolkit"
-    gem.summary = %Q{TODO}
-    gem.email = "aisha.fenton@visfleet.com"
+    gem.summary = %Q{Toolkit for building processing pipelines using Unix Pipes and AMQP messages}
+    gem.email = "labs@visfleet.com"
     gem.homepage = "http://github.com/visfleet/pipeline_toolkit"
     gem.authors = ["Aisha Fenton"]
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+    gem.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_pop.rb", "msg_sink.rb"]
   end
 
 rescue LoadError

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/{monitor.rb → msg_probe.rb}

@@ -8,4 +8,4 @@ opts = Trollop::options do
   opt :interval, "Interval", :short => "i", :default => 100
 end
 
-MessageMonitor.new(opts).start
+MessageProbe.new(opts).start

data/bin/msg_push.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'pipeline_toolkit'
+
+opts = Trollop::options do
+  opt :exchanges, "The destination exchange(s)", :short => "x", :type => :strings
+  opt :key_eval, "A string of ruby code that is evaluated to produce a routing key for a given message. 
+                  By default each message has no routing key", :short => "e", :type => :string
+  opt :key_file, "A ruby file that gets included which contains a custom route_key method", :short => "f", :type => :string
+  
+  # Msg server
+  opt :host, "The AMQP message server host", :default => "localhost"
+  opt :port, "The AMQP message server port", :default => "5672"
+  opt :user, "The AMQP message server username", :default => "guest"
+  opt :pass, "The AMQP message server username", :default => "guest"
+  opt :vhost, "The AMQP message server vhost", :default => "/"
+end
+
+mp = MessagePusher.new(opts)
+# FIXME. Should be in MessageCommand class
+Signal.trap('INT') { mp.stop }
+Signal.trap('TERM'){ mp.stop }
+mp.start

data/bin/msg_sink.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'pipeline_toolkit'
+
+# opts = Trollop::options do
+#   opt :interval, "Interval", :short => "i", :default => 100
+# end
+
+MessageSink.new.start

data/bin/{subscribe.rb → msg_subscribe.rb}

@@ -5,9 +5,11 @@ require 'trollop'
 require 'pipeline_toolkit'
 
 opts = Trollop::options do
-  opt :sources, "Sources", :short => "s", :type => :strings
+  opt :exchange, "The exchange to subscribe to", :short => "x", :type => :string
+  opt :queue, "The source queue", :short => "q", :type => :string
   opt :ack, "Switch that requires that messages are successfully processed before continuing with the next message", :short => "a"
-  opt :queue_name, "The name that the created queue is given", :short => "q", :type => :string
+  opt :topic, "The queue topic to subscribe to", :short => "t", :type => :string
+  opt :max_unackd, "The maximum number of unaknowledged messages to buffer before waiting for them to be acknowledged. Defaults to 100", :default => 100
   
   # Msg server
   opt :host, "The AMQP message server host", :default => "localhost"
@@ -17,7 +19,4 @@ opts = Trollop::options do
   opt :vhost, "The AMQP message server vhost", :default => "/"
 end
 
-ms = MessageSubscriber.new(opts)
-Signal.trap('INT') { ms.stop }
-Signal.trap('TERM'){ ms.stop }
-ms.start
+ms = MessageSubscriber.new(opts).start

data/lib/pipeline_toolkit.rb

@@ -0,0 +1,9 @@
+require "pipeline_toolkit/central_logger"
+require "pipeline_toolkit/message_coder"
+require "pipeline_toolkit/message_command"
+require "pipeline_toolkit/message_probe"
+require "pipeline_toolkit/message_popper"
+require "pipeline_toolkit/message_pusher"
+require "pipeline_toolkit/message_subscriber"
+require "pipeline_toolkit/message_sink"
+require "pipeline_toolkit/open_hash"

data/lib/pipeline_toolkit/message_coder.rb

@@ -0,0 +1,19 @@
+
+# Encodes messages
+# 
+class MessageCoder
+
+  def self.encode(msg)
+    # NB: Using Marshal here because it's 9-10x faster than to_yaml
+    # See http://gist.github.com/190849
+    str = Marshal.dump(msg)
+    str.gsub!("\n", '--\\n')
+    str
+  end
+
+  def self.decode(str)
+    str.gsub!('--\\n', "\n")
+    Marshal.load(str)
+  end
+  
+end

data/lib/pipeline_toolkit/message_command.rb

@@ -0,0 +1,96 @@
+require "rubygems"
+require "eventmachine"
+
+module MessageCommand
+  include DefaultLogging
+  
+  attr_reader :sys_pipe
+
+  def start
+    log.info("starting")
+
+    Signal.trap('INT') { EM.stop }
+    Signal.trap('TERM'){ EM.stop }
+
+    @ack_buffer ||= ""
+
+    begin
+      EM.run do
+        self.init_loop
+        EM.attach($stdin, ProcessLine, self)
+      end
+    rescue StandardError => e
+      log.info e
+      raise e
+    ensure
+      self.shutdown
+    end
+  end
+  
+  def shutdown
+    log.info("shutting down")
+    @sys_pipe && @sys_pipe.close
+  end
+  
+  def process_line(line)
+    msg = MessageCoder.decode(line)
+    
+    case msg[:msg_type]
+    when :system
+      result = process_system(msg)
+    else
+      result = process_message(msg)
+    end
+    
+    case result
+    when :ack
+      self.ack_msg(msg)
+    else
+      self.pass_on_msg(result)
+    end
+  end
+  
+  def ack_msg(msg)
+    return unless @use_ack
+    @sys_pipe.syswrite(msg.ack_id + "\n")
+  end
+
+  def pass_on_msg(msg)
+    $stdout.syswrite(MessageCoder.encode(msg) << "\n")
+  end
+
+  # Override in included class. Provides a chance to initialize any
+  # code that needs to take place once the EM loop has started.
+  def init_loop
+    # Implemented in class that includes me
+  end
+
+  # Override in included class. Processes a message. This method
+  # must return either a msg object -- which may or may not have been modified -- or the symbol :ack.
+  # Returning :ack mean that the message has been dealt with and can be acknowledged back to the queue 
+  # server. All messages must be acknowledged by at least one message_command.
+  def process_message(msg)
+    # Implemented in class that includes me
+    msg
+  end
+
+  def process_system(msg)
+    @sys_pipe = File.open(msg.sys_pipe, "w")
+    @use_ack = msg.use_ack
+    @max_unackd = msg.max_unackd
+    msg
+  end
+
+  module ProcessLine
+    include DefaultLogging
+    
+    def initialize(msg_command)
+      @msg_command = msg_command
+    end
+    
+    def notify_readable
+      @msg_command.process_line($stdin.gets)
+    end
+  end
+
+end

data/lib/pipeline_toolkit/message_popper.rb

@@ -0,0 +1,87 @@
+require "eventmachine"
+require "bunny"
+require "time"
+
+class MessagePopper
+  include DefaultLogging
+   
+  PIPE_PATH = "/tmp"
+  
+  def initialize(opts)
+    @cycle = opts[:cycle]
+    @msg_server = Bunny.new({:spec => '08'}.merge!(opts.select_keys(:host, :port, :user, :pass, :vhost)))
+    @msg_server.start
+    @sources = opts[:sources]
+    @use_ack = opts[:ack]
+  end
+
+  def start
+    self.create_sys_pipe
+    EM.run do
+      EM.add_periodic_timer(@cycle) { self.tick }
+    end
+    self.destroy_sys_pipe
+  end
+
+  def create_sys_pipe
+    log.debug("creating sys-pipe")
+    name = File.join(PIPE_PATH, "sys_pipe_#{self.generate_guid}")
+    `mkfifo #{name}`
+    @sys_pipe = File.new(name, "r+")
+    sleep(2)
+    $stdout.puts(MessageCoder.encode({:msg_type => :system, :sys_pipe => name, :use_ack => @use_ack}))
+    $stdout.flush
+  end
+
+  def destroy_sys_pipe
+    `rm #{@sys_pipe}`
+  end
+  
+  def generate_guid
+    # TODO maybe better way to do guid 
+    "#{Time.now.iso8601}_#{rand(99999)}_#{rand(99999)}_#{rand(99999)}"
+  end
+
+  def tick
+    log.debug("draining queue")
+    loop do
+      q_empty = true
+      @sources.each do |source|
+        # FIXME. @queue for each source, or not needed
+        @queue ||= @msg_server.queue(source)
+        result = @queue.pop(:ack => @use_ack)
+        next if result == :queue_empty
+        # msg = YAML.load(result)
+        q_empty = false
+        # @use_ack ? verify(msg, source) { write_msg(msg) } : write_msg(msg)
+        
+        @count ||= 0
+        @count += 1
+        @prev_time ||= Time.now
+        if @count > 100
+          t = Time.now
+          log.debug("per sec #{@count / (t - @prev_time)}")
+          @prev_time = t
+          @count = 0
+        end
+        
+      end
+      break if q_empty
+    end
+  end
+  
+  def write_msg(msg)
+    $stdout.puts(MessageCoder.encode(msg))
+    $stdout.flush
+  end
+  
+  def verify(msg, source)
+    msg.ack_id = generate_guid
+    yield
+    $stdout.flush
+    ack_id = @sys_pipe.gets.chomp!
+    raise Exception.new("ACK failed with msg #{msg.to_yaml}") unless msg.ack_id == ack_id
+    @msg_server.queue(source).ack
+  end
+    
+end

data/lib/pipeline_toolkit/message_probe.rb

@@ -0,0 +1,24 @@
+class MessageProbe
+  include MessageCommand
+
+  def initialize(opts)
+    @interval = opts.interval
+    reset
+  end
+  
+  def process_message(msg)
+    @count += 1
+    if @count > @interval
+      delta = Time.now - @prev_time
+      log.info("#{@count / delta} msgs per second")
+      reset
+    end
+    msg
+  end
+    
+  def reset
+    @count = 0
+    @prev_time = Time.now
+  end
+    
+end

data/lib/pipeline_toolkit/message_pusher.rb

@@ -0,0 +1,54 @@
+require "mq"
+
+class MessagePusher 
+  include MessageCommand
+
+  def initialize(opts)
+    @key_eval = opts.key_eval
+    if opts.key_file
+      @key_file = opts.key_file
+      load_route(@key_file)
+      self.init_route
+    end
+    @exchange_names = opts.exchanges.map { |str| str.split(":") }
+    @msg_server_config = opts.select_keys(:host, :port, :user, :pass, :vhost)
+  end
+
+  def init_loop
+    @msg_server = MQ.new(AMQP.connect(@msg_server_config))
+    self.setup_exchanges
+  end
+
+  def setup_exchanges
+    @exchanges = []
+    @exchange_names.each do |name, type|
+      type ||= :fanout
+      @exchanges << MQ::Exchange.new(@msg_server, type.to_sym, name)
+    end
+  end
+
+  def load_route(key_file)
+    require key_file
+    self.extend eval(classify(key_file.gsub(".rb", "")))
+  end
+  
+  # Stolen from rails.
+  def classify(str)
+    str.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+  end
+  
+  # is overriden by included fork_file if specified
+  def route_key(msg)
+    @key_eval ? eval(@key_eval) : nil
+  end
+
+  def process_message(msg)
+    @exchanges.each do |exchange|
+      key = route_key(msg)
+      exchange.publish(MessageCoder.encode(msg), :routing_key => key)
+      # exchange.publish(msg.to_yaml, :routing_key => key)
+    end
+    :ack
+  end
+
+end

data/lib/pipeline_toolkit/message_sink.rb

@@ -0,0 +1,8 @@
+class MessageSink
+  include MessageCommand
+  
+  def process_message(msg)
+    :ack
+  end
+    
+end

data/lib/pipeline_toolkit/message_subscriber.rb

@@ -0,0 +1,134 @@
+require "eventmachine"
+require "mq"
+require "time"
+require "socket"
+
+class MessageSubscriber
+  include DefaultLogging
+   
+  PIPE_PATH = "/tmp"
+  
+  def initialize(opts)
+    @exchange_name = opts[:exchange]
+    @queue_name = opts[:queue]
+    @use_ack = opts[:ack]
+    @topic = opts[:topic]
+    @msg_server_opts = opts.select_keys(:host, :port, :user, :pass, :vhost)
+    @unackd_msgs = {}
+    @max_unackd = opts[:max_unackd]
+  end
+
+  def start
+    Signal.trap('INT') { AMQP.stop{ EM.stop } }
+    Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+
+    begin
+      self.create_sys_pipe
+      AMQP.start(@msg_server_opts) do
+        # For ack to work appropriatly you must shutdown AMQP gracefully,
+        # otherwise all items in your queue will be returned
+        # FIXME. Doesn't shut down cleanly with these commands included. Why?
+
+        self.setup_queue
+        # NB. prefetch limits the amount of unknowledged messages that come down the pipe.
+        MQ.prefetch(@max_unackd)
+        @queue.subscribe(:ack => @use_ack) do |header, body|
+            self.process_msg(header, body)
+        end
+        
+        EM.attach(@sys_pipe, HandleAcks, @sys_pipe, @unackd_msgs)
+      end
+    rescue StandardError => e
+      log.info e
+      raise e
+    ensure
+      self.shutdown
+    end
+  end
+
+  def shutdown
+    log.info "Shutting down"
+    self.destroy_sys_pipe
+  end
+
+  def setup_queue
+    # If a queue_name is given then we treat the queue as fixed, otherwise as temporary
+    @queue = @queue_name ? MQ.queue(@queue_name, :durable => true) : 
+                           self.generate_temporary_queue
+    if @exchange_name
+      log.info("Binding to exchange:#{@exchange_name} #{@topic ? "using topic:" + @topic : ""}")
+      # OPTIMIZE. Should we be setting up the exchange just incase it hasn't been created yet?
+      @queue.bind(@exchange_name, :key => @topic)
+    end
+  end
+
+  def create_sys_pipe
+    log.debug("creating sys-pipe")
+    name = File.join(PIPE_PATH, "sys_pipe_#{self.generate_guid}")
+    `mkfifo #{name}`
+    @sys_pipe = File.new(name, "r+")
+    $stdout.puts(MessageCoder.encode({:msg_type => :system, 
+                                      :sys_pipe => name,
+                                      :use_ack => @use_ack,
+                                      :max_unackd => @max_unackd}))
+    $stdout.flush
+  end
+
+  def destroy_sys_pipe
+    `rm #{@sys_pipe.path}`
+  end
+  
+  def generate_temporary_queue
+    qname = "#{Socket.gethostname}_#{self.generate_guid}"
+    log.debug("Binding temporary queue #{qname}")
+    MQ.queue(qname, :auto_delete => true, :durable => false)
+  end
+  
+  # Generates a guid. Stolen from EM.
+  def generate_guid
+    # Cache uuidgen seed for better performance
+    if @ix and @ix >= 10_000
+      @ix = nil
+      @seed = nil
+    end
+    
+    # NB. This will only work on *nix platforms
+    @seed ||= `uuidgen`.chomp.gsub(/-/,"")
+    @ix ||= 0
+
+    "#{@seed}#{@ix += 1}"
+  end
+    
+  def process_msg(header, body)
+    msg = YAML.load(body)
+    store_ack(msg, header) if @use_ack
+    write_msg(msg)
+  end
+  
+  def write_msg(msg)
+    $stdout.syswrite(MessageCoder.encode(msg) << "\n")
+  end
+  
+  def store_ack(msg, header)
+    msg.ack_id = header.delivery_tag.to_s
+    @unackd_msgs[msg.ack_id] = header
+  end
+
+  # Handles msg acks
+  module HandleAcks
+    include DefaultLogging
+
+    def initialize(sys_pipe, unackd_msgs)
+      @sys_pipe = sys_pipe
+      @unackd_msgs = unackd_msgs
+    end
+
+    def notify_readable
+      ack_id = @sys_pipe.gets.chomp!
+      header = @unackd_msgs.delete(ack_id)
+      header.ack
+    end
+  end
+    
+end
+

data/lib/pipeline_toolkit/open_hash.rb

@@ -0,0 +1,24 @@
+# Stolen from
+#   http://github.com/karottenreibe/ohash/
+module OpenHash
+  def method_missing(meth, *args)
+    method = meth.to_s
+
+    if method =~ %r{.+=$}
+      super unless args.length == 1
+      self[method[0...-1].to_sym] = args.first
+    else
+      self[meth]
+    end
+  end
+
+  def select_keys(*keys)
+    h = {}
+    self.each do |key, value|
+      h[key] = value if keys.include?(key)
+    end
+  end
+
+end
+ 
+Hash.send(:include, OpenHash)

data/pipeline_toolkit.gemspec

@@ -0,0 +1,57 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{pipeline_toolkit}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Aisha Fenton"]
+  s.date = %q{2009-09-23}
+  s.email = %q{labs@visfleet.com}
+  s.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_pop.rb", "msg_sink.rb"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "bin/msg_pop.rb",
+     "bin/msg_probe.rb",
+     "bin/msg_push.rb",
+     "bin/msg_sink.rb",
+     "bin/msg_subscribe.rb",
+     "lib/pipeline_toolkit.rb",
+     "lib/pipeline_toolkit/message_coder.rb",
+     "lib/pipeline_toolkit/message_command.rb",
+     "lib/pipeline_toolkit/message_popper.rb",
+     "lib/pipeline_toolkit/message_probe.rb",
+     "lib/pipeline_toolkit/message_pusher.rb",
+     "lib/pipeline_toolkit/message_sink.rb",
+     "lib/pipeline_toolkit/message_subscriber.rb",
+     "lib/pipeline_toolkit/open_hash.rb",
+     "perf_test/test_pop.rb",
+     "perf_test/test_pop_async.rb",
+     "perf_test/test_push.rb",
+     "pipeline_toolkit.gemspec"
+  ]
+  s.homepage = %q{http://github.com/visfleet/pipeline_toolkit}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Toolkit for building processing pipelines using Unix Pipes and AMQP messages}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: visfleet-pipeline_toolkit
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors: 
 - Aisha Fenton
@@ -9,17 +9,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-09 00:00:00 -07:00
+date: 2009-09-23 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
 description: 
 email: labs@visfleet.com
 executables: 
-- monitor.rb
-- subscribe.rb
-- push.rb
-- pop.rb
+- msg_probe.rb
+- msg_subscribe.rb
+- msg_push.rb
+- msg_pop.rb
+- msg_sink.rb
 extensions: []
 
 extra_rdoc_files: 
@@ -31,9 +32,25 @@ files:
 - LICENSE
 - README.rdoc
 - Rakefile
+- VERSION
+- bin/msg_pop.rb
+- bin/msg_probe.rb
+- bin/msg_push.rb
+- bin/msg_sink.rb
+- bin/msg_subscribe.rb
+- lib/pipeline_toolkit.rb
+- lib/pipeline_toolkit/message_coder.rb
+- lib/pipeline_toolkit/message_command.rb
+- lib/pipeline_toolkit/message_popper.rb
+- lib/pipeline_toolkit/message_probe.rb
+- lib/pipeline_toolkit/message_pusher.rb
+- lib/pipeline_toolkit/message_sink.rb
+- lib/pipeline_toolkit/message_subscriber.rb
+- lib/pipeline_toolkit/open_hash.rb
 - perf_test/test_pop.rb
 - perf_test/test_pop_async.rb
 - perf_test/test_push.rb
+- pipeline_toolkit.gemspec
 has_rdoc: false
 homepage: http://github.com/visfleet/pipeline_toolkit
 licenses: 

data/bin/push.rb

@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'trollop'
-require 'pipeline_toolkit'
-
-opts = Trollop::options do
-  opt :destinations, "Destinations", :short => "d", :type => :strings
-  opt :route_eval, "A string of ruby code that ", :short => "e", :type => :string
-  opt :route_file, "A string of ruby code that ", :short => "f", :type => :string
-  
-  # Msg server
-  opt :host, "The AMQP message server host", :default => "localhost"
-  opt :port, "The AMQP message server port", :default => "5672"
-  opt :user, "The AMQP message server username", :default => "guest"
-  opt :pass, "The AMQP message server username", :default => "guest"
-  opt :vhost, "The AMQP message server vhost", :default => "/"
-end
-
-MessagePusher.new(opts).start
-
-# pop -s raw | thingie | push -d  my_queue
-# pop -s raw | thingie | push -f "(msg.tmu_id.hash % NUMBER_OF_FORKS).to_s"
-# pop -s raw | thingie | push -f "Socket.gethostname"
-# pop -s raw | thingie | push -f "require route_to_site"