visfleet-pipeline_toolkit 0.1.0 → 0.2.0

data/README.rdoc

@@ -1,7 +1,54 @@
-= pipeline_toolkit
+Pipeline Toolkit
 
-Description goes here.
+Command line tools for processing messages by constructing a pipeline of workers. AMQP and Unix pipes are used to construct the pipeline. Messages are simple Hashs (serialized as YAML) so they can hold any values and change throughout the processing.
+
+Provides:
+- Processing acknowledgments, ensuring the a message is only disposed of once it has been successful processed
+- Performance. Messages are moved through the pipeline fast
+- Command line tools for:
+  - Subscribing to messages from an AMQP queue
+  - Pushing messages back onto an AMQP exchange
+  - Monitoring performance (see msg_probe)
+- A base module (MessageCommand) to include into your own classes to quickly make workers.
+
+== Install
+
+  > gem sources -a http://gems.github.com
+  > sudo gem install visfleet-pipeline_toolkit
+
+=== Dependancies
+
+It is assumed that you have:
+- An AMQP msg server to pop and push messages to (e.g. http://www.rabbitmq.com/)
+- A *nix system, such as Linux or Mac OS X.
+
+== Usage
+
+1. Create your worker
+
+  class MyWorker
+    include MessageCommand
+    
+    def process_message(msg)
+      # do stuff here
+      msg
+    end
+  end
+  
+  MyWorker.new.start
+
+2. Hook it up to the main pipe (i.e. the AMQP server)
+
+  > msg_subscribe.rb -q source | my_worker.rb | msg_push.rb -x dest
+
+You can learn more about the command line tools and what options are available by using their help command.
+  
+  > msg_subscriber --help
+  > msg_push --help
+  > msg_sink --help
+  > msg_probe --help
+
+
+  
 
-== Copyright
 
-Copyright (c) 2009 Aisha Fenton. See LICENSE for details.

data/Rakefile

@@ -9,7 +9,12 @@ begin
     gem.email = "labs@visfleet.com"
     gem.homepage = "http://github.com/visfleet/pipeline_toolkit"
     gem.authors = ["Aisha Fenton"]
-    gem.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_pop.rb", "msg_sink.rb"]
+    gem.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_sink.rb"]
+    gem.add_runtime_dependency('tmm1-amqp')
+    gem.add_runtime_dependency('trollop')
+    gem.add_runtime_dependency('eventmachine')
+    # FIXME. Want to remove dependency on this for public version
+    gem.add_runtime_dependency('visfleet-default_logging')
   end
 
 rescue LoadError
@@ -36,7 +41,6 @@ rescue LoadError
   end
 end
 
-
 task :default => :test
 
 require 'rake/rdoctask'

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/pipeline_toolkit.rb

@@ -1,4 +1,4 @@
-require "pipeline_toolkit/central_logger"
+require "default_logging"
 require "pipeline_toolkit/message_coder"
 require "pipeline_toolkit/message_command"
 require "pipeline_toolkit/message_probe"

data/lib/pipeline_toolkit/message_pusher.rb

@@ -23,7 +23,7 @@ class MessagePusher
     @exchanges = []
     @exchange_names.each do |name, type|
       type ||= :fanout
-      @exchanges << MQ::Exchange.new(@msg_server, type.to_sym, name)
+      @exchanges << MQ::Exchange.new(@msg_server, type.to_sym, name, :durable => true, :passive => false)
     end
   end
 
@@ -32,9 +32,9 @@ class MessagePusher
     self.extend eval(classify(key_file.gsub(".rb", "")))
   end
   
-  # Stolen from rails.
+  # Turn path Class or Module name (i.e. strip directories and turn into camel-case)
   def classify(str)
-    str.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    str.gsub(/^.*\//, '').gsub(".rb","").gsub(/(?:^|_)(.)/) { $1.upcase }
   end
   
   # is overriden by included fork_file if specified
@@ -45,8 +45,8 @@ class MessagePusher
   def process_message(msg)
     @exchanges.each do |exchange|
       key = route_key(msg)
-      exchange.publish(MessageCoder.encode(msg), :routing_key => key)
-      # exchange.publish(msg.to_yaml, :routing_key => key)
+      exchange.publish(msg.to_yaml, :routing_key => key)
+      # OPTIMIZE. Using MessageCoder.encode(msg) instead of to_yaml is 2x faster. But won't be easy to debug. Worth it?
     end
     :ack
   end

data/lib/pipeline_toolkit/message_subscriber.rb

@@ -9,7 +9,7 @@ class MessageSubscriber
   PIPE_PATH = "/tmp"
   
   def initialize(opts)
-    @exchange_name = opts[:exchange]
+    @exchange_name, @exchange_type = opts[:exchange].split(":")
     @queue_name = opts[:queue]
     @use_ack = opts[:ack]
     @topic = opts[:topic]
@@ -56,12 +56,16 @@ class MessageSubscriber
     @queue = @queue_name ? MQ.queue(@queue_name, :durable => true) : 
                            self.generate_temporary_queue
     if @exchange_name
-      log.info("Binding to exchange:#{@exchange_name} #{@topic ? "using topic:" + @topic : ""}")
-      # OPTIMIZE. Should we be setting up the exchange just incase it hasn't been created yet?
-      @queue.bind(@exchange_name, :key => @topic)
+      create_exchange(@exchange_name, (@exchange_type || :fanout))
+      log.info("Binding to exchange:#{@exchange_str} #{@topic ? "using topic:" + @topic : ""}")
+      @queue.bind(@exchange_str, :key => @topic)
     end
   end
 
+  def create_exchange(name, type)
+    MQ::Exchange.new(MQ.default, type.to_sym, @exchange_name, :durable => true, :passive => false)    
+  end
+
   def create_sys_pipe
     log.debug("creating sys-pipe")
     name = File.join(PIPE_PATH, "sys_pipe_#{self.generate_guid}")

data/pipeline_toolkit.gemspec

@@ -2,25 +2,22 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pipeline_toolkit}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aisha Fenton"]
-  s.date = %q{2009-09-23}
+  s.date = %q{2009-09-26}
   s.email = %q{labs@visfleet.com}
-  s.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_pop.rb", "msg_sink.rb"]
+  s.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_sink.rb"]
   s.extra_rdoc_files = [
-    "LICENSE",
-     "README.rdoc"
+    "README.rdoc"
   ]
   s.files = [
     ".document",
      ".gitignore",
-     "LICENSE",
      "README.rdoc",
      "Rakefile",
      "VERSION",
-     "bin/msg_pop.rb",
      "bin/msg_probe.rb",
      "bin/msg_push.rb",
      "bin/msg_sink.rb",
@@ -28,7 +25,6 @@ Gem::Specification.new do |s|
      "lib/pipeline_toolkit.rb",
      "lib/pipeline_toolkit/message_coder.rb",
      "lib/pipeline_toolkit/message_command.rb",
-     "lib/pipeline_toolkit/message_popper.rb",
      "lib/pipeline_toolkit/message_probe.rb",
      "lib/pipeline_toolkit/message_pusher.rb",
      "lib/pipeline_toolkit/message_sink.rb",
@@ -50,8 +46,20 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<tmm1-amqp>, [">= 0"])
+      s.add_runtime_dependency(%q<trollop>, [">= 0"])
+      s.add_runtime_dependency(%q<eventmachine>, [">= 0"])
+      s.add_runtime_dependency(%q<visfleet-default_logging>, [">= 0"])
     else
+      s.add_dependency(%q<tmm1-amqp>, [">= 0"])
+      s.add_dependency(%q<trollop>, [">= 0"])
+      s.add_dependency(%q<eventmachine>, [">= 0"])
+      s.add_dependency(%q<visfleet-default_logging>, [">= 0"])
     end
   else
+    s.add_dependency(%q<tmm1-amqp>, [">= 0"])
+    s.add_dependency(%q<trollop>, [">= 0"])
+    s.add_dependency(%q<eventmachine>, [">= 0"])
+    s.add_dependency(%q<visfleet-default_logging>, [">= 0"])
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: visfleet-pipeline_toolkit
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Aisha Fenton
@@ -9,31 +9,66 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-23 00:00:00 -07:00
+date: 2009-09-26 00:00:00 -07:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: tmm1-amqp
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: trollop
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: visfleet-default_logging
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 description: 
 email: labs@visfleet.com
 executables: 
 - msg_probe.rb
 - msg_subscribe.rb
 - msg_push.rb
-- msg_pop.rb
 - msg_sink.rb
 extensions: []
 
 extra_rdoc_files: 
-- LICENSE
 - README.rdoc
 files: 
 - .document
 - .gitignore
-- LICENSE
 - README.rdoc
 - Rakefile
 - VERSION
-- bin/msg_pop.rb
 - bin/msg_probe.rb
 - bin/msg_push.rb
 - bin/msg_sink.rb
@@ -41,7 +76,6 @@ files:
 - lib/pipeline_toolkit.rb
 - lib/pipeline_toolkit/message_coder.rb
 - lib/pipeline_toolkit/message_command.rb
-- lib/pipeline_toolkit/message_popper.rb
 - lib/pipeline_toolkit/message_probe.rb
 - lib/pipeline_toolkit/message_pusher.rb
 - lib/pipeline_toolkit/message_sink.rb

data/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2009 Aisha Fenton
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/msg_pop.rb

@@ -1,21 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'trollop'
-require 'pipeline_toolkit'
-require "pp"
-
-opts = Trollop::options do
-  opt :sources, "Sources", :short => "s", :type => :strings
-  opt :cycle, "Seconds between polls", :default => 1
-  opt :ack, "Switch that requires that messages are successfully processed before continuing with the next message", :short => "a"
-  
-  # Msg server
-  opt :host, "The AMQP message server host", :default => "localhost"
-  opt :port, "The AMQP message server port", :default => "5672"
-  opt :user, "The AMQP message server username", :default => "guest"
-  opt :pass, "The AMQP message server username", :default => "guest", :short => "w"
-  opt :vhost, "The AMQP message server vhost", :default => "/"
-end
-
-MessagePopper.new(opts).start

data/lib/pipeline_toolkit/message_popper.rb

@@ -1,87 +0,0 @@
-require "eventmachine"
-require "bunny"
-require "time"
-
-class MessagePopper
-  include DefaultLogging
-   
-  PIPE_PATH = "/tmp"
-  
-  def initialize(opts)
-    @cycle = opts[:cycle]
-    @msg_server = Bunny.new({:spec => '08'}.merge!(opts.select_keys(:host, :port, :user, :pass, :vhost)))
-    @msg_server.start
-    @sources = opts[:sources]
-    @use_ack = opts[:ack]
-  end
-
-  def start
-    self.create_sys_pipe
-    EM.run do
-      EM.add_periodic_timer(@cycle) { self.tick }
-    end
-    self.destroy_sys_pipe
-  end
-
-  def create_sys_pipe
-    log.debug("creating sys-pipe")
-    name = File.join(PIPE_PATH, "sys_pipe_#{self.generate_guid}")
-    `mkfifo #{name}`
-    @sys_pipe = File.new(name, "r+")
-    sleep(2)
-    $stdout.puts(MessageCoder.encode({:msg_type => :system, :sys_pipe => name, :use_ack => @use_ack}))
-    $stdout.flush
-  end
-
-  def destroy_sys_pipe
-    `rm #{@sys_pipe}`
-  end
-  
-  def generate_guid
-    # TODO maybe better way to do guid 
-    "#{Time.now.iso8601}_#{rand(99999)}_#{rand(99999)}_#{rand(99999)}"
-  end
-
-  def tick
-    log.debug("draining queue")
-    loop do
-      q_empty = true
-      @sources.each do |source|
-        # FIXME. @queue for each source, or not needed
-        @queue ||= @msg_server.queue(source)
-        result = @queue.pop(:ack => @use_ack)
-        next if result == :queue_empty
-        # msg = YAML.load(result)
-        q_empty = false
-        # @use_ack ? verify(msg, source) { write_msg(msg) } : write_msg(msg)
-        
-        @count ||= 0
-        @count += 1
-        @prev_time ||= Time.now
-        if @count > 100
-          t = Time.now
-          log.debug("per sec #{@count / (t - @prev_time)}")
-          @prev_time = t
-          @count = 0
-        end
-        
-      end
-      break if q_empty
-    end
-  end
-  
-  def write_msg(msg)
-    $stdout.puts(MessageCoder.encode(msg))
-    $stdout.flush
-  end
-  
-  def verify(msg, source)
-    msg.ack_id = generate_guid
-    yield
-    $stdout.flush
-    ack_id = @sys_pipe.gets.chomp!
-    raise Exception.new("ACK failed with msg #{msg.to_yaml}") unless msg.ack_id == ack_id
-    @msg_server.queue(source).ack
-  end
-    
-end