pipeline_toolkit 1.0.4 → 1.0.6

data/.gitignore

@@ -1,5 +0,0 @@
-*.sw?
-.DS_Store
-coverage
-rdoc
-pkg

data/README.rdoc

@@ -1,70 +0,0 @@
-= Pipeline Toolkit
-by VisFleet
-
-Command line tools for processing messages by constructing a pipeline of workers. AMQP and Unix pipes are used to construct the pipeline. Messages are simple Hashes (serialized as YAML) so they can hold any values and change throughout the processing.
-
-Provides:
-- Processing acknowledgments, ensuring the a message is only disposed of once it has been successful processed
-- Performance. Messages are moved through the pipeline fast
-- Command line tools for:
-  - Subscribing to messages from an AMQP queue
-  - Pushing messages back onto an AMQP exchange
-  - Monitoring performance (see msg_probe)
-- A base module (MessageCommand) to include into your own classes to quickly make workers.
-
-== Install
-
-  > sudo gem install pipeline_toolkit
-
-=== Dependancies
-
-It is assumed that you have:
-- An AMQP msg server to pop and push messages to (e.g. http://www.rabbitmq.com/)
-- A *nix system, such as Linux or Mac OS X.
-
-== Usage
-
-1. Create your worker
-
-  class MyWorker
-    include MessageCommand
-    
-    def process_message(msg)
-      # do stuff here
-      msg
-    end
-  end  
-  MyWorker.new.start
-
-2. Hook it up to a AMQP message source
-
-  > msg_subscribe.rb -q source | my_worker.rb | msg_push.rb -x dest
-
-You can learn more about the command line tools and what options are available by using their help command.
-  
-  > msg_subscriber --help
-  > msg_push --help
-  > msg_sink --help
-  > msg_probe --help
-
-== Examples
-
-Generate a bunch of sample messages and push them to an AMQP exchange 'test-exchange'. Hook up a pipe to consume and push results to /dev/null. Goto http://localhost:9090 to see message throughput.
-
-  > msg_generator.rb | msg_push.rb -x test-exchange
-  > msg_subscribe.rb --http-port 9090 -x test-exchange -q test_queue > /dev/null
-
-Same as above, but this time with acknowledgements switched on. This guarantees that a message isn't removed from the message server until handled. msg_sink.rb acknowledges all messages.
-
-  > msg_generator.rb | msg_push.rb -x test-exchange
-  > msg_subscribe.rb -a -x test-exchange -q test_queue | msg_sink.rb
-
-== Todo
-
-- Logging. For the moment we're using SysLogLogger (which requires Syslog). This needs to be switched to the default ruby logging, and we need to find out a way to plug in alternatives if required.
-
-- DNS-SD is broken. See http://github.com/tenderlove/dnssd/issues#issue/3
-
-- Decide if we want to support other exchange, queue and binding configurations. For example, exchanges are always fanout and durable at the moment.
-
-

data/Rakefile

@@ -1,40 +0,0 @@
-require 'rubygems'
-require 'rake'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "pipeline_toolkit"
-    gem.summary = %Q{Toolkit for building processing pipelines using Unix Pipes and AMQP messages}
-    gem.email = "labs@visfleet.com"
-    gem.homepage = "http://github.com/visfleet/pipeline_toolkit"
-    gem.authors = ["Aisha Fenton"]
-    gem.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_sink.rb", "msg_generator.rb"]
-    gem.add_runtime_dependency('amqp', ">=0.6.5")
-    gem.add_runtime_dependency('trollop', ">=1.14")
-    gem.add_runtime_dependency('eventmachine', ">=0.12.10")
-    gem.add_runtime_dependency('eventmachine_httpserver', ">=0.2.0")
-    gem.add_runtime_dependency('SyslogLogger', ">=1.4.0")
-  end
-  Jeweler::GemcutterTasks.new  
-
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
-end
-
-task :default => :build
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  if File.exist?('VERSION.yml')
-    config = YAML.load(File.read('VERSION.yml'))
-    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
-  else
-    version = ""
-  end
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "pipeline_toolkit #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end

data/VERSION

@@ -1 +0,0 @@
-1.0.4

data/bin/msg_probe.rb

@@ -1,15 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'trollop'
-require 'pipeline_toolkit'
-
-opts = Trollop::options do
-  opt :interval, "Time in seconds between updates", :short => "i", :default => 2
-  opt :http_port, "The port the HTTP server runs on. Default is a random port between 10000-11000", :type => :integer
-  opt :name, "The name of the probe. Used in monitoring", :type => :string, :short => 'n'
-  opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :short => 'd'
-end
-
-MessageProbe.new(opts).start
-

data/bin/msg_push.rb

@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'trollop'
-require 'pipeline_toolkit'
-
-opts = Trollop::options do
-  opt :exchanges, "The destination exchange(s)", :short => "x", :type => :strings
-  opt :key_eval, "A string of ruby code that is evaluated to produce a routing key for a given message. 
-                  By default each message has no routing key", :short => "e", :type => :string
-  opt :key_file, "A ruby file that gets included which contains a custom route_key method", :short => "f", :type => :string
-  
-  # Msg server
-  opt :host, "The AMQP message server host", :default => "localhost"
-  opt :port, "The AMQP message server port", :default => "5672"
-  opt :user, "The AMQP message server username", :default => "guest"
-  opt :pass, "The AMQP message server username", :default => "guest"
-  opt :vhost, "The AMQP message server vhost", :default => "/"
-end
-
-mp = MessagePusher.new(opts)
-# FIXME. Should be in MessageCommand class
-Signal.trap('INT') { mp.stop }
-Signal.trap('TERM'){ mp.stop }
-mp.start

data/bin/msg_sink.rb

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'trollop'
-require 'pipeline_toolkit'
-
-# opts = Trollop::options do
-#   opt :interval, "Interval", :short => "i", :default => 100
-# end
-
-MessageSink.new.start

data/bin/msg_subscribe.rb

@@ -1,27 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'trollop'
-require 'pipeline_toolkit'
-
-opts = Trollop::options do
-  # Monitoring
-  opt :name, "The name used to describe the entire process chain", :short => "n", :type => :string  
-  opt :http_port, "The port the HTTP monitoring server runs on. Default is a random port between 10000-11000", :type => :integer
-  opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :short => 'd'
-
-  # Messages
-  opt :queue, "The source queue", :short => "q", :type => :string, :required => true
-  opt :ack, "Switch that requires that messages are successfully processed before continuing with the next message", :short => "a"
-  opt :topic, "The queue topic to subscribe to", :short => "t", :type => :string
-  opt :max_unackd, "The maximum number of unaknowledged messages to buffer before waiting for them to be acknowledged. Defaults to 100", :default => 100
-  opt :exchange_bind, "Binds the the queue to the specified exchange. The created exchange is (for now) fanout, durable, and not passive", :short => "x", :type => :string
-  
-  # Msg server
-  opt :host, "The AMQP message server host", :default => "localhost"
-  opt :port, "The AMQP message server port", :default => "5672"
-  opt :user, "The AMQP message server username", :default => "guest"
-  opt :pass, "The AMQP message server username", :default => "guest", :short => "w"
-  opt :vhost, "The AMQP message server vhost", :default => "/"
-end
-
-ms = MessageSubscriber.new(opts).start

data/monitor/munin.rb

@@ -1,91 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'open-uri'
-require 'trollop'
-require 'nokogiri'
-
-class MuninPlugin
-
-  def initialize(is_config, opts)
-    @probe_urls = opts.probe_urls
-
-    self.get_probes
-
-    if is_config
-       self.config
-    else
-      self.data
-    end
-  end
-
-  def get_probes
-    @probes = []
-    @probe_urls.each_with_index do |url, idx|
-      @probes << get_probe(url)
-    end
-    @probes
-  end
-
-  def get_probe(url)
-    probe = Probe.new
-    doc = Nokogiri::HTML(open(url))
-    probe.name   = extract_mf_value(doc, "name")
-    probe.mps    = extract_mf_value(doc, "mps")
-    probe.uptime = extract_mf_value(doc, "uptime")
-    probe
-  end
-
-  # Extracts the microformat value from the HTML doc
-  def extract_mf_value(doc, key)
-    doc.css(".#{key}").first.content
-  end
-
-  def config
-    config=<<-EOL
-graph_title Message Probe Throughput
-graph_vlabel messages per second
-graph_category Message Probe
-    EOL
-    
-    @probes.each do |probe|
-      config << "probe_#{probe.name}.label #{probe.name}\n"
-    end
-    
-    puts config
-  end
-
-  def data
-    data = ""
-    @probes.each do |probe|
-      data << "#{probe.name}.value #{probe.mps}\n"
-    end
-    
-    puts data
-  end
-
-end
-
-class Probe
-  attr_accessor :name, :mps, :uptime
-end
-
-SUB_COMMANDS = %w{run config}
-
-global_opts = Trollop::options do
-  banner <<-EOS
-  Munin plugin
-  Usage:
-        munin.rb [run|config] [options]
-
-  For more help run
-        munin.rb [command] --help
-  EOS
-  stop_on SUB_COMMANDS
-end
-
-is_config = (ARGV.shift == "config")
-opts = Trollop::options do
-  opt :probe_urls, "The probe's URL", :short => "p", :type => :strings, :default => ["http://127.0.0.1:9070"]
-end
-
-MuninPlugin.new(is_config, opts)

data/pipeline_toolkit.gemspec

@@ -1,72 +0,0 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |s|
-  s.name = %q{pipeline_toolkit}
-  s.version = "1.0.4"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Aisha Fenton"]
-  s.date = %q{2010-01-15}
-  s.email = %q{labs@visfleet.com}
-  s.executables = ["msg_probe.rb", "msg_subscribe.rb", "msg_push.rb", "msg_sink.rb", "msg_generator.rb"]
-  s.extra_rdoc_files = [
-    "README.rdoc"
-  ]
-  s.files = [
-    ".gitignore",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "bin/msg_generator.rb",
-     "bin/msg_probe.rb",
-     "bin/msg_push.rb",
-     "bin/msg_sink.rb",
-     "bin/msg_subscribe.rb",
-     "lib/pipeline_toolkit.rb",
-     "lib/pipeline_toolkit/default_logger.rb",
-     "lib/pipeline_toolkit/message_coder.rb",
-     "lib/pipeline_toolkit/message_command.rb",
-     "lib/pipeline_toolkit/message_probe.rb",
-     "lib/pipeline_toolkit/message_pusher.rb",
-     "lib/pipeline_toolkit/message_sink.rb",
-     "lib/pipeline_toolkit/message_subscriber.rb",
-     "lib/pipeline_toolkit/open_hash.rb",
-     "lib/pipeline_toolkit/socket_util.rb",
-     "monitor/munin.rb",
-     "pipeline_toolkit.gemspec"
-  ]
-  s.homepage = %q{http://github.com/visfleet/pipeline_toolkit}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
-  s.summary = %q{Toolkit for building processing pipelines using Unix Pipes and AMQP messages}
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<amqp>, [">= 0.6.5"])
-      s.add_runtime_dependency(%q<trollop>, [">= 1.14"])
-      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.10"])
-      s.add_runtime_dependency(%q<eventmachine_httpserver>, [">= 0.2.0"])
-      s.add_runtime_dependency(%q<SyslogLogger>, [">= 1.4.0"])
-    else
-      s.add_dependency(%q<amqp>, [">= 0.6.5"])
-      s.add_dependency(%q<trollop>, [">= 1.14"])
-      s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
-      s.add_dependency(%q<eventmachine_httpserver>, [">= 0.2.0"])
-      s.add_dependency(%q<SyslogLogger>, [">= 1.4.0"])
-    end
-  else
-    s.add_dependency(%q<amqp>, [">= 0.6.5"])
-    s.add_dependency(%q<trollop>, [">= 1.14"])
-    s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
-    s.add_dependency(%q<eventmachine_httpserver>, [">= 0.2.0"])
-    s.add_dependency(%q<SyslogLogger>, [">= 1.4.0"])
-  end
-end
-