visfleet-pipeline_toolkit 0.0.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Aisha Fenton
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,7 @@
+= pipeline_toolkit
+
+Description goes here.
+
+== Copyright
+
+Copyright (c) 2009 Aisha Fenton. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "pipeline_toolkit"
+    gem.summary = %Q{TODO}
+    gem.email = "aisha.fenton@visfleet.com"
+    gem.homepage = "http://github.com/visfleet/pipeline_toolkit"
+    gem.authors = ["Aisha Fenton"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "pipeline_toolkit #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/bin/monitor.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'pipeline_toolkit'
+
+opts = Trollop::options do
+  opt :interval, "Interval", :short => "i", :default => 100
+end
+
+MessageMonitor.new(opts).start

data/bin/pop.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'pipeline_toolkit'
+require "pp"
+
+opts = Trollop::options do
+  opt :sources, "Sources", :short => "s", :type => :strings
+  opt :cycle, "Seconds between polls", :default => 1
+  opt :ack, "Switch that requires that messages are successfully processed before continuing with the next message", :short => "a"
+  
+  # Msg server
+  opt :host, "The AMQP message server host", :default => "localhost"
+  opt :port, "The AMQP message server port", :default => "5672"
+  opt :user, "The AMQP message server username", :default => "guest"
+  opt :pass, "The AMQP message server username", :default => "guest", :short => "w"
+  opt :vhost, "The AMQP message server vhost", :default => "/"
+end
+
+MessagePopper.new(opts).start

data/bin/push.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'pipeline_toolkit'
+
+opts = Trollop::options do
+  opt :destinations, "Destinations", :short => "d", :type => :strings
+  opt :route_eval, "A string of ruby code that ", :short => "e", :type => :string
+  opt :route_file, "A string of ruby code that ", :short => "f", :type => :string
+  
+  # Msg server
+  opt :host, "The AMQP message server host", :default => "localhost"
+  opt :port, "The AMQP message server port", :default => "5672"
+  opt :user, "The AMQP message server username", :default => "guest"
+  opt :pass, "The AMQP message server username", :default => "guest"
+  opt :vhost, "The AMQP message server vhost", :default => "/"
+end
+
+MessagePusher.new(opts).start
+
+# pop -s raw | thingie | push -d  my_queue
+# pop -s raw | thingie | push -f "(msg.tmu_id.hash % NUMBER_OF_FORKS).to_s"
+# pop -s raw | thingie | push -f "Socket.gethostname"
+# pop -s raw | thingie | push -f "require route_to_site"

data/bin/subscribe.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'pipeline_toolkit'
+
+opts = Trollop::options do
+  opt :sources, "Sources", :short => "s", :type => :strings
+  opt :ack, "Switch that requires that messages are successfully processed before continuing with the next message", :short => "a"
+  opt :queue_name, "The name that the created queue is given", :short => "q", :type => :string
+  
+  # Msg server
+  opt :host, "The AMQP message server host", :default => "localhost"
+  opt :port, "The AMQP message server port", :default => "5672"
+  opt :user, "The AMQP message server username", :default => "guest"
+  opt :pass, "The AMQP message server username", :default => "guest", :short => "w"
+  opt :vhost, "The AMQP message server vhost", :default => "/"
+end
+
+ms = MessageSubscriber.new(opts)
+Signal.trap('INT') { ms.stop }
+Signal.trap('TERM'){ ms.stop }
+ms.start

data/perf_test/test_pop.rb

@@ -0,0 +1,32 @@
+require "rubygems"
+require "bunny"
+
+@msg_server = Bunny.new(:spec => '08')
+@msg_server.start
+
+def start
+  count ||= 0
+  prev_time ||= Time.now
+  queue = @msg_server.queue("process_telemetry")
+
+  loop do
+    result = queue.pop
+    next if result == :queue_empty
+  
+    count += 1
+    if count > 100
+      t = Time.now
+      puts("msgs pops per sec: #{count / (t - prev_time)}")
+      prev_time = t
+      count = 0
+    end
+  end
+
+end
+
+begin
+  start()
+ensure
+  puts "stopping"
+  @msg_server.stop
+end

data/perf_test/test_pop_async.rb

@@ -0,0 +1,24 @@
+require "rubygems"
+require 'mq'
+
+Signal.trap('INT') { AMQP.stop{ EM.stop } }
+Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+ 
+count ||= 0
+prev_time ||= Time.now
+
+AMQP.start(:host => 'localhost') do
+ 
+  MQ.queue('process_telemetry').subscribe(:ack => false) do |headers, body|
+
+    count += 1
+    if count > 100
+      t = Time.now
+      puts("msgs pops per sec: #{count / (t - prev_time)}")
+      prev_time = t
+      count = 0
+    end    
+
+  end
+
+end

data/perf_test/test_push.rb

@@ -0,0 +1,27 @@
+require "rubygems"
+require "bunny"
+
+EXCHANGE = "raw_telemetry"
+QUEUE = "process_telemetry"
+
+count ||= 0
+prev_time ||= Time.now
+msg_server = Bunny.new(:spec => '08')
+msg_server.start
+
+ex = msg_server.exchange(EXCHANGE, :type  => :fanout)
+msg_server.queue(QUEUE).bind(EXCHANGE)
+
+loop do
+
+  ex.publish("This is my msg")
+  
+  count += 1
+  if count > 100
+    t = Time.now
+    puts("msgs pushes per sec: #{count / (t - prev_time)}")
+    prev_time = t
+    count = 0
+  end
+    
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: visfleet-pipeline_toolkit
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Aisha Fenton
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-09-09 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: labs@visfleet.com
+executables: 
+- monitor.rb
+- subscribe.rb
+- push.rb
+- pop.rb
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- perf_test/test_pop.rb
+- perf_test/test_pop_async.rb
+- perf_test/test_push.rb
+has_rdoc: false
+homepage: http://github.com/visfleet/pipeline_toolkit
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Toolkit for building processing pipelines using Unix Pipes and AMQP messages
+test_files: []
+