amqpop 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in amqpop.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Chris Cherry
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,95 @@
+amqpop
+======
+
+Tool for dispatching a child command for each message consumed from an AMQP
+queue.
+
+
+Benefits
+--------
+
+Designed for situations where you need to consume and act on messages from an
+AMQP queue without a long running process.
+
+
+Example
+-------
+
+    amqpop -h rabbit.domain.com -u user -n 2 -x logs -w 10 -- /usr/bin/ruby log_processor.rb
+
+This command would do the following:
+
+- Make sure only 1 instance of this amqpop command is running by using a
+lock/pid file keyed based on the arguments of the command
+- Connect to the AMQP server `rabbit.domain.com` as `user` with password looked
+up from the auth file ~/.amqpop_auth
+- Attach a unique, non durable queue to the exchange named logs
+- Pass the message payload in as STDIN to `/usr/bin/ruby log_processor.rb`
+- Run up to 2 parallel processes of `/usr/bin/ruby log_processor.rb`
+- Once the queue has been exhausted of messages, wait for another message for 10
+seconds, if one shows up inside of that time, process it immediately, and then
+wait for another 10 seconds and repeat, otherwise terminate.
+
+
+Usage
+-----
+
+    amqpop [options] -- <child-command>
+
+    [options] are:
+    -h <s>:  AMQP host (default: localhost)
+    -u <s>:  AMQP user (default: guest)
+    -p <s>:  AMQP password (default: guest)
+    -n <i>:  Number of child message processing commands that can be executed in
+             parallel (default: 1)
+    -q <s>:  Name of the queue on the to connect to, unique if not provided
+             (default: )
+        -e:  Is the queue persistent (default: true)
+        -a:  Does the queue remove itself
+    -x <s>:  Exchange to bind the queue to. Format [name]:<routing>, example:
+             logs or weather:usa.* (default: )
+    -w <i>:  Amount of time in seconds to wait for more messages to show up.
+             Waits forever if 0 (default: 30)
+
+    --verbose, -v:   Verbose logging
+    --version, -r:   Print version and exit
+       --help, -l:   Show this message
+
+
+Auth File
+---------
+
+You can put a file at ~/.amqpop_auth which is used to intelligently look up
+login credentials when calling amqpop. No more sensative information in the
+process list or in your history! Use one entry per line, in the format:
+
+    host    user    pass
+
+Example:
+
+    192.168.1.1    consumer    1Consumer2
+    192.168.1.1    other       XYother$Z
+    rabbithost.domain.com    publisher_user    98pubPass234
+
+The amount and kind of whitespace between host, user and pass don't matter, just
+don't use newlines.
+
+Note: The auth file must have world readable and writable disabled (so 660 or
+600 are ok)
+
+
+Tested On
+---------
+* OSX 10.8.2, 10.7.5
+* Ruby 1.9.3-p327, 1.8.7-p358
+* RabbitMQ 2.8.7
+  * on Erlang R14A, Debian 6.0, kernel 2.6.32-5-686
+  * on Erlang R14B04, RedHat EL5, kernel 2.6.18-238.9.1.el5xen
+
+
+Acknowledgements
+----------------
+
+Thank you to William Morgan and the contributors of Trollop
+(http://trollop.rubyforge.org/)
+

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/amqpop.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/amqpop/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Chris Cherry"]
+  gem.email         = ["ctcherry@gmail.com"]
+  gem.description   = %q{Command line AMQP consumer}
+  gem.summary       = %q{Command line tool for consuming messages off of an AMQP queue and dispatching them to a user specified command}
+  gem.homepage      = "https://github.com/ctcherry/amqpop"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^spec/})
+  gem.name          = "amqpop"
+  gem.require_paths = ["lib"]
+  gem.version       = Amqpop::VERSION
+
+  gem.add_dependency "amqp", "~> 0.9.8"
+  gem.add_development_dependency "rspec", "~> 2.12.0"
+end

data/bin/amqpop

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'amqpop/cli'
+
+Amqpop::CLI.start

data/lib/amqpop/amqpop.rb

@@ -0,0 +1,32 @@
+module Amqpop
+
+	class << self
+		attr_accessor :options
+	end
+
+	self.options = nil
+
+	def self.require_ack?
+    return nil if options.nil?
+	  !temp_queue?
+	end
+
+	def self.temp_queue?
+		return nil if options.nil?
+	  options[:queue_name] == ""
+	end
+
+  def self.eputs(msg)
+    STDERR.puts msg
+  end
+
+  def self.vputs(msg)
+    eputs "> #{msg}" if verbose?
+  end
+
+  def self.verbose?
+    return nil if options.nil?
+    options[:verbose]
+  end
+
+end

data/lib/amqpop/auth_file.rb

@@ -0,0 +1,66 @@
+require 'digest/md5'
+
+module Amqpop
+  class AuthFile
+
+  	def initialize
+      @hosts = {}
+      load_available_files
+  	end
+
+    def lookup(match_host, match_user = "")
+      host = @hosts[match_host]
+      return nil if host.nil?
+      return nil if host.keys.empty?
+
+      match_user = host.keys.first if match_user.to_s == ""
+      pass = host[match_user]
+      return nil if pass.to_s == ""
+
+      {:host => match_host, :username => match_user, :password => pass}
+    end
+
+    private
+
+      def load_available_files
+        ["~/.amqpop_auth"].each do |f|
+          f = File.expand_path(f)
+          load_file(f)
+        end
+      end
+
+      def load_file(file)
+        if File.exist?(file)
+          if world_readable?(file) || world_writable?(file)
+            STDERR.puts "WARNING: Auth file: #{file} has unsafe permissions, not loaded"
+            return false
+          end
+          parse_file(file)
+        end
+      end
+
+      def parse_file(file)
+        File.readlines(file).each do |line|
+          host, user, pass = line.split(/\s+/)
+          if host.to_s == "" || user.to_s == "" || pass.to_s == ""
+            STDERR.pust "WARNING: Auth file: #{file} has invalid line, skipped that line"
+            return false
+          end
+
+          @hosts[host] ||= {}
+          @hosts[host][user] = pass
+        end
+      end
+
+      def world_writable?(file)
+        world_write_perms = 0000002
+        (File.stat(file).mode & world_write_perms) != 0
+      end
+
+      def world_readable?(file)
+        world_read_perms = 0000004
+        (File.stat(file).mode & world_read_perms) != 0
+      end
+
+  end
+end

data/lib/amqpop/cli.rb

@@ -0,0 +1,197 @@
+require 'amqpop/amqpop'
+require 'amqpop/trollop'
+require 'amqpop/lock_file'
+require 'amqpop/auth_file'
+require 'amqpop/message'
+require 'eventmachine'
+require 'amqp'
+
+module Amqpop
+
+  class CLI
+
+    def self.start
+      self.new.start
+    end
+
+    def initialize
+      STDOUT.sync = true
+      options
+      @lock = LockFile.new(options)
+      @lock.acquire!
+      @auth = AuthFile.new
+    end
+
+    def start
+      begin
+        EventMachine.threadpool_size = options[:num_children]
+        EventMachine.run do
+          Signal.trap("INT")  { shutdown }
+          Signal.trap("TERM") { shutdown }
+
+          vputs "Running #{AMQP::VERSION} version of the AMQP gem."
+          vputs "Connecting to AMQP broker on #{connection_params[:host]} as #{connection_params[:username]}."
+          AMQP.connect(connection_params) do |connection|
+            AMQP::Channel.new(connection) do |channel|
+
+              channel.on_error do |ch, close|
+                eputs "ERROR: Channel-level exception: #{close.reply_text}, #{close.inspect}"
+                connection.close {
+                  shutdown
+                }
+              end
+
+              if options[:wait] == 0
+                vputs "No timeout set, process will stay running"
+              else
+                vputs "Timeout of #{options[:wait]} seconds set"
+              end
+
+              queue = get_queue(channel)
+
+              queue.once_declared do
+                vputs "Connected to queue: #{queue.name}"
+                vputs "Ack mode: #{Amqpop.require_ack? ? 'explicit' : 'auto'}"
+              end
+              
+              bind_queue(queue)
+
+              queue.subscribe(:confirm => proc{ wait_exit_timer }, :ack => Amqpop.require_ack?) do |meta, payload|
+                cancel_wait_exit_timer
+                m = Message.new(payload, meta)
+                vputs "Received a message: #{payload}. Executing..."
+                EventMachine.defer(m.command_proc, m.callback_proc)
+                wait_exit_timer
+              end
+
+            end
+          end
+        end
+      rescue => e
+        eputs "ERROR: #{e.class} - #{e.message}"
+        exit 2
+      end
+    end
+
+    private
+
+      def shutdown
+        EventMachine.stop
+        @lock.release!
+        exit 0
+      end
+
+      def connection_params
+        return @connection_params if defined?(@connection_params)
+
+        params = {:host => options[:host], :username => options[:user], :password => options[:pass]}
+
+        if options[:host] != "localhost" && options[:user] != "guest" && options[:pass] != "guest"
+          # Every option was changed from the default, so we don't need to lookup anything
+          @connection_params = params
+          return @connection_params
+        end
+
+        if options[:host] != "localhost" && options[:user] == "guest" && options[:pass] == "guest"
+          # They set host, but not user or pass
+          creds = @auth.lookup(options[:host], options[:user])
+          creds = @auth.lookup(options[:host]) if creds.nil?
+
+          @connection_params = (creds.nil?) ? params : creds
+
+          return @connection_params
+        end
+
+        if options[:host] != "localhost" && options[:user] != "guest" && options[:pass] == "guest"
+          # They set host and user, but not pass
+          creds = @auth.lookup(options[:host], options[:user])
+          @connection_params = (creds.nil?) ? params : creds
+
+          return @connection_params
+        end
+
+        @connection_params = params
+
+      end
+
+      def cancel_wait_exit_timer
+        @wait_timer.cancel if defined?(@wait_timer)
+      end
+
+      def wait_exit_timer
+        return if options[:wait] == 0
+        cancel_wait_exit_timer
+        @wait_timer = EventMachine::Timer.new(options[:wait]) do
+          vputs "Timeout of #{options[:wait]} seconds expired, exiting"
+          shutdown
+        end
+      end
+
+      def bind_queue(queue)
+        if options[:exchange][:name] == ""
+          vputs "Binding queue to default exchange implicitly, with routing key '#{queue.name}'"
+        else
+          vputs "Binding queue to exchange: #{options[:exchange][:name]}, with routing key '#{options[:exchange][:routing_key]}'"
+          queue.bind(options[:exchange][:name], :routing_key => options[:exchange][:routing_key])
+        end
+      end
+
+      def get_queue(channel)
+        if Amqpop.temp_queue?
+          channel.queue('', :auto_delete => true, :durable => false, :exclusive => true)
+        else
+          channel.queue(options[:queue_name], :auto_delete => false, :durable => true)
+        end
+      end
+
+      def eputs(msg)
+        Amqpop.eputs msg
+      end
+
+      def vputs(msg)
+        Amqpop.vputs msg
+      end
+
+      def options
+        return Amqpop.options unless Amqpop.options.nil?
+        @options = Trollop::options do
+          version "amqpop 0.0.1 (c) 2012 Chris Cherry"
+          banner <<-EOS
+Command line tool for consuming messages off of an AMQP queue and dispatching them to a user specified command.
+
+Usage:
+       amqpop [options] -- <child-command>
+
+[options] are:
+
+EOS
+          opt :host, "AMQP host", :type => :string, :short => "-h", :default => 'localhost'
+          opt :user, "AMQP user", :type => :string, :short => "-u", :default => 'guest'
+          opt :pass, "AMQP password", :type => :string, :short => "-p", :default => 'guest'
+          opt :num_children, "Number of child message processing commands that can be executed in parallel", :short => "-n", :default => 1
+          opt :queue_name, "Name of the queue on the to connect to, unique if not provided", :type => :string, :default => ''
+          opt :queue_durable, "Is the queue persistant", :default => true
+          opt :queue_auto_delete, "Does the queue remove itself", :default => false
+          opt :exchange, "Exchange to bind the queue to. Format [name]:<routing_key>, example: logs or weather:usa.*", :type => :string, :short => "-x", :default => ''
+
+          opt :wait, "Amount of time in seconds to wait for more messages to show up. Waits forever if 0", :default => 30
+          opt :verbose, "Verbose logging", :short => :none
+          opt :help, "Show this help message", :short => :none
+          stop_on "--"
+        end
+
+        # Break exchange into hash of its parts
+        ename, eroute = @options[:exchange].to_s.split(":")
+        @options[:exchange] = {:name => ename.to_s, :routing_key => eroute.to_s}
+
+        # 1 second is too fast, make it a minimum of 2
+        @options[:wait] = 2 if @options[:wait] == 1
+
+        # Store child command, or empty array
+        dbldash = ARGV.shift
+        @options[:child_command] = ARGV.dup
+        Amqpop.options = @options
+      end
+
+  end
+end