pipeline_toolkit 1.1.0 → 1.2.0

data/lib/pipeline_toolkit/amqp/writer.rb

@@ -1,53 +1,56 @@
 require 'pipeline_toolkit/amqp/abstract'
 require 'pipeline_toolkit/message_coder'
 
-module Amqp # :nodoc:
+module PipelineToolkit
+  module Amqp # :nodoc:
   
-  ##
-  # The Writer provides functionality specific to asynchronous message 
-  # publishing to the AMQP server.
-  #
-  module Writer
-    include Amqp::Abstract
-   
     ##
-    # Initialize the AMQP server specific to handling 
-    # of messages publishing to the server.
+    # The Writer provides functionality specific to asynchronous message 
+    # publishing to the AMQP server.
     #
-    def initialize_writer
-      DefaultLogger.debug("Amqp::Writer#initialize_writer") if options[:env] == "development"
-      initialize_connection
-      initialize_channel
-      initialize_exchange
-    end
+    module Writer
+      include Amqp::Abstract
+   
+      ##
+      # Initialize the AMQP server specific to handling 
+      # of messages publishing to the server.
+      #
+      def initialize_writer
+        DefaultLogger.debug("Amqp::Writer#initialize_writer") if options[:env] == "development"
+        initialize_connection
+        initialize_channel
+        initialize_exchange
+      end
     
-    ##
-    # Initializes the queues to publish the messages too.
-    #
-    def initialize_queues
-      DefaultLogger.debug("Amqp::Writer#initialize_queues") if options[:env] == "development"
-      options[:queues].each do |name, routing_key|
-        initialize_queue(name)
-        bind_queue(routing_key)
+      ##
+      # Initializes the queues to publish the messages too.
+      #
+      def initialize_queues
+        DefaultLogger.debug("Amqp::Writer#initialize_queues") if options[:env] == "development"
+        options[:queues].each do |name, routing_key|
+          initialize_queue(name)
+          bind_queue(routing_key)
+        end
       end
-    end
     
-    ##
-    # Handles the publishing of messages into the AMQP server.
-    #
-    # @param message<Hash> The message to publish
-    # 
-    def publish(message)
-      DefaultLogger.debug("Amqp::Writer#publish(message)") if options[:env] == "development"
-      if message.has_key?(:routing_key)
-        @exchange.publish(MessageCoder.encode(message), :key => message[:routing_key])
-      else
-        @exchange.publish(MessageCoder.encode(message))
+      ##
+      # Handles the publishing of messages into the AMQP server.
+      #
+      # @param message<Hash> The message to publish
+      # 
+      def publish(message)
+        DefaultLogger.debug("Amqp::Writer#publish(message)") if options[:env] == "development"
+        if message.has_key?(:routing_key)
+          @exchange.publish(MessageCoder.encode(message), :key => message[:routing_key])
+        else
+          @exchange.publish(MessageCoder.encode(message))
+        end
       end
-    end
     
-    def queue_names
-      options[:queues].map { |name, type| name }
+      def queue_names
+        options[:queues].map { |name, type| name }
+      end
+
     end
 
   end

data/lib/pipeline_toolkit/{commands/msg_generator/cli.rb → cli/msg_generator_cli.rb}

@@ -1,4 +1,3 @@
-require 'pp'
 require 'trollop'
 require 'eventmachine'
 require 'pipeline_toolkit'
@@ -14,10 +13,11 @@ Signal.trap('TERM') do
   exit(0)
   # TODO: complete signal trap terminate
 end
-
-module Commands # :nodoc:
-  module MsgGenerate
-    class CLI
+       
+module PipelineToolkit # :nodoc:
+  module CLI
+    
+    class MsgGeneratorCLI
       
       def self.execute(stdout, arguments=[])
 
@@ -44,5 +44,6 @@ Options:
         MessageGenerator.new(opts).start
       end
     end
+    
   end
 end

data/lib/pipeline_toolkit/{commands/msg_push/cli.rb → cli/msg_push_cli.rb}

@@ -1,10 +1,10 @@
-require 'pp'
 require 'trollop'
 require 'eventmachine'
+require 'pipeline_toolkit'
 
-module Commands # :nodoc:
-  module MsgPush
-    class CLI
+module PipelineToolkit # :nodoc:
+  module CLI
+    class MsgPushCLI
       def self.execute(stdout, arguments=[])
 
         opts = Trollop::options do
@@ -25,22 +25,30 @@ Examples:
 Options:
             EOL
 
-          # Msg exchange  
+          banner <<-EOL
+
+  Queue/exchange:
+EOL
           opt :exchange, "Exchange name", :type => :string, :required => true, :short => :x
           opt :type, "Exchange type (direct, fanout, topic)", :default => "fanout", :short => :t
           opt :passive, "If true, the exchange will not be created if it does not already exist.", :default => false, :short => :s
           opt :durable, "If true, the exchange will be marked as durable.", :default => false, :short => :d
-          
-          # Msg queues
           opt :queues, "Destination queue(s) (e.g. queue1:routing_key, queue2:routing_key). Routing keys will be ignored if exchange type is not topic.", :type => :strings, :required => false, :short => :q
           
-          # Msg server
+          banner <<-EOL
+
+  AMQP server:
+EOL
           opt :host, "AMQP message server host", :default => "localhost", :short => :h
           opt :port, "AMQP message server port", :default => 5672, :short => :p
           opt :user, "AMQP message server username", :default => "guest", :short => :u
           opt :pass, "AMQP message server password", :default => "guest", :short => :w
           opt :vhost, "AMQP message server vhost", :default => "/", :short => :v
 
+          banner <<-EOL
+
+  Misc:
+EOL
           opt :env, "Environment (development, production)", :default => "development", :short => :e
         end
                 

data/lib/pipeline_toolkit/cli/msg_sink_cli.rb

@@ -0,0 +1,28 @@
+require 'trollop'
+require 'pipeline_toolkit'
+
+module PipelineToolkit # :nodoc:
+  module CLI
+    
+    class MsgSinkCLI
+      def self.execute(stdout, arguments=[])
+
+        opts = Trollop::options do
+          banner <<-EOL
+Message Sink
+------------
+  Swallows messages but still acknowledges them (like a /dev/null, but with acks)
+  
+  Usage:
+  msg_sink
+            EOL
+
+          opt :env, "The environment to run (development, production)", :default => "development", :short => :e
+        end
+
+        MessageSink.new(opts).start
+      end
+    end
+    
+  end
+end

data/lib/pipeline_toolkit/{commands/msg_subscribe/cli.rb → cli/msg_subscribe_cli.rb}

@@ -1,10 +1,11 @@
 require 'trollop'
 require 'eventmachine'
+require 'pipeline_toolkit'
 
-module Commands # :nodoc:
-  module MsgSubscribe # :nodoc:
+module PipelineToolkit # :nodoc:
+  module CLI # :nodoc:
     
-    class CLI
+    class MsgSubscribeCLI
       def self.execute(stdout, arguments=[])
 
         opts = Trollop::options do
@@ -23,31 +24,39 @@ Examples:
                 --host www.domain.com --port 7000
 
 Options:
-            EOL
+EOL
 
-          # Msg exchange  
+          banner <<-EOL
+
+  Queue/exchange:
+EOL
           opt :exchange, "The exchange name", :type => :string, :required => true, :short => :x
           opt :type, "The exchange type (direct, fanout or topic)", :default => "fanout", :short => :t
           opt :passive, "If set to true, the server will not create the exchange if it does not already exist.", :default => false, :short => :s
           opt :durable, "If set to true, the exchange will be marked as durable.", :default => false, :short => :d
-
-          # Messages
           opt :queue, "The destination queue (queue:routing_key). Routing keys will be ignored if exchange type is not topic.", :short => :q, :type => :string, :required => true
-          opt :ack, "If this field is set to false the server does not expect acknowledgments for messages.", :short => :a, :type => :boolean, :default => true
+          opt :no_acks, "If flag is set then the server does not use acknowledgments for messages.", :type => :flag
+
+          banner <<-EOL
 
-          # Msg server
+  AMQP server:
+EOL
           opt :host, "The AMQP message server host", :default => "localhost", :short => :h
           opt :port, "The AMQP message server port", :default => 5672, :short => :p
           opt :user, "The AMQP message server username", :default => "guest", :short => :u
           opt :pass, "The AMQP message server username", :default => "guest", :short => :w
           opt :vhost, "The AMQP message server vhost", :default => "/", :short => :v
             
-          # Monitoring
-          opt :name, "The name used to describe the entire process chain", :type => :string, :short => :n  
-          opt :http_port, "The port the HTTP monitoring server runs on. Default is a random port between 10000-11000", :type => :integer, :short => :o
-          opt :content_type, "The type of response we are expecting from the HTTP monitoring server. (http, xml and json supported)", :type => :string, :default => "html", :short => :c
-          opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :type => :boolean, :default => false, :short => :b
+          banner <<-EOL
+
+  Monitoring:
+EOL
+          opt :http_port, "The port the HTTP monitoring interface runs on. A value must be given to enable the monitoring interface - by default the monitoring interface isn't enabled. An HTML or JSON response can be obtained by suffixing either .html or .json to the request", :type => :integer, :short => :o
                       
+          banner <<-EOL
+
+  Misc:
+EOL
           opt :env, "The environment to run (development, production)", :default => "development", :short => :e
         end
 

data/lib/pipeline_toolkit/cucumber.rb

@@ -1,6 +1,8 @@
-
+require 'pipeline_toolkit/open_hash'
 require 'pipeline_toolkit/cucumber/amqp'
-require 'pipeline_toolkit/cucumber/amqp_steps'
+require 'pipeline_toolkit/cucumber/in_out_process'
 require 'pipeline_toolkit/cucumber/machine'
+require 'pipeline_toolkit/cucumber/msg_sub_process'
+
 require 'pipeline_toolkit/cucumber/machine_steps'
-require 'pipeline_toolkit/cucumber/msg_sub_machine'
+require 'pipeline_toolkit/cucumber/amqp_steps'

data/lib/pipeline_toolkit/cucumber/amqp.rb

@@ -19,7 +19,7 @@ module PipelineToolkit
         exchange = @mq.exchange(exchange, :type => :fanout)
         messages.each do |message|
           message.default = nil
-          exchange.publish(MessageCoder.encode(message)) 
+          exchange.publish(MessageCoder.encode(message))
         end
       end
 

data/lib/pipeline_toolkit/cucumber/in_out_process.rb

@@ -0,0 +1,50 @@
+require 'pipeline_toolkit'
+require 'background_process'
+
+module PipelineToolkit
+  module Cucumber
+    
+    # Provides an interface for running and interacting with a stdin/stdout process which 
+    # talks messages. 
+    class InOutProcess
+      
+      def run(command)
+        @process = BackgroundProcess.run("ruby -rubygems bin/#{command}")
+        # wait for process to start
+        sleep(0.5)
+        raise @process.stderr.gets unless @process.running?
+      end
+
+      def kill
+        @process.kill
+        @process.wait
+      end
+
+      def get_messages(number, timeout = 5)
+        output_msgs = []
+        get_lines(@process.stdout, number, timeout) do |line|
+          msg = MessageCoder.decode(line)
+          output_msgs << msg
+        end
+        output_msgs
+      end
+
+    private
+    
+      def get_lines(io, number, timeout = 5)
+        count = 0
+        Timeout::timeout(timeout) do
+          while (count < number)
+            line = io.gets
+            raise "The machine quit before I received all the messages!!" if line.nil? 
+            yield line if block_given?
+            count += 1
+          end
+        end
+      rescue Timeout::Error
+        raise "Timed out waiting for messages. Received #{count} of #{number}"
+      end
+       
+    end
+  end
+end

data/lib/pipeline_toolkit/cucumber/machine.rb

@@ -4,53 +4,71 @@ require 'background_process'
 module PipelineToolkit
   module Cucumber
     
-    # Provides an interface for running and interacting with a machine process. 
-    class Machine
-
-      def run(command)
-        @process = BackgroundProcess.run("ruby -rubygems bin/#{command}")
-        # wait for process to start, and check if running
-        sleep(0.5)
-        raise @process.stderr.gets unless @process.running?
+    # Provides an interface for running and interacting with a pipeline_toolkit machine. 
+    class Machine < InOutProcess
+      
+      attr_reader :use_acknowledgements
+
+      def run(command, use_acknowledgements = true)
+        super(command)
+        
+        @use_acknowledgements = use_acknowledgements
+        setup_system
       end
 
       def kill
-        @process.kill
-        @process.wait
+        destroy_sys_pipe
+        super
       end
 
       def input_messages(messages)
         messages.each do |message|
-          # NB: required otherwise marshal complains about the Hash's default proc
-          message.default = nil
+          store_acknowledgement(message) if use_acknowledgements
           @process.stdin.puts(MessageCoder.encode(message))
           @process.stdin.flush
         end
       end
-        
-      def get_messages(number, timeout = 5)
+            
+      def get_acknowledged_messages(number, timeout = 5)
         output_msgs = []
-        output_lines(number, timeout) do |line|
+        get_lines(@sys_pipe, number, timeout) do |line|
           msg = MessageCoder.decode(line)
           output_msgs << msg
         end
         output_msgs
+        @messages_to_ack.values_at( *output_msgs.map { |msg| msg[:ack_id] } )
       end
 
     private
     
-      def output_lines(number, timeout = 5)
-        count = 0
-        Timeout::timeout(timeout) do
-          while (count < number)
-            line = @process.stdout.gets
-            yield line
-            count += 1
-          end
-        end
-      rescue Timeout::Error
-        raise "Timed out waiting for messages. Received #{count} of #{number}"
+      def setup_system
+        create_sys_pipe
+        message = { :msg_type => "system", 
+                    :sys_pipe => @sys_pipe.path, 
+                    :ack => use_acknowledgements }
+
+        # send sys_message
+        input_messages([message])
+
+        # get pipe_desc off sys_pipe
+        get_lines(@sys_pipe, 1, 5)
       end
+
+      def destroy_sys_pipe
+        `rm #{@sys_pipe.path}`
+      end
+
+      def create_sys_pipe
+        name = File.join("/tmp", "test_sys_pipe_#{Time.now.to_i}_#{rand(9999)}")
+        `mkfifo #{name}`
+        @sys_pipe = File.new(name, "r+")
+      end
+
+      def store_acknowledgement(message)
+        @messages_to_ack ||= {}
+        message[:ack_id] = "#{Time.now.to_i}_#{rand(9999)}"
+        @messages_to_ack[message[:ack_id]] = message
+      end    
        
     end
   end

data/lib/pipeline_toolkit/cucumber/machine_steps.rb

@@ -1,23 +1,29 @@
-# ---------
-# Machine lifecycle
 
 Given /^I run this machine:$/ do |command|
-  @machine = PipelineToolkit::Cucumber::Machine.new
-  @machine.run(command)
+  @process = PipelineToolkit::Cucumber::Machine.new
+  @process.run(command)
 end
 
 # Clean up backround processes after each scenario
 After do
-  @machine.kill unless @machine.nil?
+  @process.kill unless @process.nil?
 end
 
 When /^I input these messages:$/ do |messages|
-  @machine.input_messages(messages.hashes)
+  @process.input_messages(messages.hashes)
 end
 
-Then /^these messages are output:$/ do |expected_msgs|
-  received_messages = @machine.get_messages(expected_msgs.rows.size)
+Then /^these messages are passed on:$/ do |expected_msgs|
+  received_messages = @process.get_messages(expected_msgs.rows.size)
   # turn everything into str
   received_messages.map! { |h| h.each { |k,v| h[k] = v.to_s }}
   expected_msgs.diff!(received_messages)
-end
+end
+
+Then /^these messages are acknowledged:$/ do |expected_msgs|
+  received_messages = @process.get_acknowledged_messages(expected_msgs.rows.size)
+  # turn everything into str
+  received_messages.map! { |h| h.each { |k,v| h[k] = v.to_s }}
+  expected_msgs.diff!(received_messages)
+end
+