pipeline_toolkit 1.2.16 → 1.2.17

data/README.markdown

@@ -1,6 +1,19 @@
 # Pipeline Toolkit #
 by VisFleet
 
+    
+     ===============                       
+     |^^:^^^^:Mw^^^|    Pipeline Toolkit   
+     |  :   .:Mw;  |    By VisFleet        
+     \-------------/    (c) 2009           
+      |^:^^^^:Mw^^|                        
+      | :    :Mw; |                        
+      | :    :Mw; |                        
+      | :    :Mw; |                        
+      | :   .:Mw; |                        
+      -------------                        
+
+
 Command line tools for processing messages by constructing a pipeline of machines. [AMQP](http://amqp.rubyforge.org/ "AMQP") and Unix pipes are used to construct the pipeline. Messages are simple Hashes (serialized as JSON) so they can hold any values and change throughout the processing.
 
 Provides:
@@ -60,17 +73,17 @@ All gem dependancies are installed automatically, but if you're curious checkout
 
 You can learn more about the command line tools and what options are available by using their help command.
   
-  > msg_subscriber --help  
-  > msg_push --help  
-  > msg_sink --help  
-  > msg_probe --help  
+    > msg_subscriber --help  
+    > msg_push --help  
+    > msg_sink --help  
+    > msg_probe --help  
 
 ## Examples ##
 
 In the example, open two terminal windows and then execute the following commands in each:
   
-  > msg_generator | msg_push -x test-exchange  
-  > msg_subscribe -a false --http-port 9090 -x test-exchange -q test_queue > /dev/null
+    > msg_generator | msg_push -x test-exchange  
+    > msg_subscribe -a false --http-port 9090 -x test-exchange -q test_queue > /dev/null
   
 The example does the following:
 
@@ -83,8 +96,8 @@ The example does the following:
 
 Now lets switch on acknowledgements:
 
-  > msg_generator | msg_push -x test-exchange  
-  > msg_subscribe -x test-exchange -q test_queue | msg_sink
+    > msg_generator | msg_push -x test-exchange  
+    > msg_subscribe -x test-exchange -q test_queue | msg_sink
 
 This example does the same as above, but this time with acknowledgements switched on. 
 This guarantees that a message isn't removed from the message server until handled. 

data/lib/pipeline_toolkit/amqp/abstract.rb

@@ -43,13 +43,17 @@ module PipelineToolkit
       def initialize_connection
         DefaultLogger.debug("Amqp::Abstract#initialize_connection") if options[:env] == "development"
         @connection = AMQP.connect(options.select_keys(:host, :port, :user, :pass, :vhost))
+      end
 
-        # Override Signals, so that connection is shutdown gracefully
-        Signal.trap('INT')  { @connection.close; EM.stop }
-        Signal.trap('TERM') { @connection.close; EM.stop }
-        Signal.trap('PIPE') { @connection.close; EM.stop }
+      ##
+      # Gracefully shuts down the AMQP connection. Calls the given block if provided 
+      #
+      def stop_connection
+        # NB: Next tick seems to give it enough time to receieve and send outstanding acks. But I don't
+        # really understand timing, so keep an eye on it.
+        EM.next_tick { @connection.close { yield if block_given? } }
       end
-    
+          
       ##
       # Returns a new channel. A channel is a bidirectional virtual connection between the client 
       # and the AMQP server.

data/lib/pipeline_toolkit/amqp/reader.rb

@@ -19,7 +19,9 @@ module PipelineToolkit
         @ack_headers ||= {}
         initialize_connection
         initialize_channel
-        @channel.prefetch(1) # We only want to handle one message at a time
+        # NB: Used to be 1 to prevent deadlocks. Think deadlocks won't happen anymore (because doing asyn
+        # reads now), but keep an eye on this parameter.
+        @channel.prefetch(25) 
         initialize_exchange
         initialize_queue
         bind_queue

data/lib/pipeline_toolkit/cucumber/amqp.rb

@@ -13,6 +13,14 @@ module PipelineToolkit
         @mq.start
       end
       
+      def connected?
+        @mq.connected?
+      end
+      
+      def close
+        @mq.stop
+      end
+      
       def purge_queue(queue)
         @mq.queue(queue).purge
       end

data/lib/pipeline_toolkit/cucumber/amqp_steps.rb

@@ -3,6 +3,11 @@ Given /^an amqp server running on "(\w+)" at port (\d+)$/ do |host, port|
   @amqp.start(host, port)
 end
 
+# Clean up connection after each scenario
+After do
+  @amqp.close if @amqp && @amqp.connected?
+end
+
 Given /^amqp queue "(\w+)" is empty$/ do |queue|
   @amqp.purge_queue(queue)
 end

data/lib/pipeline_toolkit/cucumber/io_process.rb

@@ -18,6 +18,10 @@ module PipelineToolkit
         @process.kill
         @process.wait
       end
+      
+      def running?
+        @process.running?
+      end
 
       def pid
         @process.pid

data/lib/pipeline_toolkit/cucumber/machine_steps.rb

@@ -6,7 +6,11 @@ end
 
 # Clean up backround processes after each scenario
 After do
-  @process.kill unless @process.nil?
+  @process.kill if @process && @process.running?
+end
+
+When /^I kill the machine$/ do
+  @process.kill
 end
 
 # Lets you use JSON to express a complex message 

data/lib/pipeline_toolkit/message_command.rb

@@ -61,9 +61,9 @@ module PipelineToolkit
     def start
       DefaultLogger.debug("MessageCommand#start") if @options[:env] == "development"
 
-      Signal.trap('INT')  { EM.stop }
-      Signal.trap('TERM') { EM.stop }
-      Signal.trap('PIPE') { EM.stop }
+      Signal.trap('INT')  { self.stop }
+      Signal.trap('TERM') { self.stop }
+      Signal.trap('PIPE') { self.stop }
 
       begin
         EM.run do
@@ -77,16 +77,23 @@ module PipelineToolkit
         DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
         raise e
       ensure
-        shutdown
+        cleanup
       end
     end
   
     ##
-    # Stops the machine
+    # Stops the command. This stops the event loop, giving it enough time to clear its buffers
     #
-    def shutdown
+    def stop
       DefaultLogger.info("Shutting down #{self.class.name}")
       DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+      EM.next_tick { EM.stop }
+    end
+
+    ##
+    # Cleans up any used resource, such as pipes and files.
+    #
+    def cleanup
       @ack_pipe.close if @ack_pipe
     end
   
@@ -110,6 +117,7 @@ module PipelineToolkit
     # @param message<Hash> The message to acknowledge
     #
     def acknowledge(message)
+      return if message[:ack_id].nil?
       DefaultLogger.debug("MessageCommand#acknowledge(message)") if options[:env] == "development"
       ack_message = {:msg_type => "ack", :ack_id => message[:ack_id]}
       write_to_pipe(ack_message, @ack_pipe)

data/lib/pipeline_toolkit/message_pusher.rb

@@ -34,7 +34,16 @@ module PipelineToolkit
     def initialize_machine
       DefaultLogger.debug("MessagePusher#initialize_machine") if options[:env] == "development"
       initialize_writer
-      initialize_queues    
+      initialize_queues
+    end
+
+    ##
+    # Override stop, so that we can close AMQP connection gracefully.
+    #
+    def stop
+      DefaultLogger.info("Shutting down #{self.class.name}")
+      DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+      stop_connection { EM.stop }
     end
 
     def description

data/lib/pipeline_toolkit/message_sink.rb

@@ -8,6 +8,7 @@ module PipelineToolkit
         $stdout.puts(message.inspect) 
         $stdout.flush
       end
+      sleep(0.1)
       acknowledge(message) if @options[:acknowledge]
     end
     

data/lib/pipeline_toolkit/message_subscriber.rb

@@ -54,6 +54,10 @@ module PipelineToolkit
       begin
         create_sys_pipe
 
+        Signal.trap('INT')  { self.stop }
+        Signal.trap('TERM') { self.stop }
+        Signal.trap('PIPE') { self.stop }
+
         EM.run do
           @start_time = Time.now
 
@@ -73,16 +77,23 @@ module PipelineToolkit
         DefaultLogger.error "#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n")
         raise e
       ensure
-        shutdown
+        cleanup
       end
     end
   
     ##
-    # Stop the subscriber
+    # Stops the command. This stops the event loop, giving it enough time to clear its buffers
     #
-    def shutdown
+    def stop
       DefaultLogger.info("Shutting down #{self.class.name}")
       DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+      stop_connection { EM.stop }
+    end
+  
+    ##
+    # Cleans up any used resource, such as pipes and files.
+    #
+    def cleanup
       destroy_sys_pipe
     end
     

data/lib/pipeline_toolkit/monitoring/monitor.erb

@@ -48,7 +48,7 @@
           <td>host:</td><td><%= hostname %></td>
         </tr>
         <tr>
-          <td>throughput:</td><td><%= mps.to_i %></td>
+          <td>throughput:</td><td><%= "%.3f" % (mps || 0) %></td>
         </tr>
         <tr>
           <td>uptime:</td><td><%= uptime / 60 %>mins</td>

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 1
   - 2
-  - 16
-  version: 1.2.16
+  - 17
+  version: 1.2.17
 platform: ruby
 authors: 
 - Aisha Fenton
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-30 00:00:00 +12:00
+date: 2010-08-31 00:00:00 +12:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency