krakow 0.3.12 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec832b26d703bbb80cc726f4253e665489f1d959
-  data.tar.gz: bc00e4942eff7a0fbf36248fc88166e51cbac638
+  metadata.gz: 01fd8940f20cb30ff26c163b8851fa88cad2f986
+  data.tar.gz: d6f277aeaaaf5adcadf08dbfa51fd6baf5370218
 SHA512:
-  metadata.gz: 35b372c4ea0551163a44fed973eb1b456dd41f3f219b23f6645117858d77ace6f817f515b880b01d1194cc9316d9b42a4bcbe893c91fa664abf97445e08814fb
-  data.tar.gz: 4046e076dfcfd812796622c37a2ec671be021d2ec842efb6ba7898307c91de0b5eca93b7ee06b210cf96822fee2cf1dc73ff56a72666b7bb2dde77701737aec3
+  metadata.gz: 86b969cfe4a3894983809cbe142497a658fc5bf369021fc6294629cde4443646be242bc4c3a870dab1efcd2da1615d7ba89c30ad04806187fb8448633d897778
+  data.tar.gz: 793e1ae1de7689b41fdd8c495404c238f7acd77a4520a08d4803607815205ec21cce4008c01781cc974128f347347e40a397b1657019da02d0d3964481ca6ea9

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## v0.4.0
+* Prevent duplicate connection consumption loops (#23) (thanks @phopkins)
+* Refactor connection implementation to prevent crashing blocks (#22) (thanks @phopkins)
+* Properly calculate message sizes using byte length, not string length (#24) (thanks @phopkins)
+* Clear responses prior to message transmission (#20) (thanks @i2amsam)
+* Rebuild testing specs (not fully covered, but a good start)
+* Consumer and Producer provide better connection failure recovery
+* Fix in-flight issues within distribution on connection failures
+
+_NOTE: Large portions of `Krakow::Connection` has been refactored_
+
 ## v0.3.12
 * Update Consumer#confirm and Consumer#touch to rescue out lookups and abort
 

data/README.md

@@ -218,6 +218,8 @@ Or, run part of them:
 bundle exec ruby test/specs/consumer_spec.rb
 ```
 
+*NOTE*: the specs expect that `nsqd` and `nsqlookupd` are available in `$PATH`
+
 ### It doesn't work
 
 Create an issue on the github repository
@@ -240,6 +242,8 @@ Create an issue, or even better, send a PR.
 
 # Contributors
 
+* Pete Hopkins (@phopkins)
+* Sam Phillips (@i2amsam)
 * Brendan Schwartz (@bschwartz)
 * Thomas Holmes (@thomas-holmes)
 * Jeremy Hinegardner (@copiousfreetime)

data/krakow.gemspec

@@ -10,10 +10,13 @@ Gem::Specification.new do |s|
   s.description = 'NSQ ruby library'
   s.license = 'Apache 2.0'
   s.require_path = 'lib'
-  s.add_dependency 'celluloid-io'
-  s.add_dependency 'http'
-  s.add_dependency 'multi_json'
-  s.add_dependency 'digest-crc'
+  s.add_runtime_dependency 'celluloid'
+  s.add_runtime_dependency 'http'
+  s.add_runtime_dependency 'multi_json'
+  s.add_runtime_dependency 'digest-crc'
+  s.add_development_dependency 'childprocess'
+  s.add_development_dependency 'snappy'
+  s.add_development_dependency 'minitest'
   s.files = Dir['lib/**/*'] + %w(krakow.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
   s.extra_rdoc_files = %w(CHANGELOG.md CONTRIBUTING.md LICENSE)
 end

data/lib/krakow.rb

@@ -1,4 +1,10 @@
 require 'krakow/version'
+require 'celluloid'
+
+if(ENV['DEBUG'])
+  Celluloid.task_class = Celluloid::TaskThread
+end
+
 require 'celluloid/autostart'
 require 'multi_json'
 
@@ -13,6 +19,7 @@ module Krakow
   autoload :Distribution, 'krakow/distribution'
   autoload :Error, 'krakow/exceptions'
   autoload :FrameType, 'krakow/frame_type'
+  autoload :Ksocket, 'krakow/ksocket'
   autoload :Producer, 'krakow/producer'
   autoload :Utils, 'krakow/utils'
 

data/lib/krakow/command/auth.rb

@@ -18,7 +18,7 @@ module Krakow
 
       def to_line
         scrt = secret.to_s
-        [name, "\n", scrt.length, scrt].pack('a*a*a*a*l>a*')
+        [name, "\n", scrt.bytesize, scrt].pack('a*a*a*a*l>a*')
       end
 
       class << self

data/lib/krakow/command/identify.rb

@@ -36,7 +36,7 @@ module Krakow
           end.compact.flatten
         ]
         payload = MultiJson.dump(filtered)
-        [name, "\n", payload.length, payload].pack('a*a*l>a*')
+        [name, "\n", payload.bytesize, payload].pack('a*a*l>a*')
       end
 
       class << self

data/lib/krakow/command/mpub.rb

@@ -19,9 +19,9 @@ module Krakow
       def to_line
         formatted_messages = messages.map do |message|
           message = message.to_s
-          [message.length, message].pack('l>a*')
+          [message.bytesize, message].pack('l>a*')
         end.join
-        [name, ' ', topic_name, "\n", formatted_messages.length, messages.size, formatted_messages].pack('a*a*a*a*l>l>a*')
+        [name, ' ', topic_name, "\n", formatted_messages.bytesize, messages.size, formatted_messages].pack('a*a*a*a*l>l>a*')
       end
 
       class << self

data/lib/krakow/command/pub.rb

@@ -19,7 +19,7 @@ module Krakow
 
       def to_line
         msg = message.to_s
-        [name, ' ', topic_name, "\n", msg.length, msg].pack('a*a*a*a*l>a*')
+        [name, ' ', topic_name, "\n", msg.bytesize, msg].pack('a*a*a*a*l>a*')
       end
 
       class << self

data/lib/krakow/connection.rb

@@ -1,5 +1,4 @@
 require 'krakow'
-require 'celluloid/io'
 
 module Krakow
 
@@ -21,7 +20,7 @@ module Krakow
     # @!parse include Krakow::Utils::Lazy::InstanceMethods
     # @!parse extend Krakow::Utils::Lazy::ClassMethods
 
-    include Celluloid::IO
+    include Celluloid
 
     # Available connection features
     FEATURES = [
@@ -43,14 +42,14 @@ module Krakow
     # List of features that may be enabled by the client
     ENABLEABLE_FEATURES = [:tls_v1, :snappy, :deflate, :auth_required]
 
-    finalizer :goodbye_my_love!
+    finalizer :connection_cleanup
 
     # @return [Hash] current configuration for endpoint
     attr_reader :endpoint_settings
-    # @return [Socket-ish] underlying socket like instance
+    # @return [Ksocket] underlying socket like instance
     attr_reader :socket
-
-    attr_reader :reconnect_notifier, :running
+    # @return [TrueClass, FalseClass]
+    attr_reader :running
 
     # @!group Attributes
 
@@ -64,7 +63,7 @@ module Krakow
     attribute :topic, String
     attribute :channel, String
     attribute :version, String, :default => 'v2'
-    attribute :queue, Queue, :default => ->{ Queue.new }
+    attribute :queue, [Queue, Consumer::Queue], :default => ->{ Queue.new }
     attribute :callbacks, Hash, :default => ->{ Hash.new }
     attribute :responses, Queue, :default => ->{ Queue.new }
     attribute :notifier, [Celluloid::Signals, Celluloid::Condition, Celluloid::Actor]
@@ -95,10 +94,6 @@ module Krakow
     # @option args [Hash] :feature_args options for connection features
     def initialize(args={})
       super
-      @reconnect_notifier = Celluloid::Signals.new
-      @socket_retries = 0
-      @socket_max_retries = 10
-      @reconnect_pause = 0.5
       @endpoint_settings = {}
       @running = false
     end
@@ -118,6 +113,7 @@ module Krakow
     # @return [nil]
     def init!
       connect!
+      async.process_to_queue!
       nil
     end
 
@@ -126,13 +122,16 @@ module Krakow
     # @param message [Krakow::Message] message to send
     # @return [TrueClass, Krakow::FrameType] response if expected or true
     def transmit(message)
+      unless(message.respond_to?(:to_line))
+        abort TypeError.new("Expecting type `Krakow::FrameType` but received `#{message.class}`")
+      end
       output = message.to_line
       response_wait = wait_time_for(message)
       if(response_wait > 0)
         transmit_with_response(message, response_wait)
       else
         debug ">>> #{output}"
-        safe_socket{|socket| socket.write output }
+        socket.put(output)
         true
       end
     end
@@ -142,8 +141,8 @@ module Krakow
     # @param message [Krakow::Message] message to send
     # @return [Krakow::FrameType] response
     def transmit_with_response(message, wait_time)
-      safe_socket{|socket| socket.write(message.to_line) }
       responses.clear
+      socket.put(message.to_line)
       response = nil
       (wait_time / response_interval).to_i.times do |i|
         response = responses.pop unless responses.empty?
@@ -168,16 +167,11 @@ module Krakow
     # Destructor method for cleanup
     #
     # @return [nil]
-    def goodbye_my_love!
+    def connection_cleanup
       debug 'Tearing down connection'
-      if(socket && !socket.closed?)
-        [lambda{ socket.write Command::Cls.new.to_line}, lambda{socket.close}].each do |action|
-          begin
-            action.call
-          rescue IOError, SystemCallError => e
-            warn "Socket error encountered during teardown: #{e.class}: #{e}"
-          end
-        end
+      @running = false
+      if(connected?)
+        socket.terminate
       end
       @socket = nil
       info 'Connection torn down'
@@ -190,22 +184,19 @@ module Krakow
     # @raise [Error::ConnectionUnavailable] socket is closed
     def receive
       debug 'Read wait for frame start'
-      buf = socket.recv(8)
+      buf = socket.get(8)
       if(buf)
         @receiving = true
         debug "<<< #{buf.inspect}"
         struct = FrameType.decode(buf)
         debug "Decoded structure: #{struct.inspect}"
-        struct[:data] = socket.read(struct[:size])
+        struct[:data] = socket.get(struct[:size])
         debug "<<< #{struct[:data].inspect}"
         @receiving = false
         frame = FrameType.build(struct)
         debug "Struct: #{struct.inspect} Frame: #{frame.inspect}"
         frame
       else
-        if(socket.closed?)
-          abort Error::ConnectionUnavailable.new("#{self} encountered closed socket!")
-        end
         nil
       end
     end
@@ -219,19 +210,20 @@ module Krakow
     #
     # @return [nil]
     def process_to_queue!
-      @running = true
-      while(@running)
-        begin
+      unless(@running)
+        @running = true
+        while(@running)
           message = handle(receive)
           if(message)
             debug "Adding message to queue #{message}"
             queue << message
-            notifier.broadcast(message) if notifier
+            if(notifier)
+              warn "Sending new message notification: #{notifier} - #{message}"
+              notifier.broadcast(message)
+            end
+          else
+            debug 'Received `nil` message. Ignoring.'
           end
-        rescue Error::ConnectionUnavailable => e
-          warn "Failed to receive message: #{e.class} - #{e}"
-          @running = false
-          async.reconnect!
         end
       end
       nil
@@ -270,7 +262,11 @@ module Krakow
       callback = callbacks[type]
       if(callback)
         debug "Processing connection callback for #{type.inspect} (#{callback.inspect})"
-        callback[:actor].send(callback[:method], *(args + [current_actor]))
+        if(callback[:actor].alive?)
+          callback[:actor].send(callback[:method], *(args + [current_actor]))
+        else
+          error "Expected actor for callback processing is not alive! (type: `#{type.inspect}`)"
+        end
       else
         debug "No connection callback defined for #{type.inspect}"
         args.size == 1 ? args.first : args
@@ -313,7 +309,7 @@ module Krakow
       ident = Command::Identify.new(
         expected_features
       )
-      safe_socket{|socket| socket.write(ident.to_line) }
+      socket.put(ident.to_line)
       response = receive
       if(expected_features[:feature_negotiation])
         begin
@@ -387,87 +383,33 @@ module Krakow
 
     # @return [TrueClass, FalseClass] underlying socket is connected
     def connected?
-      socket && !socket.closed?
-    end
-
-    protected
-
-    # Destruct the underlying socket
-    #
-    # @return [nil]
-    def teardown_socket
-      if(socket && (socket.closed? || socket.eof?))
-        socket.close unless socket.closed?
-        @socket = nil
-        warn 'Existing socket instance has been destroyed from this connection'
-      end
-      nil
-    end
-
-    # Provides socket failure state handling around given block
-    #
-    # @yield [socket] execute within socket safety layer
-    # @yieldparam [socket] underlying socket
-    # @return [Object] result of executed block
-    def safe_socket(*args)
       begin
-        if(socket.nil? || socket.closed?)
-          raise Error::ConnectionUnavailable.new 'Current connection is closed!'
-        end
-        result = yield socket if block_given?
-        result
-      rescue Error::ConnectionUnavailable, SystemCallError, IOError => e
-        warn "Safe socket encountered error (socket in failed state): #{e.class}: #{e}"
-        reconnect!
-        retry
-      rescue Celluloid::Error => e
-        warn "Internal error encountered. Allowing exception to bubble. #{e.class}: #{e}"
-        abort e
-      rescue Exception => e
-        warn "!!! Unexpected error encountered within safe socket: #{e.class}: #{e}"
-        raise
+        !!(socket && socket.alive?)
+      rescue Celluloid::DeadActorError
+        false
       end
     end
 
-    # Reconnect the underlying socket
-    #
-    # @return [nil]
-    def reconnect!
-      begin
-        if(@socket_max_retries <= @socket_retries)
-          abort ConnectionFailure.new "Failed to re-establish connection after #{@socket_retries} tries."
-        end
-        pause_interval = @reconnect_pause * @socket_retries
-        @socket_retries += 1
-        warn "Pausing for #{pause_interval} seconds before reconnect"
-        sleep(pause_interval)
-        init!
-        @socket_retries = 0
-      rescue Celluloid::Error => e
-        warn "Internal error encountered. Allowing exception to bubble. #{e.class}: #{e}"
-        abort e
-      rescue SystemCallError, IOError => e
-        error "Reconnect error encountered: #{e.class} - #{e}"
-        retry
-      end
-      callback_for(:reconnect)
-      nil
-    end
+    protected
 
     # Connect the underlying socket
     #
     # @return [nil]
     def connect!
       debug 'Initializing connection'
-      if(@socket)
-        @socket.close unless @socket.closed?
-        @socket = nil
+      unless(@connecting)
+        @connecting = true
+        if(socket && socket.alive?)
+          socket.terminate
+          @socket = nil
+        end
+        @socket = Ksocket.new(:host => host, :port => port)
+        self.link socket
+        socket.put version.rjust(4).upcase
+        identify_and_negotiate
+        info 'Connection initialized'
+        @connecting = false
       end
-      @socket = Celluloid::IO::TCPSocket.new(host, port)
-      safe_socket{|socket| socket.write version.rjust(4).upcase}
-      identify_and_negotiate
-      async.process_to_queue!
-      info 'Connection initialized'
       nil
     end
 

data/lib/krakow/consumer.rb

@@ -4,6 +4,8 @@ module Krakow
   # Consume messages from a server
   class Consumer
 
+    autoload :Queue, 'krakow/consumer/queue'
+
     include Utils::Lazy
     # @!parse include Krakow::Utils::Lazy::InstanceMethods
     # @!parse extend Krakow::Utils::Lazy::ClassMethods
@@ -11,7 +13,7 @@ module Krakow
     include Celluloid
 
     trap_exit :connection_failure
-    finalizer :goodbye_my_love!
+    finalizer :consumer_cleanup
 
     attr_reader :connections, :discovery, :distribution, :queue
 
@@ -42,30 +44,53 @@ module Krakow
         arguments[:connection_options] || {}
       )
       @connections = {}
+      @queue = Queue.new(
+        current_actor,
+        :removal_callback => :remove_message
+      )
       @distribution = Distribution::Default.new(
         :max_in_flight => max_in_flight,
         :backoff_interval => backoff_interval,
         :consumer => current_actor
       )
-      @queue = Queue.new
       if(nsqlookupd)
         debug "Connections will be established via lookup #{nsqlookupd.inspect}"
         @discovery = Discovery.new(:nsqlookupd => nsqlookupd)
         discover
       elsif(host && port)
-        debug "Connection will be established via direct connection #{host}:#{port}"
-        connection = build_connection(host, port, queue)
-        if(register(connection))
-          info "Registered new connection #{connection}"
-          distribution.redistribute!
-        else
-          abort Error::ConnectionFailure.new("Failed to establish subscription at provided end point (#{host}:#{port}")
-        end
+        direct_connect
       else
         abort Error::ConfigurationError.new('No connection information provided!')
       end
     end
 
+    # @return [TrueClass, FalseClass] currently connected to at least
+    #   one nsqd
+    def connected?
+      !!connections.values.any? do |con|
+        begin
+          con.connected?
+        rescue Celluloid::DeadActorError
+          false
+        end
+      end
+    end
+
+    # Connect to nsqd instance directly
+    #
+    # @return [Connection]
+    def direct_connect
+      debug "Connection will be established via direct connection #{host}:#{port}"
+      connection = build_connection(host, port, queue)
+      if(register(connection))
+        info "Registered new connection #{connection}"
+        distribution.redistribute!
+      else
+        abort Error::ConnectionFailure.new("Failed to establish subscription at provided end point (#{host}:#{port}")
+      end
+      connection
+    end
+
     # Returns [Krakow::Connection] associated to key
     #
     # @param key [Object] identifier
@@ -82,12 +107,17 @@ module Krakow
     # Instance destructor
     #
     # @return [nil]
-    def goodbye_my_love!
+    def consumer_cleanup
       debug 'Tearing down consumer'
+      if(distribution && distribution.alive?)
+        distribution.terminate
+      end
+      if(queue && queue.alive?)
+        queue.terminate
+      end
       connections.values.each do |con|
         con.terminate if con.alive?
       end
-      distribution.terminate if distribution && distribution.alive?
       info 'Consumer torn down'
       nil
     end
@@ -113,13 +143,11 @@ module Krakow
             :handle => {
               :actor => current_actor,
               :method => :process_message
-            },
-            :reconnect => {
-              :actor => current_actor,
-              :method => :connection_reconnect
             }
           }
         )
+        queue.register_connection(connection)
+        connection
       rescue => e
         error "Failed to build connection (host: #{host} port: #{port} queue: #{queue}) - #{e.class}: #{e}"
         debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
@@ -132,22 +160,30 @@ module Krakow
     # @param message [Krakow::FrameType]
     # @param connection [Krakow::Connection]
     # @return [Krakow::FrameType]
+    # @note If we receive a message that is already in flight, attempt
+    #   to scrub message from wait queue. If message is found, retry
+    #   distribution registration. If message is not found, assume it
+    #   is currently being processed and do not allow new message to
+    #   be queued
     def process_message(message, connection)
+      discard = false
       if(message.is_a?(FrameType::Message))
-        distribution.register_message(message, connection.identifier)
         message.origin = current_actor
+        message.connection = connection
+        retried = false
+        begin
+          distribution.register_message(message, connection.identifier)
+        rescue KeyError => e
+          if(!retried && queue.scrub_duplicate_message(message))
+            retried = true
+            retry
+          else
+            error "Received message is currently in flight and not in wait queue. Discarding! (#{message})"
+            discard = true
+          end
+        end
       end
-      message
-    end
-
-    # Action to take when a connection has reconnected
-    #
-    # @param connection [Krakow::Connection]
-    # @return [nil]
-    def connection_reconnect(connection)
-      connection.transmit(Command::Sub.new(:topic_name => topic, :channel_name => channel))
-      distribution.set_ready_for(connection)
-      nil
+      discard ? nil : message
     end
 
     # Send RDY for connection based on distribution rules
@@ -214,17 +250,30 @@ module Krakow
     # @param reason [Exception] reason for termination
     # @return [nil]
     def connection_failure(actor, reason)
-      connections.delete_if do |key, value|
-        if(value == actor && reason.nil?)
-          warn "Connection failure detected. Removing connection: #{key} - #{reason || 'no reason provided'}"
-          begin
-            distribution.remove_connection(key)
-          rescue Error::ConnectionUnavailable, Error::ConnectionFailure
-            warn 'Caught connection unavailability'
-          end
-          distribution.redistribute!
-          true
+      if(reason && key = connections.key(actor))
+        warn "Connection failure detected. Removing connection: #{key} - #{reason}"
+        connections.delete(key)
+        begin
+          distribution.remove_connection(key)
+        rescue Error::ConnectionUnavailable, Error::ConnectionFailure
+          warn 'Caught connection unavailability'
         end
+        queue.deregister_connection(key)
+        distribution.redistribute!
+        direct_connect unless discovery
+      end
+      nil
+    end
+
+    # Remove message
+    #
+    # @param messages [Array<FrameType::Message>]
+    # @return [NilClass]
+    # @note used mainly for queue callback
+    def remove_message(messages)
+      [messages].flatten.compact.each do |msg|
+        distribution.unregister_message(msg.message_id)
+        update_ready!(msg.connection)
       end
       nil
     end
@@ -237,11 +286,12 @@ module Krakow
     def confirm(message_id)
       message_id = message_id.message_id if message_id.respond_to?(:message_id)
       begin
-        distribution.in_flight_lookup(message_id) do |connection|
-          distribution.unregister_message(message_id)
+        begin
+          connection = distribution.in_flight_lookup(message_id)
           connection.transmit(Command::Fin.new(:message_id => message_id))
           distribution.success(connection.identifier)
-          update_ready!(connection)
+        rescue => e
+          abort e
         end
         true
       rescue KeyError => e
@@ -251,7 +301,12 @@ module Krakow
         error "Lookup of message for confirmation failed! <Message ID: #{message_id} - Error: #{e}>"
         abort e
       rescue Error::ConnectionUnavailable => e
-        retry
+        abort e
+      rescue Celluloid::DeadActorError
+        abort Error::ConnectionUnavailable.new
+      ensure
+        con = distribution.unregister_message(message_id)
+        update_ready!(con) if con
       end
     end
     alias_method :finish, :confirm

data/lib/krakow/consumer/queue.rb

@@ -0,0 +1,151 @@
+require 'krakow'
+
+module Krakow
+  # Consume messages from a server
+  class Consumer
+
+    class Queue
+
+      include Celluloid
+      include Utils::Lazy
+
+      # @return [Consumer]
+      attr_reader :consumer
+      # @return [Array] order of message removal
+      attr_reader :pop_order
+      # @return [Symbol] callback method name
+      attr_reader :removal_callback
+
+      # Create new consumer queue instance
+      #
+      # @param consumer [Consumer]
+      # @return [self]
+      def initialize(consumer, *args)
+        opts = args.detect{|x| x.is_a?(Hash)}
+        @consumer = consumer
+        @removal_callback = opts[:removal_callback]
+        @messages = {}
+        @pop_order = []
+        @cleaner = nil
+      end
+
+      # Message container
+      #
+      # @yieldparam [Hash] messages
+      # @return [Hash] messages or block result
+      def messages
+        if(block_given?)
+          yield @messages
+        else
+          @messages
+        end
+      end
+
+      # Register a new connection
+      #
+      # @param connection [Connection]
+      # @return [TrueClass]
+      def register_connection(connection)
+        messages do |collection|
+          collection[connection.identifier] = []
+        end
+        true
+      end
+
+      # Remove connection registration and remove all messages
+      #
+      # @param identifier [String] connection identifier
+      # @return [Array<FrameType::Message>] messages queued for deregistered connection
+      def deregister_connection(identifier)
+        messages do |collection|
+          removed = collection.delete(identifier)
+          pop_order.delete(identifier)
+          removed
+        end
+      end
+
+      # Push new message into queue
+      #
+      # @param message [FrameType::Message]
+      # @return [self]
+      def push(message)
+        unless(message.is_a?(FrameType::Message))
+          abort TypeError.new "Expecting `FrameType::Message` but received `#{message.class}`!"
+        end
+        messages do |collection|
+          begin
+            collection[message.connection.identifier] << message
+            pop_order << message.connection.identifier
+          rescue Celluloid::DeadActorError
+            abort Error::ConnectionUnavailable.new
+          end
+        end
+        signal(:new_message)
+        current_actor
+      end
+      alias_method :<<, :push
+      alias_method :enq, :push
+
+      # Pop first item off the queue
+      #
+      # @return [Object]
+      def pop
+        message = nil
+        until(message)
+          wait(:new_message) if pop_order.empty?
+          messages do |collection|
+            key = pop_order.shift
+            if(key)
+              message = collection[key].shift
+              message = validate_message(message)
+            end
+          end
+        end
+        message
+      end
+      alias_method :deq, :pop
+
+      # @return [Integer] number of queued messages
+      def size
+        messages do |collection|
+          collection.values.map(&:size).inject(&:+)
+        end
+      end
+
+      # Remove duplicate message from queue if possible
+      #
+      # @param message [FrameType::Message]
+      # @return [TrueClass, FalseClass]
+      def scrub_duplicate_message(message)
+        messages do |collection|
+          idx = collection[message.connection.identifier].index do |msg|
+            msg.message_id == message.message_id
+          end
+          if(idx)
+            msg = collection[message.connection.identifier].delete_at(idx)
+            if(removal_callback)
+              consumer.send(removal_callback, [message])
+            end
+            true
+          else
+            false
+          end
+        end
+      end
+
+      # Validate message
+      def validate_message(message)
+        if(message.instance_stamp > message.instance_stamp + (message.connection.endpoint_settings[:msg_timeout] / 1000.0))
+          warn "Message exceeded timeout! Discarding. (#{message})"
+          if(removal_callback)
+            consumer.send(removal_callback, [message])
+          end
+          nil
+        else
+          message
+        end
+      end
+
+    end
+  end
+end

data/lib/krakow/distribution.rb

@@ -52,15 +52,24 @@ module Krakow
     # Remove message metadata from registry
     #
     # @param message [Krakow::FrameType::Message, String] message or ID
-    # @return [Krakow::Connection]
+    # @return [Krakow::Connection, NilClass]
     def unregister_message(message)
       msg_id = message.respond_to?(:message_id) ? message.message_id : message.to_s
       connection = connection_lookup(flight_record[msg_id])
-      registry_info = registry_lookup(connection.identifier)
       flight_record.delete(msg_id)
-      registry_info[:in_flight] -= 1
-      calculate_ready!(connection.identifier)
-      connection
+      if(connection)
+        begin
+          ident = connection.identifier
+          registry_info = registry_lookup(ident)
+          registry_info[:in_flight] -= 1
+          calculate_ready!(ident)
+          connection
+        rescue Celluloid::DeadActorError
+          warn 'Connection is dead. No recalculation applied on ready.'
+        end
+      else
+        warn 'No connection associated to message via lookup. No recalculation applied on ready.'
+      end
     end
 
     # Return the currently configured RDY value for given connnection
@@ -97,10 +106,14 @@ module Krakow
     # @param connection_identifier [String]
     # @return [Integer]
     def register_message(message, connection_identifier)
-      registry_info = registry_lookup(connection_identifier)
-      registry_info[:in_flight] += 1
-      flight_record[message.message_id] = connection_identifier
-      calculate_ready!(connection_identifier)
+      if(flight_record[message.message_id])
+        abort KeyError.new "Message is already registered in flight record! (#{message.message_id})"
+      else
+        registry_info = registry_lookup(connection_identifier)
+        registry_info[:in_flight] += 1
+        flight_record[message.message_id] = connection_identifier
+        calculate_ready!(connection_identifier)
+      end
     end
 
     # Add connection to make available for RDY distribution
@@ -128,8 +141,8 @@ module Krakow
       registry.delete(connection_identifier)
       # remove any in flight messages
       flight_record.delete_if do |k,v|
-        if(k == connection_identifier)
-          warn "Removing in flight reference due to failed connection: #{k}"
+        if(v == connection_identifier)
+          warn "Removing in flight reference due to failed connection: #{v}"
           true
         end
       end
@@ -156,7 +169,11 @@ module Krakow
         abort Krakow::Error::LookupFailed.new("Failed to locate in flight message (ID: #{msg_id})")
       end
       if(block_given?)
-        yield connection
+        begin
+          yield connection
+        rescue => e
+          abort e
+        end
       else
         connection
       end

data/lib/krakow/frame_type/message.rb

@@ -5,8 +5,9 @@ module Krakow
     # Message received from server
     class Message < FrameType
 
-      # @return [Krakow::Consumer]
-      attr_accessor :origin
+      # @return [Float] time of message instance creation
+      attr_reader :instance_stamp
+      attr_accessor :origin, :connection
 
       # @!group Attributes
 
@@ -22,6 +23,11 @@ module Krakow
 
       # @!endgroup
 
+      def initialize(*args)
+        super
+        @instance_stamp = Time.now.to_f
+      end
+
       # Message content
       #
       # @return [String]
@@ -38,6 +44,15 @@ module Krakow
         @origin
       end
 
+      # @return [Krakow::Connection]
+      def connection
+        unless(@connection)
+          error 'No origin connection has been specified for this message'
+          abort Krakow::Error::ConnectionNotFound.new('No connection specified for this message')
+        end
+        @connection
+      end
+
       # Proxy to [Krakow::Consumer#confirm]
       def confirm(*args)
         origin.confirm(*[self, *args].compact)

data/lib/krakow/ksocket.rb

@@ -0,0 +1,102 @@
+require 'krakow'
+require 'socket'
+
+module Krakow
+  class Ksocket
+
+    include Utils::Lazy
+    include Celluloid
+
+    # @return [String]
+    attr_reader :buffer
+
+    finalizer :closedown_socket
+
+    # Teardown helper
+    def closedown_socket
+      @writing = @reading = false
+      if(socket && !socket.closed?)
+        socket.close
+      end
+    end
+
+    # Create new socket wrapper
+    #
+    # @param args [Hash]
+    # @option args [Socket-ish] :socket
+    # @option args [String] :host
+    # @option args [Integer] :port
+    # @return [self]
+    def initialize(args={})
+      if(args[:socket])
+        @socket = args[:socket]
+      else
+        unless([:host, :port].all?{|k| args.include?(k)})
+          raise ArgumentError.new 'Missing required arguments. Expecting `:socket` or `:host` and `:port`.'
+        end
+        @socket = TCPSocket.new(args[:host], args[:port])
+      end
+      @buffer = ''
+      async.read_loop
+    end
+
+    # @return [TrueClass, FalseClass] read loop enabled
+    def reading?
+      !!@reading
+    end
+
+    # Read from socket and push into local Queue
+    def read_loop
+      unless(reading?)
+        @reading = true
+        while(reading?)
+          res = defer do
+            Kernel.select([socket], nil, nil, nil)
+            socket{|s| s.readpartial(1024)}
+          end
+          if(res)
+            debug "Received content from socket: #{res.inspect}"
+            buffer << res
+            signal(:content_read)
+          else
+            debug 'No content received from socket read. Ignoring.'
+          end
+        end
+      end
+    end
+
+    # Fetch bytes from socket
+    #
+    # @param n [Integer]
+    # @return [String]
+    def get(n)
+      until(buffer.length >= n)
+        wait(:content_read)
+      end
+      buffer.slice!(0, n)
+    end
+    alias_method :recv, :get
+    alias_method :read, :get
+    alias_method :sysread, :get
+    alias_method :readpartial, :get
+
+    # Push bytes to socket
+    #
+    # @param line [String]
+    # @return [Integer]
+    def put(line)
+      socket{|s| s.write(line)}
+    end
+    alias_method :write, :put
+
+    # @return [Socket]
+    def socket
+      if(block_given?)
+        yield @socket
+      else
+        @socket
+      end
+    end
+
+  end
+end

data/lib/krakow/producer.rb

@@ -14,12 +14,13 @@ module Krakow
     include Celluloid
 
     trap_exit  :connection_failure
-    finalizer :goodbye_my_love!
+    finalizer :producer_cleanup
 
     # set exclusive methods
     exclusive :write
 
     attr_reader :connection
+    attr_reader :notifier
 
     # @!group Attributes
 
@@ -49,6 +50,7 @@ module Krakow
     #
     # @return nil
     def connect
+      @connecting = true
       info "Establishing connection to: #{host}:#{port}"
       begin
         con_args = connection_options[:options].dup.tap do |args|
@@ -62,13 +64,14 @@ module Krakow
           end
         end
         @connection = Connection.new(con_args)
-        connection.init!
-        self.link connection
-        info "Connection established: #{connection}"
+        @connection.init!
+        self.link @connection
+        info "Connection established: #{@connection}"
         nil
       rescue => e
         abort e
       end
+      @connecting = false
     end
 
     # @return [String] stringify object
@@ -78,28 +81,38 @@ module Krakow
 
     # @return [TrueClass, FalseClass] currently connected to server
     def connected?
-      !!(connection && connection.alive? && connection.connected?)
+      begin
+        !!(!@connecting &&
+          connection &&
+          connection.alive? &&
+          connection.connected?)
+      rescue Celluloid::DeadActorError
+        false
+      end
     end
 
     # Process connection failure and attempt reconnection
     #
     # @return [TrueClass]
-    def connection_failure(*args)
-      @connection = nil
-      begin
-        warn "Connection failure detected for #{host}:#{port}"
-        connect
-      rescue => e
-        warn "Failed to establish connection to #{host}:#{port}. Pausing #{reconnect_interval} before retry"
-        sleep reconnect_interval
-        connect
+    def connection_failure(obj, reason)
+      if(obj == connection && !reason.nil?)
+        begin
+          @connection = nil
+          warn "Connection failure detected for #{host}:#{port} - #{reason}"
+          obj.terminate if obj.alive?
+          connect
+        rescue => reason
+          warn "Failed to establish connection to #{host}:#{port}. Pausing #{reconnect_interval} before retry"
+          sleep reconnect_interval
+          retry
+        end
       end
       true
     end
 
     # Instance destructor
     # @return nil
-    def goodbye_my_love!
+    def producer_cleanup
       debug 'Tearing down producer'
       if(connection && connection.alive?)
         connection.terminate
@@ -112,13 +125,15 @@ module Krakow
     # Write message to server
     #
     # @param message [String] message to write
-    # @return [Krakow::FrameType::Error,nil]
+    # @return [Krakow::FrameType, TrueClass]
+    # @note if connection response wait is set to 0, writes will
+    #   return a `true` value on completion
     # @raise [Krakow::Error::ConnectionUnavailable]
     def write(*message)
       if(message.empty?)
         abort ArgumentError.new 'Expecting one or more messages to send. None provided.'
       end
-      if(connection && connection.alive?)
+      begin
         if(message.size > 1)
           debug 'Multiple message publish'
           connection.transmit(
@@ -136,8 +151,10 @@ module Krakow
             )
           )
         end
-      else
-        abort Error::ConnectionUnavailable.new 'Remote connection is unavailable!'
+      rescue Celluloid::Task::TerminatedError
+        abort Error::ConnectionUnavailable.new 'Connection is currently unavailable'
+      rescue => e
+        abort e
       end
     end
 

data/lib/krakow/producer/http.rb

@@ -72,7 +72,12 @@ module Krakow
         begin
           response = MultiJson.load(response.body.to_s)
         rescue MultiJson::LoadError
-          response = {'status_code' => response.code, 'status_txt' => response.body.to_s, 'data' => nil}
+          response = {
+            'status_code' => response.code,
+            'status_txt' => response.body.to_s,
+            'response' => response.body.to_s,
+            'data' => nil,
+          }
         end
         Response.new(response)
       end

data/lib/krakow/version.rb

@@ -1,4 +1,4 @@
 module Krakow
   # Current version
-  VERSION = Gem::Version.new('0.3.12')
+  VERSION = Gem::Version.new('0.4.0')
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: krakow
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.4.0
 platform: ruby
 authors:
 - Chris Roberts
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-15 00:00:00.000000000 Z
+date: 2015-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: celluloid-io
+  name: celluloid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -66,6 +66,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: childprocess
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: NSQ ruby library
 email: code@chrisroberts.org
 executables: []
@@ -99,6 +141,7 @@ files:
 - lib/krakow/connection_features/snappy_frames.rb
 - lib/krakow/connection_features/ssl.rb
 - lib/krakow/consumer.rb
+- lib/krakow/consumer/queue.rb
 - lib/krakow/discovery.rb
 - lib/krakow/distribution.rb
 - lib/krakow/distribution/default.rb
@@ -107,6 +150,7 @@ files:
 - lib/krakow/frame_type/error.rb
 - lib/krakow/frame_type/message.rb
 - lib/krakow/frame_type/response.rb
+- lib/krakow/ksocket.rb
 - lib/krakow/producer.rb
 - lib/krakow/producer/http.rb
 - lib/krakow/utils.rb