krakow 0.2.2 → 0.3.0

data/lib/krakow/connection_features/ssl.rb

@@ -1,24 +1,47 @@
+require 'openssl'
+require 'krakow'
+
 module Krakow
   module ConnectionFeatures
+    # SSL functionality
     module Ssl
+      # SSL-able IO
       class Io
 
         attr_reader :_socket
 
+        # Create new SSL-able IO
+        #
+        # @param io [IO] IO to wrap
+        # @param args [Hash]
+        # @option args [Hash] :ssl_context
+        # @return [Io]
         def initialize(io, args={})
           ssl_socket_arguments = [io]
           if(args[:ssl_context])
-            # ssl_socket_arguments << SSLContext.new
+            validate_ssl_args!(args[:ssl_context])
+            context = OpenSSL::SSL::SSLContext.new
+            context.cert = OpenSSL::X509::Certificate.new(File.open(args[:ssl_context][:certificate]))
+            context.key = OpenSSL::PKey::RSA.new(File.open(args[:ssl_context][:key]))
+            ssl_socket_arguments << context
           end
           @_socket = Celluloid::IO::SSLSocket.new(*ssl_socket_arguments)
           _socket.sync = true
           _socket.connect
         end
 
+        # Proxy to underlying socket
+        #
+        # @param args [Object]
+        # @return [Object]
         def method_missing(*args)
           _socket.send(*args)
         end
 
+        # Receive bytes from the IO
+        #
+        # @param len [Integer] nuber of bytes
+        # @return [String]
         def recv(len)
           str = readpartial(len)
           if(len > str.length)
@@ -27,6 +50,25 @@ module Krakow
           str
         end
 
+        private
+
+        # Validate the SSL configuration provided
+        #
+        # @param args [Hash]
+        # @option args [String] :certificate path to certificate
+        # @option args [String] :key path to key
+        # @raise [ArgumentError, LoadError]
+        def validate_ssl_args!(args)
+          [:key, :certificate].each do |arg_key|
+            unless(args.has_key?(arg_key))
+              raise ArgumentError.new "The `:ssl_context` option requires `#{arg_key.inspect}` to be set"
+            end
+            unless(File.readable?(args[arg_key]))
+              raise LoadError.new "Unable to read the `#{arg_key.inspect}` file from the `:ssl_context` arguments"
+            end
+          end
+        end
+
       end
     end
   end

data/lib/krakow/connection_features.rb

@@ -1,6 +1,7 @@
 require 'krakow'
 
 module Krakow
+  # Features that wrap the connection
   module ConnectionFeatures
     autoload :SnappyFrames, 'krakow/connection_features/snappy_frames'
     autoload :Deflate, 'krakow/connection_features/deflate'

data/lib/krakow/consumer.rb

@@ -1,7 +1,13 @@
+require 'krakow'
+
 module Krakow
+  # Consume messages from a server
   class Consumer
 
     include Utils::Lazy
+    # @!parse include Krakow::Utils::Lazy::InstanceMethods
+    # @!parse extend Krakow::Utils::Lazy::ClassMethods
+
     include Celluloid
 
     trap_exit :connection_failure
@@ -9,25 +15,43 @@ module Krakow
 
     attr_reader :connections, :discovery, :distribution, :queue
 
+    # @!group Attributes
+
+    # @!macro [attach] attribute
+    #   @!method $1
+    #     @return [$2] the $1 $0
+    #   @!method $1?
+    #     @return [TrueClass, FalseClass] truthiness of the $1 $0
+    attribute :topic, String, :required => true
+    attribute :channel, String, :required => true
+    attribute :host, String
+    attribute :port, [String, Integer]
+    attribute :nsqlookupd, [Array, String]
+    attribute :max_in_flight, Integer, :default => 1
+    attribute :backoff_interval, Numeric
+    attribute :discovery_interval, Numeric, :default => 30
+    attribute :discovery_jitter, Numeric, :default => 10.0
+    attribute :notifier, Celluloid::Actor
+    attribute :connection_options, Hash, :default => ->{ Hash.new }
+
+    # @!endgroup
+
     def initialize(args={})
       super
-      required! :topic, :channel
-      optional :host, :port, :nslookupd, :nsqlookupd, :max_in_flight, :backoff_interval, :discovery_interval, :notifier, :connection_features
-      arguments[:max_in_flight] ||= 1
-      arguments[:discovery_interval] ||= 30
-      arguments[:connection_features] ||= {}
-      arguments[:nsqlookupd] ||= arguments[:nslookupd]
+      arguments[:connection_options] = {:features => {}, :config => {}}.merge(
+        arguments[:connection_options] || {}
+      )
       @connections = {}
       @distribution = Distribution::Default.new(
         :max_in_flight => max_in_flight,
-        :backoff_interval => backoff_interval
+        :backoff_interval => backoff_interval,
+        :consumer => current_actor
       )
       @queue = Queue.new
       if(nsqlookupd)
         debug "Connections will be established via lookup #{nsqlookupd.inspect}"
         @discovery = Discovery.new(:nsqlookupd => nsqlookupd)
-        init!
-        every(discovery_interval){ init! }
+        discover
       elsif(host && port)
         debug "Connection will be established via direct connection #{host}:#{port}"
         connection = build_connection(host, port, queue)
@@ -42,10 +66,22 @@ module Krakow
       end
     end
 
+    # Returns [Krakow::Connection] associated to key
+    #
+    # @param key [Object] identifier
+    # @return [Krakow::Connection] associated connection
+    def connection(key)
+      @connections[key]
+    end
+
+    # @return [String] stringify object
     def to_s
       "<#{self.class.name}:#{object_id} T:#{topic} C:#{channel}>"
     end
 
+    # Instance destructor
+    #
+    # @return [nil]
     def goodbye_my_love!
       debug 'Tearing down consumer'
       connections.values.each do |con|
@@ -53,43 +89,79 @@ module Krakow
       end
       distribution.terminate if distribution && distribution.alive?
       info 'Consumer torn down'
+      nil
     end
 
-    # host:: remote address
-    # port:: remote port
-    # queue:: message store queue
-    # Build new `Connection`
+    # Build a new [Krakow::Connection]
+    #
+    # @param host [String] remote host
+    # @param port [String, Integer] remote port
+    # @param queue [Queue] queue for messages
+    # @return [Krakow::Connection, nil] new connection or nil
     def build_connection(host, port, queue)
-      connection = Connection.new(
-        :host => host,
-        :port => port,
-        :queue => queue,
-        :notifier => notifier,
-        :features => connection_features,
-        :callback => {
-          :actor => current_actor,
-          :method => :process_message
-        }
-      )
+      begin
+        connection = Connection.new(
+          :host => host,
+          :port => port,
+          :queue => queue,
+          :topic => topic,
+          :channel => channel,
+          :notifier => notifier,
+          :features => connection_options[:features],
+          :features_args => connection_options[:config],
+          :callbacks => {
+            :handle => {
+              :actor => current_actor,
+              :method => :process_message
+            },
+            :reconnect => {
+              :actor => current_actor,
+              :method => :connection_reconnect
+            }
+          }
+        )
+      rescue => e
+        error "Failed to build connection (host: #{host} port: #{port} queue: #{queue}) - #{e.class}: #{e}"
+        debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+        nil
+      end
     end
 
-    # message:: FrameType
-    # connection:: Connection
-    # Process message if required
+    # Process a given message if required
+    #
+    # @param message [Krakow::FrameType]
+    # @param connection [Krakow::Connection]
+    # @return [Krakow::FrameType]
     def process_message(message, connection)
       if(message.is_a?(FrameType::Message))
-        distribution.register_message(message, connection)
+        distribution.register_message(message, connection.identifier)
+        message.origin = current_actor
       end
       message
     end
 
-    # connection:: Connection
+    # Action to take when a connection has reconnected
+    #
+    # @param connection [Krakow::Connection]
+    # @return [nil]
+    def connection_reconnect(connection)
+      connection.transmit(Command::Sub.new(:topic_name => topic, :channel_name => channel))
+      distribution.set_ready_for(connection)
+      nil
+    end
+
     # Send RDY for connection based on distribution rules
+    #
+    # @param connection [Krakow::Connection]
+    # @return [nil]
     def update_ready!(connection)
       distribution.set_ready_for(connection)
+      nil
     end
 
-    # Requests lookup and adds connections
+    # Initialize the consumer by starting lookup and adding connections
+    #
+    # @return [nil]
     def init!
       debug 'Running consumer `init!` connection builds'
       found = discovery.lookup(topic)
@@ -97,7 +169,7 @@ module Krakow
       connection = nil
       found.each do |node|
         debug "Processing discovery result: #{node.inspect}"
-        key = "#{node[:broadcast_address]}_#{node[:tcp_port]}"
+        key = Connection.identifier(node[:broadcast_address], node[:tcp_port], topic, channel)
         unless(connections[key])
           connection = build_connection(node[:broadcast_address], node[:tcp_port], queue)
           info "Registered new connection #{connection}" if register(connection)
@@ -106,16 +178,27 @@ module Krakow
         end
       end
       distribution.redistribute! if connection
+      nil
+    end
+
+    # Start the discovery interval lookup
+    #
+    # @return [nil]
+    def discover
+      init!
+      after(discovery_interval + (discovery_jitter * rand)){ discover }
     end
 
-    # connection:: Connection
-    # Registers connection with subscription. Returns false if failed
+    # Register connection with distribution
+    #
+    # @param connection [Krakow::Connection]
+    # @return [TrueClass, FalseClass] true if subscription was successful
     def register(connection)
       begin
         connection.init!
         connection.transmit(Command::Sub.new(:topic_name => topic, :channel_name => channel))
         self.link connection
-        connections["#{connection.host}_#{connection.port}"] = connection
+        connections[connection.identifier] = connection
         distribution.add_connection(connection)
         true
       rescue Error::BadResponse => e
@@ -125,40 +208,56 @@ module Krakow
       end
     end
 
-    # con:: actor
-    # reason:: Exception
-    # Remove connection from register if found
-    def connection_failure(con, reason)
+    # Remove connection references when connection is terminated
+    #
+    # @param actor [Object] terminated actor
+    # @param reason [Exception] reason for termination
+    # @return [nil]
+    def connection_failure(actor, reason)
       connections.delete_if do |key, value|
-        if(value == con)
-          warn "Connection failure detected. Removing connection: #{key}"
-          distribution.remove_connection(con)
+        if(value == actor && reason.nil?)
+          warn "Connection failure detected. Removing connection: #{key} - #{reason || 'no reason provided'}"
+          begin
+            distribution.remove_connection(key)
+          rescue Error::ConnectionUnavailable, Error::ConnectionFailure
+            warn 'Caught connection unavailability'
+          end
+          distribution.redistribute!
           true
         end
       end
-      distribution.redistribute!
+      nil
     end
 
-    # message_id:: Message ID (or message if you want to be lazy)
     # Confirm message has been processed
+    #
+    # @param message_id [String, Krakow::FrameType::Message]
+    # @return [TrueClass]
+    # @raise [KeyError] connection not found
     def confirm(message_id)
       message_id = message_id.message_id if message_id.respond_to?(:message_id)
       begin
         distribution.in_flight_lookup(message_id) do |connection|
           distribution.unregister_message(message_id)
           connection.transmit(Command::Fin.new(:message_id => message_id))
-          distribution.success(connection)
+          distribution.success(connection.identifier)
           update_ready!(connection)
         end
         true
-      rescue => e
+      rescue KeyError => e
+        error "Message confirmation failed: #{e}"
         abort e
+      rescue Error::ConnectionUnavailable => e
+        retry
       end
     end
+    alias_method :finish, :confirm
 
-    # message_id:: Message ID
-    # timeout:: Requeue timeout (default is none)
-    # Requeue message (processing failure)
+    # Requeue message (generally due to processing failure)
+    #
+    # @param message_id [String, Krakow::FrameType::Message]
+    # @param timeout [Numeric]
+    # @return [TrueClass]
     def requeue(message_id, timeout=0)
       message_id = message_id.message_id if message_id.respond_to?(:message_id)
       distribution.in_flight_lookup(message_id) do |connection|
@@ -169,11 +268,25 @@ module Krakow
             :timeout => timeout
           )
         )
-        distribution.failure(connection)
+        distribution.failure(connection.identifier)
         update_ready!(connection)
       end
       true
     end
 
+    # Touch message (to extend timeout)
+    #
+    # @param message_id [String, Krakow::FrameType::Message]
+    # @return [TrueClass]
+    def touch(message_id)
+      message_id = message_id.message_id if message_id.respond_to?(:message_id)
+      distribution.in_flight_lookup(message_id) do |connection|
+        connection.transmit(
+          Command::Touch.new(:message_id => message_id)
+        )
+      end
+      true
+    end
+
   end
 end

data/lib/krakow/discovery.rb

@@ -1,19 +1,30 @@
 require 'uri'
 require 'http'
 require 'multi_json'
+require 'krakow'
 
 module Krakow
+
+  # Provides queue topic discovery
   class Discovery
 
     include Utils::Lazy
 
-    def initialize(args={})
-      super
-      required! :nsqlookupd
-    end
+    # @!group Attributes
+
+    # @!macro [attach] attribute
+    #   @!method $1
+    #     @return [$2] the $1 $0
+    #   @!method $1?
+    #     @return [TrueClass, FalseClass] truthiness of the $1 $0
+    attribute :nsqlookupd, [Array, String], :required => true
+
+    # @!endgroup
 
-    # topic:: Topic name
-    # Return list of end points with given topic name available
+    # Get list of end points with given topic name available
+    #
+    # @param topic [String] topic name
+    # @return [Array<Hash>]
     def lookup(topic)
       result = [nsqlookupd].flatten.map do |location|
         uri = URI.parse(location)

data/lib/krakow/distribution/default.rb

@@ -1,5 +1,9 @@
+require 'krakow'
+
 module Krakow
   class Distribution
+    # Default distribution implementation. This uses a round-robin
+    # approach for less than ideal states.
     class Default < Distribution
 
       attr_reader :less_than_ideal_stack, :watch_dog
@@ -9,7 +13,7 @@ module Krakow
         @ideal = registry.size < 1 ? 0 : max_in_flight / registry.size
         debug "Distribution calculated ideal: #{ideal}"
         if(less_than_ideal?)
-          registry.each do |connection, reg_info|
+          registry.each do |connection_id, reg_info|
             reg_info[:ready] = 0
           end
           max_in_flight.times do
@@ -28,9 +32,9 @@ module Krakow
             @watch_dog = nil
           end
           connections.each do |connection|
-            current_ready = ready_for(connection)
-            calculate_ready!(connection)
-            unless(current_ready == ready_for(connection))
+            current_ready = ready_for(connection.identifier)
+            calculate_ready!(connection.identifier)
+            unless(current_ready == ready_for(connection.identifier))
               debug "Redistribution ready setting update for connection #{connection}"
               set_ready_for(connection)
             end
@@ -38,12 +42,16 @@ module Krakow
         end
       end
 
-      # Returns if `ideal` is less than 1
+      # Is ideal less than 1
+      #
+      # @return [TrueClass, FalseClass]
       def less_than_ideal?
         ideal < 1
       end
 
-      # Returns next connection to receive RDY count
+      # Find next connection to receive RDY count
+      #
+      # @return [Krakow::Connection, nil]
       def less_than_ideal_ready!
         admit_defeat = false
         connection = nil
@@ -53,18 +61,24 @@ module Krakow
             admit_defeat = true
           end
           con = less_than_ideal_stack.pop
-          connection = con unless registry_lookup(con)[:backoff_until] > Time.now.to_i
+          if(con)
+            unless(registry_lookup(con.identifier)[:backoff_until] > Time.now.to_i)
+              connection = con
+            end
+          end
         end
         if(connection)
-          registry_lookup(connection)[:ready] = 1
+          registry_lookup(connection.identifier)[:ready] = 1
           connection
         end
       end
 
-      # connection:: Connection
-      # args:: optional args (:force)
-      # Provides customized RDY set when less than ideal to round
-      # robin through connections
+      # Adds extra functionality to provide round robin RDY setting
+      # when in less than ideal state
+      #
+      # @param connection [Krakow::Connection]
+      # @param args [Symbol]
+      # @return [Krakow::FrameType::Error, nil]
       def set_ready_for(connection, *args)
         super connection
         if(less_than_ideal? && !args.include?(:force))
@@ -79,51 +93,65 @@ module Krakow
         end
       end
 
-      # connection:: Connection
       # Update connection ready count
-      def calculate_ready!(connection)
-        registry_info = registry_lookup(connection)
-        unless(less_than_ideal?)
-          registry_info[:ready] = ideal - registry_info[:in_flight]
-          if(registry_info[:ready] < 0 || registry_info[:backoff_until] > Time.now.to_i)
-            registry_info[:ready] = 0
-            registry_info[:backoff_timer].cancel if registry[:backoff_timer]
-            registry_info[:backoff_timer] = after(registry_info[:backoff_until] - Time.now.to_i) do
-              calculate_ready!(connection)
-              set_ready_for(conection) unless less_than_ideal?
+      # @param connection_identifier [String]
+      # @return [Integer, nil]
+      def calculate_ready!(connection_identifier)
+        begin
+          registry_info = registry_lookup(connection_identifier)
+          unless(less_than_ideal?)
+            registry_info[:ready] = ideal - registry_info[:in_flight]
+            if(registry_info[:ready] < 0 || registry_info[:backoff_until] > Time.now.to_i)
+              registry_info[:ready] = 0
+              registry_info[:backoff_timer].cancel if registry[:backoff_timer]
+              registry_info[:backoff_timer] = after(registry_info[:backoff_until] - Time.now.to_i) do
+                calculate_ready!(connection_identifier)
+                set_ready_for(connection_lookup(connection_identifier)) unless less_than_ideal?
+              end
             end
+            registry_info[:ready]
+          else
+            registry_info[:ready] = 0
           end
-          registry_info[:ready]
-        else
-          registry_info[:ready] = 0
+        rescue Error::ConnectionFailure
+          warn 'Failed connection encountered!'
+        rescue Error::ConnectionUnavailable
+          warn 'Unavailable connection encountered!'
         end
       end
 
-      # Returns all connections without RDY state
+      # All connections without RDY state
+      #
+      # @return [Array<Krakow::Connection>]
       def waiting_connections
-        registry.find_all do |connection, info|
+        registry.find_all do |conn_id, info|
           info[:ready] < 1 && info[:in_flight] < 1 && info[:backoff_until] < Time.now.to_i
-        end.map(&:first).compact
+        end.map{|conn_id, info| connection_lookup(conn_id) }.compact
       end
 
-      # Returns all connections with RDY state
+      # All connections with RDY state
+      #
+      # @return [Array<Krakow::Connection>]
       def rdy_connections
-        registry.find_all do |connection, info|
+        registry.find_all do |conn_id, info|
           info[:ready] > 0
-        end.map(&:first).compact
+        end.map{|conn_id, info| connection_lookup(conn_id) }.compact
       end
 
       # Force a connection to give up RDY state so next in stack can receive
+      #
+      # @return [nil]
       def force_unready
         debug 'Forcing a connection into an unready state due to less than ideal state'
         connection = rdy_connections.shuffle.first
         if(connection)
           debug "Stripping RDY state from connection: #{connection}"
-          calculate_ready!(connection)
+          calculate_ready!(connection.identifier)
           set_ready_for(connection)
         else
           warn "Failed to locate available connection for RDY aquisition!"
         end
+        nil
       end
 
     end