fleck 0.6.0 → 0.7.0.rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9cb6bd9fd8eca01dd00c764ddc5d1abee5fe560c
-  data.tar.gz: c702b0fdea97d2c297193ad061c56410c093b928
+  metadata.gz: 4b46350b79b78ae75620f8c191ada73f553a83e6
+  data.tar.gz: 5825b21f3219f3db7719b1648acdfb8b21386d41
 SHA512:
-  metadata.gz: 0e0a2d7ab220633fa07e9aa7579ec26a5745f33362911cc31aefda2711c338be682b01ba146edd48f2759e84589b0de00fd446987100ba2860706cfa07a328dd
-  data.tar.gz: b20cdc4ff3037135439e6816df722a59b9864450ce6ecdca335ff2295757d0b697ca28820a71104175a082e9423cb6db2b01d6e7fee24e9602bda8bd3d99aac8
+  metadata.gz: d9e896ba9246a9abcb40ed81510a860bb93950ea2af49186641fdd6a013d4c0836925140e51bed88cf4d272f9803bc311b689e41ed6f93e667b7f2aadc29a220
+  data.tar.gz: 08ea666e0b39cc4a53b15ddab98733d42bebc7afc254d4822dec5f0b8b4fc3bf51d23b3fef37ea56f0ddd096151f7bea27477a6ef92d160d2a4f7489be02fc6e

data/.travis.yml

@@ -1,4 +1,13 @@
+sudo: required
+services:
+  - rabbitmq
+
 language: ruby
 rvm:
-  - 2.2.1
+  - 2.2
+
 before_install: gem install bundler -v 1.10.6
+script:
+  - gem build fleck.gemspec
+  - gem install fleck-* --local
+  - bundle exec rspec

data/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## develop ##
 
+## v0.7.0 (21 June 2016)
+  - **NEW** Added multihost support to Fleck configuration, that allows to manage network failure situations and to choose the best options from the list of available hosts.
+            This feature uses `Fleck::HostRating` to collect TCP latency data about each provided host, so that when a new connection is required, the host with lowest latency
+            will be choosed. If a host becomes unreachable, it gets the lowest rating and will be used as the last option, allowing you to automatically manage network failures.
+  - **NEW** Implemented a basic log filter for headers and params in requests and responses.
+  - **NEW** Log each processed request in `Fleck::Consumer::Response`.
+  - **NEW** Implemented `:deprecated?` method for `Fleck::Consumer::Response`.
+  - **NEW** Store client IP address to requests headers, in order to be able to trace requests origin when multiple clients making requests to the same consumer type.
+
 ## v0.6.0 (16 June 2016)
   - **NEW** __(BREAKING CHANGE)__ Use `"fleck"` exchange for RPC simulation, so that reply queues could be used in a RabbitMQ Federation configuration.
             Be careful when upgrading `Fleck::Consumer` from version `v0.5.x` or below, because now `Fleck::Consumer` will send responses to a `:direct` exchange

data/examples/example.rb

@@ -15,8 +15,7 @@ Fleck.configure do |config|
   config.loglevel     = Logger::DEBUG
 end
 
-connection = Fleck.connection(host: "127.0.0.1", port: 5672, user: user, pass: pass, vhost: "/")
-client = Fleck::Client.new(connection, "example.queue", concurrency: CONCURRENCY.to_i)
+client = Fleck::Client.new(Fleck.connection, "example.queue", concurrency: CONCURRENCY.to_i)
 
 count   = 0
 success = 0
@@ -45,7 +44,7 @@ end
 
 Thread.new do
   SAMPLES.times do |i|
-    client.request(action: 'incr', params: {num: i}, async: true, timeout: 1, rmq_options: { priority: (rand * 9).round(0), mandatory: false}) do |request, response|
+    client.request(action: 'incr', params: {num: i, secret: 'supersecret'}, async: true, timeout: 1, rmq_options: { priority: (rand * 9).round(0), mandatory: false}) do |request, response|
       if response.status == 200
         request.logger.debug response.body
       else

data/lib/fleck/client/request.rb

@@ -27,8 +27,10 @@ module Fleck
       @multiple_responses = multiple_responses
       @ztimer_slot        = nil
       @expired            = false
+      @params             = params.filtered!
 
       headers[:version] = @version
+      headers[:ip]      = @client.local_ip
 
       @options = {
         routing_key:      @routing_key,
@@ -40,12 +42,12 @@ module Fleck
         persistent:       rmq_options[:persistent] || false,
         content_type:     'application/json',
         content_encoding: 'UTF-8'
-      }
+      }.filtered!
       @options[:priority]   = rmq_options[:priority] unless rmq_options[:priority].nil?
       @options[:app_id]     = rmq_options[:app_id] || Fleck.config.app_name
       @options[:expiration] = @timeout
 
-      @message = Oj.dump({headers: headers, params: params}, mode: :compat)
+      @message = Oj.dump({headers: headers, params: @params}, mode: :compat)
 
       logger.debug "Request prepared"
     end
@@ -63,7 +65,7 @@ module Fleck
     def send!(async = false)
       @started_at = Time.now.to_f
       @async = async
-      logger.debug("Sending request with (options: #{@options}, message: #{@message})")
+      logger.debug("Sending request with (options: #{@options}, message: #{@params})")
 
       if @timeout
         @ztimer_slot = Ztimer.after(@timeout){ expire! }

data/lib/fleck/client/response.rb

@@ -5,7 +5,7 @@ module Fleck
 
     attr_accessor :status, :headers, :body, :errors, :deprecated
     def initialize(payload)
-      @data       = Oj.load(payload, mode: :compat).to_hash_with_indifferent_access
+      @data       = Oj.load(payload, mode: :compat).to_hash_with_indifferent_access.filtered!
       @status     = @data["status"]
       @headers    = @data["headers"] || {}
       @body       = @data["body"]
@@ -16,5 +16,32 @@ module Fleck
     def deprecated?
       @deprecated
     end
+
+    def to_json(filter: false)
+      data = {
+        "status"     => @status,
+        "errors"     => @errors,
+        "headers"    => @headers,
+        "body"       => @body,
+        "deprecated" => @deprecated
+      }
+      data.filter! if filter
+
+      return Oj.dump(data, mode: :compat)
+    rescue => e
+      logger.error e.inspect + "\n" + e.backtrace.join("\n")
+      return Oj.dump({
+        "status" => 500,
+        "errors" => ['Internal Server Error', 'Failed to dump the response to JSON']
+      }, mode: :compat)
+    end
+
+    def to_s
+      return "#<#{self.class} #{self.to_json(filter: true)}>"
+    end
+
+    def inspect
+      self.to_s
+    end
   end
 end

data/lib/fleck/client.rb

@@ -3,6 +3,8 @@ module Fleck
   class Client
     include Fleck::Loggable
 
+    attr_reader :local_ip, :remote_ip
+
     def initialize(connection, queue_name = "", exchange_type: :direct, exchange_name: "", multiple_responses: false, concurrency: 1)
       @connection         = connection
       @queue_name         = queue_name
@@ -13,6 +15,8 @@ module Fleck
       @subscriptions      = ThreadSafe::Array.new
       @terminated         = false
       @mutex              = Mutex.new
+      @local_ip           = @connection.transport.socket.local_address.ip_address
+      @remote_ip          = @connection.transport.socket.remote_address.ip_address
 
       @channel     = @connection.create_channel
       @exchange    = Bunny::Exchange.new(@channel, :direct, 'fleck')

data/lib/fleck/configuration.rb

@@ -2,8 +2,9 @@
 module Fleck
   class Configuration
 
-    attr_reader :logfile, :loglevel, :progname
-    attr_accessor :default_user, :default_pass, :default_host, :default_port, :default_vhost, :default_queue, :app_name
+    attr_reader :logfile, :loglevel, :progname, :hosts
+    attr_accessor :default_user, :default_pass, :default_host, :default_port, :default_vhost, :default_queue,
+                  :app_name, :filters
 
     def initialize
       @logfile       = STDOUT
@@ -16,14 +17,47 @@ module Fleck
       @default_pass  = nil
       @default_vhost = "/"
       @default_queue = "default"
+      @filters       = ["password", "secret", "token"]
+      @hosts         = []
+      @credentials   = {}
+    end
+
+    def hosts=(*args)
+      args.flatten.each do |host|
+        add_host host
+      end
+      return @hosts
+    end
+
+    def add_host(data)
+      if data.is_a?(String)
+        host, port = data.split(":")
+        port = port ? port.to_i : 5672
+        @hosts << Fleck::HostRating.new(host: host, port: port)
+        @credentials["#{host}:#{port}"] ||= { user: @default_user, pass: @default_pass }
+      elsif data.is_a?(Hash)
+        data = data.to_hash_with_indifferent_access
+        host = data[:host] || @default_host
+        port = data[:port] || @default_port
+        @hosts << Fleck::HostRating.new(host: data[:host] || @default_host, port: data[:port] || @default_port)
+        @credentials["#{host}:#{port}"] ||= { user: data[:user] || @default_user, pass: data[:pass] || @default_pass }
+      else
+        raise ArgumentError.new("Invalid host type #{data.inspect}: String or Hash expected")
+      end
     end
 
     def default_options
+      best = @hosts.sort.first
       opts = {}
-      opts[:host]  = @default_host
-      opts[:port]  = @default_port
-      opts[:user]  = @default_user
-      opts[:pass]  = @default_pass
+
+      host = best ? best.host : @default_host
+      port = best ? best.port : @default_port
+      credentials = @credentials["#{host}:#{port}"] || {user: @default_user, pass: @default_pass}
+
+      opts[:host]  = host
+      opts[:port]  = port
+      opts[:user]  = credentials[:user] || @default_user
+      opts[:pass]  = credentials[:pass] || @default_pass
       opts[:vhost] = @default_vhost
       opts[:queue] = @default_queue
 

data/lib/fleck/consumer/request.rb

@@ -3,7 +3,7 @@ module Fleck
   class Consumer::Request
     include Fleck::Loggable
 
-    attr_reader :id, :metadata, :payload, :action, :data, :headers, :action, :version, :params, :status, :errors
+    attr_reader :id, :metadata, :payload, :action, :data, :headers, :action, :version, :ip, :params, :status, :errors
 
     def initialize(metadata, payload, delivery_info)
       @id              = metadata.correlation_id
@@ -17,6 +17,7 @@ module Fleck
       @headers       = (@metadata.headers || {}).to_hash_with_indifferent_access
       @action        = @metadata.type
       @version       = nil
+      @ip            = nil
       @params        = {}
       @status        = 200
       @errors        = []
@@ -27,14 +28,15 @@ module Fleck
     protected
 
     def parse_request!
-      logger.debug "Parsing request (exchange: #{@exchange}, queue: #{@queue}, options: #{@metadata}, message: #{@payload})"
+      @data = Oj.load(@payload, mode: :compat).to_hash_with_indifferent_access.filtered!
+      @headers.merge!(@data["headers"] || {}).filtered!
 
-      @data = Oj.load(@payload, mode: :compat).to_hash_with_indifferent_access
-      @headers.merge!(@data["headers"] || {})
+      logger.debug "Processing request (exchange: #{@exchange}, queue: #{@queue}, options: #{@headers}, message: #{@data})"
 
       @action            ||= @headers["action"]
       @headers["action"] ||= @action
       @version             = @headers["version"]
+      @ip                  = @headers["ip"]
       @params              = @data["params"] || {}
     rescue Oj::ParseError => e
       logger.error(e.inspect + "\n" + e.backtrace.join("\n"))

data/lib/fleck/consumer/response.rb

@@ -35,6 +35,10 @@ module Fleck
       @deprecated = true
     end
 
+    def deprecated?
+      @deprecated
+    end
+
     def not_found(msg = nil)
       @status = 404
       @errors << 'Resource Not Found'
@@ -50,14 +54,17 @@ module Fleck
       end
     end
 
-    def to_json
-      return Oj.dump({
+    def to_json(filter: false)
+      data = {
         "status"     => @status,
         "errors"     => @errors,
         "headers"    => @headers,
         "body"       => @body,
         "deprecated" => @deprecated
-      }, mode: :compat)
+      }
+      data.filter! if filter
+
+      return Oj.dump(data, mode: :compat)
     rescue => e
       logger.error e.inspect + "\n" + e.backtrace.join("\n")
       return Oj.dump({
@@ -67,7 +74,7 @@ module Fleck
     end
 
     def to_s
-      return "#<#{self.class} #{self.to_json}>"
+      return "#<#{self.class} #{self.to_json(filter: true)}>"
     end
   end
 end

data/lib/fleck/consumer.rb

@@ -226,6 +226,7 @@ module Fleck
       options[:consumer_tag] = @__consumer_tag if @__consumer_tag
 
       @__subscription = @__queue.subscribe(options) do |delivery_info, metadata, payload|
+        started_at = Time.now.to_f
         @__response = Fleck::Consumer::Response.new(metadata.correlation_id)
         begin
           @__request  = Fleck::Consumer::Request.new(metadata, payload, delivery_info)
@@ -242,8 +243,6 @@ module Fleck
         end
 
         if @__response.rejected?
-          # the request was rejected, so we have to notify the reject
-          logger.warn "Request #{@__response.id} was rejected!"
           @__channel.reject(delivery_info.delivery_tag, @__response.requeue?)
         else
           logger.debug "Sending response: #{@__response}"
@@ -254,6 +253,26 @@ module Fleck
             @__channel.ack(delivery_info.delivery_tag)
           end
         end
+
+        exec_time  = ((Time.now.to_f - started_at) * 1000).round(2)
+        ex_type    = @__exchange_type.to_s[0].upcase
+        ex_name    = @__exchange_name.to_s == "" ? "".inspect : @__exchange_name
+        status     = @__response.status
+        status     = 406 if @__response.rejected?
+        status     = 503 if @__channel.closed?
+
+        message    = "#{@__request.ip} #{metadata[:app_id]} => "
+        message   += "(#{@__exchange_name.to_s.inspect}|#{ex_type}|#{@__queue_name}) "
+        message   += "##{@__request.id} \"#{@__request.action} /#{@__request.version || 'v1'}\" #{status} "
+        message   += "(#{exec_time}ms) #{'DEPRECATED!' if @__response.deprecated?}"
+
+        if status >= 500
+          logger.error message
+        elsif status >= 400 || @__response.deprecated?
+          logger.warn message
+        else
+          logger.info message
+        end
       end
     end
 

data/lib/fleck/hash_with_indifferent_access.rb

@@ -31,6 +31,14 @@ class HashWithIndifferentAccess < Hash
     end
   end
 
+  def inspect
+    super
+  end
+
+  def to_s
+    super
+  end
+
   protected
 
   def copy_from(original)
@@ -45,4 +53,33 @@ class Hash
   def to_hash_with_indifferent_access
     return HashWithIndifferentAccess.new(self)
   end
+
+  def to_s
+    if @filtered
+      return self.dup.filter!.inspect
+    else
+      super
+    end
+  end
+
+  def filtered!
+    @filtered = true
+    self.keys.each do |key|
+      self[key].filtered! if self[key].is_a?(Hash)
+    end
+    return self
+  end
+
+  def filter!
+    filters = Fleck.config.filters
+    self.keys.each do |key|
+      if filters.include?(key.to_s)
+        self[key] = "[FILTERED]"
+      elsif self[key].is_a?(Hash)
+        self[key] = self[key].dup.filter!
+      end
+    end
+
+    return self
+  end
 end

data/lib/fleck/host_rating.rb

@@ -0,0 +1,74 @@
+require 'socket'
+
+module Fleck
+  class HostRating
+    include Fleck::Loggable
+
+    CONN_TIMEOUT = 5
+
+    attr_reader :host, :port, :avg, :history
+
+    def initialize(host: 'localhost', port: 5672, refresh_rate: 30000, period: 300000)
+      @host         = host
+      @port         = port
+      @refresh_rate = refresh_rate
+      @period       = period
+
+      # metrics
+      @reachable  = false
+      @avg        = 0
+      @updated_at = nil
+      @history    = []
+
+      refresh!
+      @timer = Ztimer.every(@refresh_rate){ refresh! }
+    end
+
+    def reachable?
+      @reachable
+    end
+
+    def close
+      @timer.cancel!
+    end
+
+    def <=>(other_host)
+      return 1 if !self.reachable? && other_host.reachable? # the other host is reachable, so it comes first
+      return 0  if !(self.reachable? || other_host.reachable?) # both host are unreachable, so they have the same priority
+      return -1  if self.reachable? && !other_host.reachable? # the current host comes first, because it's reachable, while the other host is unreachable
+
+      # when both hosts are reachable, use avg latency to order them
+      return self.avg <=> other_host.avg
+    end
+
+    private
+
+    def refresh!
+      # Get host info and open a new socket
+      addr = Socket.getaddrinfo(@host, nil)
+      sock_addr = Socket.pack_sockaddr_in(@port, addr[0][3])
+      socket = Socket.new(:AF_INET, :SOCK_STREAM, 0)
+      socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+
+      started_at = Time.now.to_f
+      begin
+        socket.connect_nonblock(sock_addr)
+      rescue IO::WaitWritable
+        IO.select(nil, [socket], nil, CONN_TIMEOUT) or raise Timeout::Error
+      end
+      latency = (Time.now.to_f - started_at) * 1000 # ms
+      socket.close
+
+      @history << latency
+      @history.shift if @history.size > @period / @refresh_rate
+      @avg = @history.inject(:+).to_f / @history.size
+      @reachable = true
+    rescue SocketError, Timeout::Error => e
+      socket.close if socket
+      @reachable = false
+      logger.error "Connection error: #{@host}:#{@port} (#{e.inspect})"
+    ensure
+      @updated_at = Time.now
+    end
+  end
+end

data/lib/fleck/version.rb

@@ -1,3 +1,3 @@
 module Fleck
-  VERSION = "0.6.0"
+  VERSION = "0.7.0.rc"
 end

data/lib/fleck.rb

@@ -8,8 +8,9 @@ require "oj"
 require "ztimer"
 require "fleck/version"
 require "fleck/hash_with_indifferent_access"
-require "fleck/configuration"
 require "fleck/loggable"
+require "fleck/host_rating"
+require "fleck/configuration"
 require "fleck/consumer"
 require "fleck/client"
 
@@ -33,7 +34,7 @@ module Fleck
     end
   end
 
-  def self.connection(options)
+  def self.connection(options = {})
     opts = Fleck.config.default_options.merge(options)
     key  = "ampq://#{opts[:user]}@#{opts[:host]}:#{opts[:port]}#{opts[:vhost]}"
     conn = @connections[key]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fleck
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0.rc
 platform: ruby
 authors:
 - Groza Sergiu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-20 00:00:00.000000000 Z
+date: 2016-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -160,6 +160,7 @@ files:
 - lib/fleck/consumer/request.rb
 - lib/fleck/consumer/response.rb
 - lib/fleck/hash_with_indifferent_access.rb
+- lib/fleck/host_rating.rb
 - lib/fleck/loggable.rb
 - lib/fleck/version.rb
 homepage: https://github.com/serioja90/fleck
@@ -177,9 +178,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.8