fleck 0.6.0 → 0.7.0.rc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9cb6bd9fd8eca01dd00c764ddc5d1abee5fe560c
4
- data.tar.gz: c702b0fdea97d2c297193ad061c56410c093b928
3
+ metadata.gz: 4b46350b79b78ae75620f8c191ada73f553a83e6
4
+ data.tar.gz: 5825b21f3219f3db7719b1648acdfb8b21386d41
5
5
  SHA512:
6
- metadata.gz: 0e0a2d7ab220633fa07e9aa7579ec26a5745f33362911cc31aefda2711c338be682b01ba146edd48f2759e84589b0de00fd446987100ba2860706cfa07a328dd
7
- data.tar.gz: b20cdc4ff3037135439e6816df722a59b9864450ce6ecdca335ff2295757d0b697ca28820a71104175a082e9423cb6db2b01d6e7fee24e9602bda8bd3d99aac8
6
+ metadata.gz: d9e896ba9246a9abcb40ed81510a860bb93950ea2af49186641fdd6a013d4c0836925140e51bed88cf4d272f9803bc311b689e41ed6f93e667b7f2aadc29a220
7
+ data.tar.gz: 08ea666e0b39cc4a53b15ddab98733d42bebc7afc254d4822dec5f0b8b4fc3bf51d23b3fef37ea56f0ddd096151f7bea27477a6ef92d160d2a4f7489be02fc6e
data/.travis.yml CHANGED
@@ -1,4 +1,13 @@
1
+ sudo: required
2
+ services:
3
+ - rabbitmq
4
+
1
5
  language: ruby
2
6
  rvm:
3
- - 2.2.1
7
+ - 2.2
8
+
4
9
  before_install: gem install bundler -v 1.10.6
10
+ script:
11
+ - gem build fleck.gemspec
12
+ - gem install fleck-* --local
13
+ - bundle exec rspec
data/CHANGELOG.md CHANGED
@@ -2,6 +2,15 @@
2
2
 
3
3
  ## develop ##
4
4
 
5
+ ## v0.7.0 (21 June 2016)
6
+ - **NEW** Added multihost support to Fleck configuration, that allows to manage network failure situations and to choose the best options from the list of available hosts.
7
+ This feature uses `Fleck::HostRating` to collect TCP latency data about each provided host, so that when a new connection is required, the host with lowest latency
8
+ will be choosed. If a host becomes unreachable, it gets the lowest rating and will be used as the last option, allowing you to automatically manage network failures.
9
+ - **NEW** Implemented a basic log filter for headers and params in requests and responses.
10
+ - **NEW** Log each processed request in `Fleck::Consumer::Response`.
11
+ - **NEW** Implemented `:deprecated?` method for `Fleck::Consumer::Response`.
12
+ - **NEW** Store client IP address to requests headers, in order to be able to trace requests origin when multiple clients making requests to the same consumer type.
13
+
5
14
  ## v0.6.0 (16 June 2016)
6
15
  - **NEW** __(BREAKING CHANGE)__ Use `"fleck"` exchange for RPC simulation, so that reply queues could be used in a RabbitMQ Federation configuration.
7
16
  Be careful when upgrading `Fleck::Consumer` from version `v0.5.x` or below, because now `Fleck::Consumer` will send responses to a `:direct` exchange
data/examples/example.rb CHANGED
@@ -15,8 +15,7 @@ Fleck.configure do |config|
15
15
  config.loglevel = Logger::DEBUG
16
16
  end
17
17
 
18
- connection = Fleck.connection(host: "127.0.0.1", port: 5672, user: user, pass: pass, vhost: "/")
19
- client = Fleck::Client.new(connection, "example.queue", concurrency: CONCURRENCY.to_i)
18
+ client = Fleck::Client.new(Fleck.connection, "example.queue", concurrency: CONCURRENCY.to_i)
20
19
 
21
20
  count = 0
22
21
  success = 0
@@ -45,7 +44,7 @@ end
45
44
 
46
45
  Thread.new do
47
46
  SAMPLES.times do |i|
48
- client.request(action: 'incr', params: {num: i}, async: true, timeout: 1, rmq_options: { priority: (rand * 9).round(0), mandatory: false}) do |request, response|
47
+ client.request(action: 'incr', params: {num: i, secret: 'supersecret'}, async: true, timeout: 1, rmq_options: { priority: (rand * 9).round(0), mandatory: false}) do |request, response|
49
48
  if response.status == 200
50
49
  request.logger.debug response.body
51
50
  else
@@ -27,8 +27,10 @@ module Fleck
27
27
  @multiple_responses = multiple_responses
28
28
  @ztimer_slot = nil
29
29
  @expired = false
30
+ @params = params.filtered!
30
31
 
31
32
  headers[:version] = @version
33
+ headers[:ip] = @client.local_ip
32
34
 
33
35
  @options = {
34
36
  routing_key: @routing_key,
@@ -40,12 +42,12 @@ module Fleck
40
42
  persistent: rmq_options[:persistent] || false,
41
43
  content_type: 'application/json',
42
44
  content_encoding: 'UTF-8'
43
- }
45
+ }.filtered!
44
46
  @options[:priority] = rmq_options[:priority] unless rmq_options[:priority].nil?
45
47
  @options[:app_id] = rmq_options[:app_id] || Fleck.config.app_name
46
48
  @options[:expiration] = @timeout
47
49
 
48
- @message = Oj.dump({headers: headers, params: params}, mode: :compat)
50
+ @message = Oj.dump({headers: headers, params: @params}, mode: :compat)
49
51
 
50
52
  logger.debug "Request prepared"
51
53
  end
@@ -63,7 +65,7 @@ module Fleck
63
65
  def send!(async = false)
64
66
  @started_at = Time.now.to_f
65
67
  @async = async
66
- logger.debug("Sending request with (options: #{@options}, message: #{@message})")
68
+ logger.debug("Sending request with (options: #{@options}, message: #{@params})")
67
69
 
68
70
  if @timeout
69
71
  @ztimer_slot = Ztimer.after(@timeout){ expire! }
@@ -5,7 +5,7 @@ module Fleck
5
5
 
6
6
  attr_accessor :status, :headers, :body, :errors, :deprecated
7
7
  def initialize(payload)
8
- @data = Oj.load(payload, mode: :compat).to_hash_with_indifferent_access
8
+ @data = Oj.load(payload, mode: :compat).to_hash_with_indifferent_access.filtered!
9
9
  @status = @data["status"]
10
10
  @headers = @data["headers"] || {}
11
11
  @body = @data["body"]
@@ -16,5 +16,32 @@ module Fleck
16
16
  def deprecated?
17
17
  @deprecated
18
18
  end
19
+
20
+ def to_json(filter: false)
21
+ data = {
22
+ "status" => @status,
23
+ "errors" => @errors,
24
+ "headers" => @headers,
25
+ "body" => @body,
26
+ "deprecated" => @deprecated
27
+ }
28
+ data.filter! if filter
29
+
30
+ return Oj.dump(data, mode: :compat)
31
+ rescue => e
32
+ logger.error e.inspect + "\n" + e.backtrace.join("\n")
33
+ return Oj.dump({
34
+ "status" => 500,
35
+ "errors" => ['Internal Server Error', 'Failed to dump the response to JSON']
36
+ }, mode: :compat)
37
+ end
38
+
39
+ def to_s
40
+ return "#<#{self.class} #{self.to_json(filter: true)}>"
41
+ end
42
+
43
+ def inspect
44
+ self.to_s
45
+ end
19
46
  end
20
47
  end
data/lib/fleck/client.rb CHANGED
@@ -3,6 +3,8 @@ module Fleck
3
3
  class Client
4
4
  include Fleck::Loggable
5
5
 
6
+ attr_reader :local_ip, :remote_ip
7
+
6
8
  def initialize(connection, queue_name = "", exchange_type: :direct, exchange_name: "", multiple_responses: false, concurrency: 1)
7
9
  @connection = connection
8
10
  @queue_name = queue_name
@@ -13,6 +15,8 @@ module Fleck
13
15
  @subscriptions = ThreadSafe::Array.new
14
16
  @terminated = false
15
17
  @mutex = Mutex.new
18
+ @local_ip = @connection.transport.socket.local_address.ip_address
19
+ @remote_ip = @connection.transport.socket.remote_address.ip_address
16
20
 
17
21
  @channel = @connection.create_channel
18
22
  @exchange = Bunny::Exchange.new(@channel, :direct, 'fleck')
@@ -2,8 +2,9 @@
2
2
  module Fleck
3
3
  class Configuration
4
4
 
5
- attr_reader :logfile, :loglevel, :progname
6
- attr_accessor :default_user, :default_pass, :default_host, :default_port, :default_vhost, :default_queue, :app_name
5
+ attr_reader :logfile, :loglevel, :progname, :hosts
6
+ attr_accessor :default_user, :default_pass, :default_host, :default_port, :default_vhost, :default_queue,
7
+ :app_name, :filters
7
8
 
8
9
  def initialize
9
10
  @logfile = STDOUT
@@ -16,14 +17,47 @@ module Fleck
16
17
  @default_pass = nil
17
18
  @default_vhost = "/"
18
19
  @default_queue = "default"
20
+ @filters = ["password", "secret", "token"]
21
+ @hosts = []
22
+ @credentials = {}
23
+ end
24
+
25
+ def hosts=(*args)
26
+ args.flatten.each do |host|
27
+ add_host host
28
+ end
29
+ return @hosts
30
+ end
31
+
32
+ def add_host(data)
33
+ if data.is_a?(String)
34
+ host, port = data.split(":")
35
+ port = port ? port.to_i : 5672
36
+ @hosts << Fleck::HostRating.new(host: host, port: port)
37
+ @credentials["#{host}:#{port}"] ||= { user: @default_user, pass: @default_pass }
38
+ elsif data.is_a?(Hash)
39
+ data = data.to_hash_with_indifferent_access
40
+ host = data[:host] || @default_host
41
+ port = data[:port] || @default_port
42
+ @hosts << Fleck::HostRating.new(host: data[:host] || @default_host, port: data[:port] || @default_port)
43
+ @credentials["#{host}:#{port}"] ||= { user: data[:user] || @default_user, pass: data[:pass] || @default_pass }
44
+ else
45
+ raise ArgumentError.new("Invalid host type #{data.inspect}: String or Hash expected")
46
+ end
19
47
  end
20
48
 
21
49
  def default_options
50
+ best = @hosts.sort.first
22
51
  opts = {}
23
- opts[:host] = @default_host
24
- opts[:port] = @default_port
25
- opts[:user] = @default_user
26
- opts[:pass] = @default_pass
52
+
53
+ host = best ? best.host : @default_host
54
+ port = best ? best.port : @default_port
55
+ credentials = @credentials["#{host}:#{port}"] || {user: @default_user, pass: @default_pass}
56
+
57
+ opts[:host] = host
58
+ opts[:port] = port
59
+ opts[:user] = credentials[:user] || @default_user
60
+ opts[:pass] = credentials[:pass] || @default_pass
27
61
  opts[:vhost] = @default_vhost
28
62
  opts[:queue] = @default_queue
29
63
 
@@ -3,7 +3,7 @@ module Fleck
3
3
  class Consumer::Request
4
4
  include Fleck::Loggable
5
5
 
6
- attr_reader :id, :metadata, :payload, :action, :data, :headers, :action, :version, :params, :status, :errors
6
+ attr_reader :id, :metadata, :payload, :action, :data, :headers, :action, :version, :ip, :params, :status, :errors
7
7
 
8
8
  def initialize(metadata, payload, delivery_info)
9
9
  @id = metadata.correlation_id
@@ -17,6 +17,7 @@ module Fleck
17
17
  @headers = (@metadata.headers || {}).to_hash_with_indifferent_access
18
18
  @action = @metadata.type
19
19
  @version = nil
20
+ @ip = nil
20
21
  @params = {}
21
22
  @status = 200
22
23
  @errors = []
@@ -27,14 +28,15 @@ module Fleck
27
28
  protected
28
29
 
29
30
  def parse_request!
30
- logger.debug "Parsing request (exchange: #{@exchange}, queue: #{@queue}, options: #{@metadata}, message: #{@payload})"
31
+ @data = Oj.load(@payload, mode: :compat).to_hash_with_indifferent_access.filtered!
32
+ @headers.merge!(@data["headers"] || {}).filtered!
31
33
 
32
- @data = Oj.load(@payload, mode: :compat).to_hash_with_indifferent_access
33
- @headers.merge!(@data["headers"] || {})
34
+ logger.debug "Processing request (exchange: #{@exchange}, queue: #{@queue}, options: #{@headers}, message: #{@data})"
34
35
 
35
36
  @action ||= @headers["action"]
36
37
  @headers["action"] ||= @action
37
38
  @version = @headers["version"]
39
+ @ip = @headers["ip"]
38
40
  @params = @data["params"] || {}
39
41
  rescue Oj::ParseError => e
40
42
  logger.error(e.inspect + "\n" + e.backtrace.join("\n"))
@@ -35,6 +35,10 @@ module Fleck
35
35
  @deprecated = true
36
36
  end
37
37
 
38
+ def deprecated?
39
+ @deprecated
40
+ end
41
+
38
42
  def not_found(msg = nil)
39
43
  @status = 404
40
44
  @errors << 'Resource Not Found'
@@ -50,14 +54,17 @@ module Fleck
50
54
  end
51
55
  end
52
56
 
53
- def to_json
54
- return Oj.dump({
57
+ def to_json(filter: false)
58
+ data = {
55
59
  "status" => @status,
56
60
  "errors" => @errors,
57
61
  "headers" => @headers,
58
62
  "body" => @body,
59
63
  "deprecated" => @deprecated
60
- }, mode: :compat)
64
+ }
65
+ data.filter! if filter
66
+
67
+ return Oj.dump(data, mode: :compat)
61
68
  rescue => e
62
69
  logger.error e.inspect + "\n" + e.backtrace.join("\n")
63
70
  return Oj.dump({
@@ -67,7 +74,7 @@ module Fleck
67
74
  end
68
75
 
69
76
  def to_s
70
- return "#<#{self.class} #{self.to_json}>"
77
+ return "#<#{self.class} #{self.to_json(filter: true)}>"
71
78
  end
72
79
  end
73
80
  end
@@ -226,6 +226,7 @@ module Fleck
226
226
  options[:consumer_tag] = @__consumer_tag if @__consumer_tag
227
227
 
228
228
  @__subscription = @__queue.subscribe(options) do |delivery_info, metadata, payload|
229
+ started_at = Time.now.to_f
229
230
  @__response = Fleck::Consumer::Response.new(metadata.correlation_id)
230
231
  begin
231
232
  @__request = Fleck::Consumer::Request.new(metadata, payload, delivery_info)
@@ -242,8 +243,6 @@ module Fleck
242
243
  end
243
244
 
244
245
  if @__response.rejected?
245
- # the request was rejected, so we have to notify the reject
246
- logger.warn "Request #{@__response.id} was rejected!"
247
246
  @__channel.reject(delivery_info.delivery_tag, @__response.requeue?)
248
247
  else
249
248
  logger.debug "Sending response: #{@__response}"
@@ -254,6 +253,26 @@ module Fleck
254
253
  @__channel.ack(delivery_info.delivery_tag)
255
254
  end
256
255
  end
256
+
257
+ exec_time = ((Time.now.to_f - started_at) * 1000).round(2)
258
+ ex_type = @__exchange_type.to_s[0].upcase
259
+ ex_name = @__exchange_name.to_s == "" ? "".inspect : @__exchange_name
260
+ status = @__response.status
261
+ status = 406 if @__response.rejected?
262
+ status = 503 if @__channel.closed?
263
+
264
+ message = "#{@__request.ip} #{metadata[:app_id]} => "
265
+ message += "(#{@__exchange_name.to_s.inspect}|#{ex_type}|#{@__queue_name}) "
266
+ message += "##{@__request.id} \"#{@__request.action} /#{@__request.version || 'v1'}\" #{status} "
267
+ message += "(#{exec_time}ms) #{'DEPRECATED!' if @__response.deprecated?}"
268
+
269
+ if status >= 500
270
+ logger.error message
271
+ elsif status >= 400 || @__response.deprecated?
272
+ logger.warn message
273
+ else
274
+ logger.info message
275
+ end
257
276
  end
258
277
  end
259
278
 
@@ -31,6 +31,14 @@ class HashWithIndifferentAccess < Hash
31
31
  end
32
32
  end
33
33
 
34
+ def inspect
35
+ super
36
+ end
37
+
38
+ def to_s
39
+ super
40
+ end
41
+
34
42
  protected
35
43
 
36
44
  def copy_from(original)
@@ -45,4 +53,33 @@ class Hash
45
53
  def to_hash_with_indifferent_access
46
54
  return HashWithIndifferentAccess.new(self)
47
55
  end
56
+
57
+ def to_s
58
+ if @filtered
59
+ return self.dup.filter!.inspect
60
+ else
61
+ super
62
+ end
63
+ end
64
+
65
+ def filtered!
66
+ @filtered = true
67
+ self.keys.each do |key|
68
+ self[key].filtered! if self[key].is_a?(Hash)
69
+ end
70
+ return self
71
+ end
72
+
73
+ def filter!
74
+ filters = Fleck.config.filters
75
+ self.keys.each do |key|
76
+ if filters.include?(key.to_s)
77
+ self[key] = "[FILTERED]"
78
+ elsif self[key].is_a?(Hash)
79
+ self[key] = self[key].dup.filter!
80
+ end
81
+ end
82
+
83
+ return self
84
+ end
48
85
  end
@@ -0,0 +1,74 @@
1
+ require 'socket'
2
+
3
+ module Fleck
4
+ class HostRating
5
+ include Fleck::Loggable
6
+
7
+ CONN_TIMEOUT = 5
8
+
9
+ attr_reader :host, :port, :avg, :history
10
+
11
+ def initialize(host: 'localhost', port: 5672, refresh_rate: 30000, period: 300000)
12
+ @host = host
13
+ @port = port
14
+ @refresh_rate = refresh_rate
15
+ @period = period
16
+
17
+ # metrics
18
+ @reachable = false
19
+ @avg = 0
20
+ @updated_at = nil
21
+ @history = []
22
+
23
+ refresh!
24
+ @timer = Ztimer.every(@refresh_rate){ refresh! }
25
+ end
26
+
27
+ def reachable?
28
+ @reachable
29
+ end
30
+
31
+ def close
32
+ @timer.cancel!
33
+ end
34
+
35
+ def <=>(other_host)
36
+ return 1 if !self.reachable? && other_host.reachable? # the other host is reachable, so it comes first
37
+ return 0 if !(self.reachable? || other_host.reachable?) # both host are unreachable, so they have the same priority
38
+ return -1 if self.reachable? && !other_host.reachable? # the current host comes first, because it's reachable, while the other host is unreachable
39
+
40
+ # when both hosts are reachable, use avg latency to order them
41
+ return self.avg <=> other_host.avg
42
+ end
43
+
44
+ private
45
+
46
+ def refresh!
47
+ # Get host info and open a new socket
48
+ addr = Socket.getaddrinfo(@host, nil)
49
+ sock_addr = Socket.pack_sockaddr_in(@port, addr[0][3])
50
+ socket = Socket.new(:AF_INET, :SOCK_STREAM, 0)
51
+ socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
52
+
53
+ started_at = Time.now.to_f
54
+ begin
55
+ socket.connect_nonblock(sock_addr)
56
+ rescue IO::WaitWritable
57
+ IO.select(nil, [socket], nil, CONN_TIMEOUT) or raise Timeout::Error
58
+ end
59
+ latency = (Time.now.to_f - started_at) * 1000 # ms
60
+ socket.close
61
+
62
+ @history << latency
63
+ @history.shift if @history.size > @period / @refresh_rate
64
+ @avg = @history.inject(:+).to_f / @history.size
65
+ @reachable = true
66
+ rescue SocketError, Timeout::Error => e
67
+ socket.close if socket
68
+ @reachable = false
69
+ logger.error "Connection error: #{@host}:#{@port} (#{e.inspect})"
70
+ ensure
71
+ @updated_at = Time.now
72
+ end
73
+ end
74
+ end
data/lib/fleck/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Fleck
2
- VERSION = "0.6.0"
2
+ VERSION = "0.7.0.rc"
3
3
  end
data/lib/fleck.rb CHANGED
@@ -8,8 +8,9 @@ require "oj"
8
8
  require "ztimer"
9
9
  require "fleck/version"
10
10
  require "fleck/hash_with_indifferent_access"
11
- require "fleck/configuration"
12
11
  require "fleck/loggable"
12
+ require "fleck/host_rating"
13
+ require "fleck/configuration"
13
14
  require "fleck/consumer"
14
15
  require "fleck/client"
15
16
 
@@ -33,7 +34,7 @@ module Fleck
33
34
  end
34
35
  end
35
36
 
36
- def self.connection(options)
37
+ def self.connection(options = {})
37
38
  opts = Fleck.config.default_options.merge(options)
38
39
  key = "ampq://#{opts[:user]}@#{opts[:host]}:#{opts[:port]}#{opts[:vhost]}"
39
40
  conn = @connections[key]
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fleck
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.7.0.rc
5
5
  platform: ruby
6
6
  authors:
7
7
  - Groza Sergiu
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-06-20 00:00:00.000000000 Z
11
+ date: 2016-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -160,6 +160,7 @@ files:
160
160
  - lib/fleck/consumer/request.rb
161
161
  - lib/fleck/consumer/response.rb
162
162
  - lib/fleck/hash_with_indifferent_access.rb
163
+ - lib/fleck/host_rating.rb
163
164
  - lib/fleck/loggable.rb
164
165
  - lib/fleck/version.rb
165
166
  homepage: https://github.com/serioja90/fleck
@@ -177,9 +178,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
177
178
  version: '2.0'
178
179
  required_rubygems_version: !ruby/object:Gem::Requirement
179
180
  requirements:
180
- - - ">="
181
+ - - ">"
181
182
  - !ruby/object:Gem::Version
182
- version: '0'
183
+ version: 1.3.1
183
184
  requirements: []
184
185
  rubyforge_project:
185
186
  rubygems_version: 2.4.8