fleck 0.7.0.rc → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +11 -10
- data/CHANGELOG.md +89 -71
- data/Gemfile +6 -4
- data/examples/actions.rb +59 -53
- data/examples/blocking_consumer.rb +42 -42
- data/examples/consumer_initialization.rb +44 -42
- data/examples/deprecation.rb +50 -57
- data/examples/example.rb +76 -74
- data/examples/expired.rb +72 -76
- data/examples/fanout.rb +62 -64
- data/fleck.gemspec +37 -36
- data/lib/fleck/client.rb +124 -124
- data/lib/fleck/configuration.rb +149 -144
- data/lib/fleck/consumer.rb +7 -287
- data/lib/fleck/core/consumer/action_param.rb +106 -0
- data/lib/fleck/core/consumer/actions.rb +76 -0
- data/lib/fleck/core/consumer/base.rb +111 -0
- data/lib/fleck/core/consumer/configuration.rb +69 -0
- data/lib/fleck/core/consumer/decorators.rb +77 -0
- data/lib/fleck/core/consumer/helpers_definers.rb +55 -0
- data/lib/fleck/core/consumer/logger.rb +88 -0
- data/lib/fleck/core/consumer/request.rb +89 -0
- data/lib/fleck/core/consumer/response.rb +77 -0
- data/lib/fleck/core/consumer/response_helpers.rb +81 -0
- data/lib/fleck/core/consumer/validation.rb +163 -0
- data/lib/fleck/core/consumer.rb +166 -0
- data/lib/fleck/core.rb +9 -0
- data/lib/fleck/loggable.rb +15 -10
- data/lib/fleck/{hash_with_indifferent_access.rb → utilities/hash_with_indifferent_access.rb} +80 -85
- data/lib/fleck/utilities/host_rating.rb +104 -0
- data/lib/fleck/version.rb +6 -3
- data/lib/fleck.rb +81 -72
- metadata +44 -35
- data/lib/fleck/consumer/request.rb +0 -52
- data/lib/fleck/consumer/response.rb +0 -80
- data/lib/fleck/host_rating.rb +0 -74
@@ -0,0 +1,166 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH << File.expand_path(__dir__)
|
4
|
+
|
5
|
+
module Fleck
|
6
|
+
module Core
|
7
|
+
# `Fleck::Core::Consumer` implements the core functionality for `Fleck::Consumer`,
|
8
|
+
# so that new `Fleck::Consumers` can be easily created by using inheritance.
|
9
|
+
class Consumer
|
10
|
+
autoload :Base, 'consumer/base.rb'
|
11
|
+
autoload :Actions, 'consumer/actions.rb'
|
12
|
+
autoload :Configuration, 'consumer/configuration.rb'
|
13
|
+
autoload :Decorators, 'consumer/decorators.rb'
|
14
|
+
autoload :HelpersDefiners, 'consumer/helpers_definers.rb'
|
15
|
+
autoload :Logger, 'consumer/logger.rb'
|
16
|
+
|
17
|
+
autoload :Request, 'consumer/request.rb'
|
18
|
+
autoload :Response, 'consumer/response.rb'
|
19
|
+
|
20
|
+
autoload :ActionHeader, 'consumer/action_header.rb'
|
21
|
+
autoload :ActionParam, 'consumer/action_param.rb'
|
22
|
+
autoload :Validation, 'consumer/validation.rb'
|
23
|
+
|
24
|
+
include Fleck::Loggable
|
25
|
+
include Logger
|
26
|
+
include Actions
|
27
|
+
include Configuration
|
28
|
+
include Decorators
|
29
|
+
include HelpersDefiners
|
30
|
+
include Base
|
31
|
+
|
32
|
+
require_relative 'consumer/response_helpers'
|
33
|
+
|
34
|
+
attr_accessor :connection, :channel, :queue, :publisher, :consumer_tag, :request, :subscription, :exchange,
|
35
|
+
:consumer_id
|
36
|
+
|
37
|
+
def initialize(consumer_id = nil)
|
38
|
+
self.consumer_id = consumer_id
|
39
|
+
|
40
|
+
instance_eval(&self.class.initialize_block) if self.class.initialize_block
|
41
|
+
|
42
|
+
start if autostart?
|
43
|
+
|
44
|
+
at_exit do
|
45
|
+
terminate
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def pause
|
50
|
+
return if subscription.nil? || channel.nil? || channel.closed?
|
51
|
+
|
52
|
+
cancel_ok = subscription.cancel
|
53
|
+
self.consumer_tag = cancel_ok.consumer_tag
|
54
|
+
end
|
55
|
+
|
56
|
+
def resume
|
57
|
+
subscribe!
|
58
|
+
end
|
59
|
+
|
60
|
+
def headers
|
61
|
+
request.headers
|
62
|
+
end
|
63
|
+
|
64
|
+
def params
|
65
|
+
request.params
|
66
|
+
end
|
67
|
+
|
68
|
+
def response
|
69
|
+
request.response
|
70
|
+
end
|
71
|
+
|
72
|
+
def deprecated!
|
73
|
+
logger.warn "DEPRECATION: the method `#{caller_locations(1, 1)[0].label}` is going to be deprecated. " \
|
74
|
+
'Please, consider using a newer version of this method.'
|
75
|
+
response&.deprecated!
|
76
|
+
end
|
77
|
+
|
78
|
+
protected
|
79
|
+
|
80
|
+
def connect!
|
81
|
+
self.connection = Fleck.connection(
|
82
|
+
host: rmq_host,
|
83
|
+
port: rmq_port,
|
84
|
+
user: rmq_user,
|
85
|
+
pass: rmq_pass,
|
86
|
+
vhost: rmq_vhost
|
87
|
+
)
|
88
|
+
end
|
89
|
+
|
90
|
+
def create_channel!
|
91
|
+
if channel && !channel.closed?
|
92
|
+
logger.info('Closing the opened channel...')
|
93
|
+
channel.close
|
94
|
+
end
|
95
|
+
|
96
|
+
logger.debug "Creating a new channel for #{self.class.to_s.color(:yellow)} consumer"
|
97
|
+
self.channel = connection.create_channel
|
98
|
+
channel.prefetch(prefetch_size) # consume messages in batches
|
99
|
+
self.publisher = Bunny::Exchange.new(connection.create_channel, :direct, 'fleck')
|
100
|
+
if rmq_exchange_type == :direct && rmq_exchange_name == ''
|
101
|
+
self.queue = channel.queue(queue_name, auto_delete: false)
|
102
|
+
else
|
103
|
+
self.exchange = Bunny::Exchange.new(channel, rmq_exchange_type, rmq_exchange_name)
|
104
|
+
self.queue = channel.queue('', exclusive: true, auto_delete: true).bind(exchange, routing_key: queue_name)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def subscribe!
|
109
|
+
logger.debug "Consuming from queue: #{queue_name.color(:green)}"
|
110
|
+
|
111
|
+
options = { manual_ack: true }
|
112
|
+
options[:consumer_tag] = consumer_tag if consumer_tag
|
113
|
+
|
114
|
+
self.subscription = queue.subscribe(options) do |delivery_info, metadata, payload|
|
115
|
+
process_request!(metadata, payload, delivery_info)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def process_request!(metadata, payload, delivery_info)
|
120
|
+
self.request = Request.new(metadata, payload, delivery_info)
|
121
|
+
|
122
|
+
catch(INTERRUPT_NAME) do
|
123
|
+
execute_action! unless request.failed?
|
124
|
+
end
|
125
|
+
rescue StandardError => e
|
126
|
+
log_error(e)
|
127
|
+
internal_server_error! e.inspect, interrupt: false
|
128
|
+
ensure
|
129
|
+
send_response!
|
130
|
+
log_request
|
131
|
+
end
|
132
|
+
|
133
|
+
def send_response!
|
134
|
+
return reject_request! if request.rejected?
|
135
|
+
|
136
|
+
logger.debug "Sending response: #{response}"
|
137
|
+
|
138
|
+
if channel.closed?
|
139
|
+
return logger.warn "Channel already closed! The response #{request.id} is going to be dropped."
|
140
|
+
end
|
141
|
+
|
142
|
+
publish_response!
|
143
|
+
request.processed!
|
144
|
+
end
|
145
|
+
|
146
|
+
def reject_request!
|
147
|
+
channel.reject(request.delivery_tag, request.requeue?)
|
148
|
+
end
|
149
|
+
|
150
|
+
def publish_response!
|
151
|
+
publisher.publish(
|
152
|
+
response.to_json,
|
153
|
+
routing_key: request.reply_to,
|
154
|
+
correlation_id: request.id,
|
155
|
+
mandatory: ack_mandatory?
|
156
|
+
)
|
157
|
+
channel.ack(request.delivery_tag)
|
158
|
+
end
|
159
|
+
|
160
|
+
def restart!
|
161
|
+
create_channel!
|
162
|
+
subscribe!
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
data/lib/fleck/core.rb
ADDED
data/lib/fleck/loggable.rb
CHANGED
@@ -1,10 +1,15 @@
|
|
1
|
-
|
2
|
-
module Fleck::Loggable
|
3
|
-
def logger
|
4
|
-
return @logger if @logger
|
5
|
-
|
6
|
-
@logger
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
1
|
+
|
2
|
+
module Fleck::Loggable
|
3
|
+
def logger
|
4
|
+
return @logger if @logger
|
5
|
+
|
6
|
+
@logger = Fleck.logger.clone
|
7
|
+
@logger.progname = self.class.name
|
8
|
+
|
9
|
+
@logger
|
10
|
+
end
|
11
|
+
|
12
|
+
def log_error(error)
|
13
|
+
logger.error "#{error.inspect}\n#{error.backtrace.join("\n")}"
|
14
|
+
end
|
15
|
+
end
|
data/lib/fleck/{hash_with_indifferent_access.rb → utilities/hash_with_indifferent_access.rb}
RENAMED
@@ -1,85 +1,80 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
value
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
end
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
def
|
54
|
-
return
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
end
|
82
|
-
|
83
|
-
return self
|
84
|
-
end
|
85
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# `HashWithIndifferentAccess` extends `Hash` class and adds the possibility to access values
|
4
|
+
# by using both string and symbol keys indifferently.
|
5
|
+
class HashWithIndifferentAccess < Hash
|
6
|
+
def initialize(original)
|
7
|
+
super(nil)
|
8
|
+
copy_from(original)
|
9
|
+
end
|
10
|
+
|
11
|
+
def []=(key, value)
|
12
|
+
super(key.to_s, self.class.convert_value(value))
|
13
|
+
end
|
14
|
+
|
15
|
+
def [](key)
|
16
|
+
super(key.to_s)
|
17
|
+
end
|
18
|
+
|
19
|
+
def fetch(key, *extras)
|
20
|
+
super(key.to_s, *extras)
|
21
|
+
end
|
22
|
+
|
23
|
+
def delete(key)
|
24
|
+
super(key.to_s)
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.convert_value(value)
|
28
|
+
case value
|
29
|
+
when Hash
|
30
|
+
value.to_hash_with_indifferent_access
|
31
|
+
when Array
|
32
|
+
value.map! { |item| item.is_a?(Hash) || item.is_a?(Array) ? HashWithIndifferentAccess.convert_value(item) : item }
|
33
|
+
else
|
34
|
+
value
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
protected
|
39
|
+
|
40
|
+
def copy_from(original)
|
41
|
+
original.each do |key, value|
|
42
|
+
self[key] = self.class.convert_value(value)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Open `Hash` class to add `#to_hash_with_indifferent_access` method and some filter features.
|
48
|
+
class Hash
|
49
|
+
def to_hash_with_indifferent_access
|
50
|
+
HashWithIndifferentAccess.new(self)
|
51
|
+
end
|
52
|
+
|
53
|
+
def to_s
|
54
|
+
return dup.filter!.inspect if @filtered
|
55
|
+
|
56
|
+
super
|
57
|
+
end
|
58
|
+
|
59
|
+
def filtered!
|
60
|
+
@filtered = true
|
61
|
+
keys.each do |key|
|
62
|
+
self[key].filtered! if self[key].is_a?(Hash)
|
63
|
+
end
|
64
|
+
|
65
|
+
self
|
66
|
+
end
|
67
|
+
|
68
|
+
def filter!
|
69
|
+
filters = Fleck.config.filters
|
70
|
+
keys.each do |key|
|
71
|
+
if filters.include?(key.to_s)
|
72
|
+
self[key] = '[FILTERED]'
|
73
|
+
elsif self[key].is_a?(Hash)
|
74
|
+
self[key] = self[key].dup.filter!
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
self
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'socket'
|
4
|
+
|
5
|
+
# Open `Fleck` module to add `HostRating` class.
|
6
|
+
module Fleck
|
7
|
+
# `HostRating` class allows to test host latency on a regular basis and to compare the latency
|
8
|
+
# with other hosts.
|
9
|
+
class HostRating
|
10
|
+
include Fleck::Loggable
|
11
|
+
|
12
|
+
CONN_TIMEOUT = 5
|
13
|
+
|
14
|
+
attr_reader :host, :port, :avg, :history
|
15
|
+
|
16
|
+
def initialize(host: 'localhost', port: 5672, refresh_rate: 30_000, period: 300_000)
|
17
|
+
@host = host
|
18
|
+
@port = port
|
19
|
+
@refresh_rate = refresh_rate
|
20
|
+
@period = period
|
21
|
+
|
22
|
+
# metrics
|
23
|
+
@reachable = false
|
24
|
+
@avg = 0
|
25
|
+
@updated_at = nil
|
26
|
+
@history = []
|
27
|
+
|
28
|
+
refresh!
|
29
|
+
@timer = Ztimer.every(@refresh_rate) { refresh! }
|
30
|
+
end
|
31
|
+
|
32
|
+
def reachable?
|
33
|
+
@reachable
|
34
|
+
end
|
35
|
+
|
36
|
+
def close
|
37
|
+
@timer.cancel!
|
38
|
+
end
|
39
|
+
|
40
|
+
def <=>(other)
|
41
|
+
# the other host is reachable, so it comes first
|
42
|
+
return 1 if !reachable? && other.reachable?
|
43
|
+
|
44
|
+
# both host are unreachable, so they have the same priority
|
45
|
+
return 0 unless reachable? || other.reachable?
|
46
|
+
|
47
|
+
# the current host comes first, because it's reachable, while the other host is unreachable
|
48
|
+
return -1 if reachable? && !other.reachable?
|
49
|
+
|
50
|
+
# when both hosts are reachable, use avg latency to order them
|
51
|
+
avg <=> other.avg
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def refresh!
|
57
|
+
@history << measure_latency
|
58
|
+
@history.shift if @history.size > @period / @refresh_rate
|
59
|
+
@avg = @history.inject(:+).to_f / @history.size
|
60
|
+
@reachable = true
|
61
|
+
rescue SocketError, Timeout::Error => e
|
62
|
+
@reachable = false
|
63
|
+
logger.error "Connection error: #{@host}:#{@port} (#{e.inspect})"
|
64
|
+
ensure
|
65
|
+
@updated_at = Time.now
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
# Use a socket to test connection latency.
|
72
|
+
def measure_latency
|
73
|
+
socket = create_socket
|
74
|
+
|
75
|
+
started_at = Time.now.to_f
|
76
|
+
begin
|
77
|
+
socket.connect_nonblock(sock_addr)
|
78
|
+
rescue IO::WaitWritable
|
79
|
+
IO.select(nil, [socket], nil, CONN_TIMEOUT) or raise Timeout::Error
|
80
|
+
end
|
81
|
+
|
82
|
+
(Time.now.to_f - started_at) * 1000 # ms
|
83
|
+
ensure
|
84
|
+
socket&.close
|
85
|
+
end
|
86
|
+
|
87
|
+
# Create a new socket for connection test.
|
88
|
+
def create_socket
|
89
|
+
socket = Socket.new(:AF_INET, :SOCK_STREAM, 0)
|
90
|
+
socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
|
91
|
+
|
92
|
+
socket
|
93
|
+
end
|
94
|
+
|
95
|
+
# Resolve domain name in order to obtain IP address to test.
|
96
|
+
def sock_addr
|
97
|
+
return @sock_addr if @sock_addr
|
98
|
+
|
99
|
+
addr = Socket.getaddrinfo(@host, nil)
|
100
|
+
@sock_addr = Socket.pack_sockaddr_in(@port, addr[0][3])
|
101
|
+
|
102
|
+
@sock_addr
|
103
|
+
end
|
104
|
+
end
|
data/lib/fleck/version.rb
CHANGED
data/lib/fleck.rb
CHANGED
@@ -1,72 +1,81 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
12
|
-
require
|
13
|
-
require
|
14
|
-
require
|
15
|
-
require
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
logger
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
Fleck.logger.
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH << File.expand_path(__dir__)
|
4
|
+
|
5
|
+
require 'English'
|
6
|
+
require 'logger'
|
7
|
+
require 'rainbow'
|
8
|
+
require 'rainbow/ext/string'
|
9
|
+
require 'bunny'
|
10
|
+
require 'thread_safe'
|
11
|
+
require 'securerandom'
|
12
|
+
require 'oj'
|
13
|
+
require 'ztimer'
|
14
|
+
# require 'fleck/version'
|
15
|
+
require 'fleck/utilities/hash_with_indifferent_access'
|
16
|
+
# require 'fleck/loggable'
|
17
|
+
# require 'fleck/host_rating'
|
18
|
+
# require 'fleck/configuration'
|
19
|
+
# require 'fleck/consumer'
|
20
|
+
# require 'fleck/client'
|
21
|
+
|
22
|
+
# `Fleck` module implements the features for `Fleck` configuration and messages production/consumption.
|
23
|
+
module Fleck
|
24
|
+
autoload :VERSION, 'fleck/version.rb'
|
25
|
+
autoload :Loggable, 'fleck/loggable.rb'
|
26
|
+
autoload :Configuration, 'fleck/configuration.rb'
|
27
|
+
autoload :Core, 'fleck/core.rb'
|
28
|
+
autoload :Consumer, 'fleck/consumer.rb'
|
29
|
+
autoload :Client, 'fleck/client.rb'
|
30
|
+
|
31
|
+
@config = Configuration.new
|
32
|
+
@consumers = ThreadSafe::Array.new
|
33
|
+
@connections = ThreadSafe::Hash.new
|
34
|
+
|
35
|
+
class << self
|
36
|
+
attr_reader :config, :consumers
|
37
|
+
|
38
|
+
def configure
|
39
|
+
yield @config if block_given?
|
40
|
+
@config
|
41
|
+
end
|
42
|
+
|
43
|
+
def logger
|
44
|
+
@config.logger
|
45
|
+
end
|
46
|
+
|
47
|
+
def register_consumer(consumer_class)
|
48
|
+
return if @consumers.include?(consumer_class)
|
49
|
+
|
50
|
+
@consumers << consumer_class
|
51
|
+
end
|
52
|
+
|
53
|
+
def connection(options = {})
|
54
|
+
opts = Fleck.config.default_options.merge(options)
|
55
|
+
key = "ampq://#{opts[:user]}@#{opts[:host]}:#{opts[:port]}#{opts[:vhost]}"
|
56
|
+
conn = @connections[key]
|
57
|
+
if !conn || conn.closed?
|
58
|
+
opts[:logger] = Fleck.logger.clone
|
59
|
+
opts[:logger].progname += '::Bunny'
|
60
|
+
logger.info "New connection #{key}"
|
61
|
+
conn = Bunny.new(opts)
|
62
|
+
conn.start
|
63
|
+
@connections[key] = conn
|
64
|
+
end
|
65
|
+
|
66
|
+
conn
|
67
|
+
end
|
68
|
+
|
69
|
+
def terminate
|
70
|
+
@connections.each do |key, connection|
|
71
|
+
Fleck.logger.info "Closing connection #{key}"
|
72
|
+
connection.close
|
73
|
+
rescue StandardError => e
|
74
|
+
Fleck.logger.error e.inspect
|
75
|
+
end
|
76
|
+
@connections.clear
|
77
|
+
|
78
|
+
true
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|