ffwd 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/ffwd.rb +34 -10
- data/lib/ffwd/core.rb +13 -11
- data/lib/ffwd/core/emitter.rb +0 -2
- data/lib/ffwd/core/interface.rb +5 -2
- data/lib/ffwd/core/processor.rb +4 -5
- data/lib/ffwd/debug/monitor_session.rb +3 -6
- data/lib/ffwd/debug/tcp.rb +17 -12
- data/lib/ffwd/handler.rb +1 -1
- data/lib/ffwd/plugin.rb +20 -15
- data/lib/ffwd/plugin/json.rb +12 -20
- data/lib/ffwd/plugin/json/connection.rb +4 -2
- data/lib/ffwd/plugin/log.rb +17 -2
- data/lib/ffwd/plugin/log/writer.rb +4 -0
- data/lib/ffwd/plugin_channel.rb +8 -8
- data/lib/ffwd/processor.rb +11 -4
- data/lib/ffwd/processor/count.rb +16 -6
- data/lib/ffwd/processor/histogram.rb +26 -15
- data/lib/ffwd/processor/rate.rb +16 -6
- data/lib/ffwd/protocol/tcp.rb +47 -88
- data/lib/ffwd/protocol/tcp/bind.rb +22 -10
- data/lib/ffwd/protocol/tcp/connection.rb +23 -6
- data/lib/ffwd/protocol/tcp/flushing_connect.rb +42 -25
- data/lib/ffwd/protocol/tcp/plain_connect.rb +15 -3
- data/lib/ffwd/protocol/udp.rb +42 -17
- data/lib/ffwd/protocol/udp/bind.rb +21 -10
- data/lib/ffwd/protocol/udp/connect.rb +22 -9
- data/lib/ffwd/retrier.rb +4 -1
- data/lib/ffwd/schema.rb +10 -3
- data/lib/ffwd/statistics/collector.rb +31 -23
- data/lib/ffwd/test/protocol.rb +42 -0
- data/lib/ffwd/tunnel/tcp.rb +3 -1
- data/lib/ffwd/utils.rb +7 -0
- data/lib/ffwd/version.rb +1 -1
- metadata +56 -68
- data/lib/ffwd/circular_buffer.rb +0 -78
- data/lib/ffwd/statistics.rb +0 -29
- data/lib/ffwd/tunnel.rb +0 -27
@@ -19,7 +19,9 @@ require 'ffwd/logging'
|
|
19
19
|
require 'ffwd/connection'
|
20
20
|
|
21
21
|
module FFWD::Plugin::JSON
|
22
|
-
|
22
|
+
class Connection < FFWD::Connection
|
23
|
+
include FFWD::Logging
|
24
|
+
|
23
25
|
EVENT_FIELDS = [
|
24
26
|
["key", :key],
|
25
27
|
["value", :value],
|
@@ -39,7 +41,7 @@ module FFWD::Plugin::JSON
|
|
39
41
|
["attributes", :attributes]
|
40
42
|
]
|
41
43
|
|
42
|
-
def initialize bind, core
|
44
|
+
def initialize bind, core, config
|
43
45
|
@bind = bind
|
44
46
|
@core = core
|
45
47
|
end
|
data/lib/ffwd/plugin/log.rb
CHANGED
@@ -25,6 +25,8 @@ module FFWD::Plugin
|
|
25
25
|
include FFWD::Plugin
|
26
26
|
include FFWD::Logging
|
27
27
|
|
28
|
+
DEFAULT_PREFIX = ''
|
29
|
+
|
28
30
|
register_plugin "log",
|
29
31
|
:description => "A simple plugin that outputs to the primary log.",
|
30
32
|
:options => [
|
@@ -33,8 +35,21 @@ module FFWD::Plugin
|
|
33
35
|
]),
|
34
36
|
]
|
35
37
|
|
36
|
-
|
37
|
-
|
38
|
+
class Setup
|
39
|
+
attr_reader :config
|
40
|
+
|
41
|
+
def initialize config
|
42
|
+
@config = config
|
43
|
+
@config[:prefix] ||= DEFAULT_PREFIX
|
44
|
+
end
|
45
|
+
|
46
|
+
def connect core
|
47
|
+
Writer.new core, @config[:prefix]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.setup_output config
|
52
|
+
Setup.new config
|
38
53
|
end
|
39
54
|
end
|
40
55
|
end
|
@@ -25,6 +25,8 @@ module FFWD::Plugin::Log
|
|
25
25
|
subs = []
|
26
26
|
|
27
27
|
core.output.starting do
|
28
|
+
log.info "Started (prefix: '#{@p}')"
|
29
|
+
|
28
30
|
subs << core.output.event_subscribe do |e|
|
29
31
|
log.info "Event: #{@p}#{e.to_h}"
|
30
32
|
end
|
@@ -35,6 +37,8 @@ module FFWD::Plugin::Log
|
|
35
37
|
end
|
36
38
|
|
37
39
|
core.output.stopping do
|
40
|
+
log.info "Stopped"
|
41
|
+
|
38
42
|
subs.each(&:unsubscribe).clear
|
39
43
|
end
|
40
44
|
end
|
data/lib/ffwd/plugin_channel.rb
CHANGED
@@ -28,19 +28,19 @@ module FFWD
|
|
28
28
|
|
29
29
|
setup_reporter :keys => [:metrics, :events]
|
30
30
|
|
31
|
-
attr_reader :
|
31
|
+
attr_reader :id, :events, :metrics, :reporter_meta
|
32
32
|
|
33
|
-
def self.build
|
34
|
-
events = FFWD::Channel.new log, "#{
|
35
|
-
metrics = FFWD::Channel.new log, "#{
|
36
|
-
new
|
33
|
+
def self.build id
|
34
|
+
events = FFWD::Channel.new log, "#{id}.events"
|
35
|
+
metrics = FFWD::Channel.new log, "#{id}.metrics"
|
36
|
+
new id, metrics, events
|
37
37
|
end
|
38
38
|
|
39
|
-
def initialize
|
40
|
-
@
|
39
|
+
def initialize id, events, metrics
|
40
|
+
@id = id
|
41
41
|
@events = events
|
42
42
|
@metrics = metrics
|
43
|
-
@reporter_meta = {:plugin_channel => @
|
43
|
+
@reporter_meta = {:plugin_channel => @id, :type => "plugin_channel"}
|
44
44
|
end
|
45
45
|
|
46
46
|
def event_subscribe &block
|
data/lib/ffwd/processor.rb
CHANGED
@@ -23,14 +23,14 @@ module FFWD::Processor
|
|
23
23
|
class Setup
|
24
24
|
attr_reader :name
|
25
25
|
|
26
|
-
def initialize name,
|
26
|
+
def initialize name, factory, options
|
27
27
|
@name = name
|
28
|
-
@
|
28
|
+
@factory = factory
|
29
29
|
@options = options
|
30
30
|
end
|
31
31
|
|
32
32
|
def setup emitter
|
33
|
-
@
|
33
|
+
@factory.new emitter, @options
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
@@ -91,6 +91,13 @@ module FFWD::Processor
|
|
91
91
|
|
92
92
|
# setup hash of processor setup classes.
|
93
93
|
def self.load_processors config
|
94
|
-
registry.map
|
94
|
+
registry.map do |name, klass|
|
95
|
+
unless klass.respond_to? :prepare
|
96
|
+
raise "Processor #{klass} does not have a 'prepare' method"
|
97
|
+
end
|
98
|
+
|
99
|
+
config = klass.prepare Hash[config[name] || {}]
|
100
|
+
Setup.new name, klass, config
|
101
|
+
end
|
95
102
|
end
|
96
103
|
end
|
data/lib/ffwd/processor/count.rb
CHANGED
@@ -34,16 +34,26 @@ module FFWD::Processor
|
|
34
34
|
:keys => [:dropped, :received]
|
35
35
|
)
|
36
36
|
|
37
|
-
def
|
37
|
+
def self.prepare config
|
38
|
+
config[:cache_limit] ||= 1000
|
39
|
+
config[:timeout] ||= 300
|
40
|
+
config[:window] ||= 30
|
41
|
+
config
|
42
|
+
end
|
43
|
+
|
44
|
+
def initialize emitter, config={}
|
38
45
|
@emitter = emitter
|
39
|
-
|
40
|
-
@
|
41
|
-
@
|
46
|
+
|
47
|
+
@cache_limit = config[:cache_limit]
|
48
|
+
@timeout = config[:timeout]
|
49
|
+
@window = config[:window]
|
50
|
+
|
42
51
|
@cache = {}
|
43
52
|
@timer = nil
|
44
53
|
|
45
54
|
starting do
|
46
|
-
log.info "
|
55
|
+
log.info "Started"
|
56
|
+
log.info " config: #{config.inspect}"
|
47
57
|
end
|
48
58
|
|
49
59
|
stopping do
|
@@ -81,7 +91,7 @@ module FFWD::Processor
|
|
81
91
|
|
82
92
|
log.debug "Starting timer"
|
83
93
|
|
84
|
-
@timer = EM::Timer.new(@
|
94
|
+
@timer = EM::Timer.new(@window) do
|
85
95
|
@timer = nil
|
86
96
|
digest! Time.now
|
87
97
|
end
|
@@ -49,13 +49,23 @@ module FFWD::Processor
|
|
49
49
|
DEFAULT_MISSING = 0
|
50
50
|
|
51
51
|
DEFAULT_PERCENTILES = {
|
52
|
-
:p50 => {:
|
53
|
-
:p75 => {:
|
54
|
-
:p95 => {:
|
55
|
-
:p99 => {:
|
56
|
-
:p999 => {:
|
52
|
+
:p50 => {:q => 0.50, :info => "50th"},
|
53
|
+
:p75 => {:q => 0.75, :info => "75th"},
|
54
|
+
:p95 => {:q => 0.95, :info => "95th"},
|
55
|
+
:p99 => {:q => 0.99, :info => "99th"},
|
56
|
+
:p999 => {:q => 0.999, :info => "99.9th"},
|
57
57
|
}
|
58
58
|
|
59
|
+
def self.prepare config
|
60
|
+
config[:window] ||= 10
|
61
|
+
config[:cache_limit] ||= 1000
|
62
|
+
config[:bucket_limit] ||= 10000
|
63
|
+
config[:precision] ||= 3
|
64
|
+
config[:missing] ||= DEFAULT_MISSING
|
65
|
+
config[:percentiles] ||= DEFAULT_PERCENTILES
|
66
|
+
config
|
67
|
+
end
|
68
|
+
|
59
69
|
#
|
60
70
|
# Options:
|
61
71
|
#
|
@@ -66,22 +76,23 @@ module FFWD::Processor
|
|
66
76
|
# :precision - Precision of emitted metrics.
|
67
77
|
# :percentiles - Configuration hash of percentile metrics.
|
68
78
|
# Structure:
|
69
|
-
# {:p10 => {:info => "Some description", :
|
70
|
-
def initialize emitter,
|
79
|
+
# {:p10 => {:info => "Some description", :q => 0.1}, ...}
|
80
|
+
def initialize emitter, config={}
|
71
81
|
@emitter = emitter
|
72
82
|
|
73
|
-
@window =
|
74
|
-
@cache_limit =
|
75
|
-
@bucket_limit =
|
76
|
-
@precision =
|
77
|
-
@missing =
|
78
|
-
@percentiles =
|
83
|
+
@window = config[:window]
|
84
|
+
@cache_limit = config[:cache_limit]
|
85
|
+
@bucket_limit = config[:bucket_limit]
|
86
|
+
@precision = config[:precision]
|
87
|
+
@missing = config[:missing]
|
88
|
+
@percentiles = config[:percentiles]
|
79
89
|
|
80
90
|
# Dropped values that would have gone into a bucket.
|
81
91
|
@cache = {}
|
82
92
|
|
83
93
|
starting do
|
84
|
-
log.info "
|
94
|
+
log.info "Started"
|
95
|
+
log.info " config: #{config.inspect}"
|
85
96
|
end
|
86
97
|
|
87
98
|
stopping do
|
@@ -130,7 +141,7 @@ module FFWD::Processor
|
|
130
141
|
map = {}
|
131
142
|
|
132
143
|
@percentiles.each do |k, v|
|
133
|
-
index = (total * v[:
|
144
|
+
index = (total * v[:q]).ceil - 1
|
134
145
|
|
135
146
|
if (c = map[index]).nil?
|
136
147
|
info = "#{v[:info]} percentile"
|
data/lib/ffwd/processor/rate.rb
CHANGED
@@ -44,13 +44,22 @@ module FFWD::Processor
|
|
44
44
|
# :ttl - Allowed age of items in cache in seconds.
|
45
45
|
# If this is nil, items will never expire, so old elements will not be
|
46
46
|
# expunged until data type is restarted.
|
47
|
-
def
|
47
|
+
def self.prepare config={}
|
48
|
+
config[:precision] ||= 3
|
49
|
+
config[:cache_limit] ||= 10000
|
50
|
+
config[:min_age] ||= 0.5
|
51
|
+
config[:ttl] ||= 600
|
52
|
+
config
|
53
|
+
end
|
54
|
+
|
55
|
+
def initialize emitter, config={}
|
48
56
|
@emitter = emitter
|
49
57
|
|
50
|
-
@precision =
|
51
|
-
@limit =
|
52
|
-
@min_age =
|
53
|
-
@ttl =
|
58
|
+
@precision = config[:precision]
|
59
|
+
@limit = config[:cache_limit]
|
60
|
+
@min_age = config[:min_age]
|
61
|
+
@ttl = config[:ttl]
|
62
|
+
|
54
63
|
# keep a reference to the expire cache to prevent having to allocate it
|
55
64
|
# all the time.
|
56
65
|
@expire = Hash.new
|
@@ -58,8 +67,9 @@ module FFWD::Processor
|
|
58
67
|
@cache = Hash.new
|
59
68
|
|
60
69
|
starting do
|
61
|
-
log.info "Starting rate processor (ttl: #{@ttl})"
|
62
70
|
@timer = EM.add_periodic_timer(@ttl){expire!} unless @ttl.nil?
|
71
|
+
log.info "Started"
|
72
|
+
log.info " config: #{config.inspect}"
|
63
73
|
end
|
64
74
|
|
65
75
|
stopping do
|
data/lib/ffwd/protocol/tcp.rb
CHANGED
@@ -13,9 +13,7 @@
|
|
13
13
|
# License for the specific language governing permissions and limitations under
|
14
14
|
# the License.
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
require_relative '../tunnel'
|
16
|
+
require_relative '../tunnel/tcp'
|
19
17
|
|
20
18
|
require_relative 'tcp/bind'
|
21
19
|
require_relative 'tcp/plain_connect'
|
@@ -27,100 +25,61 @@ module FFWD::TCP
|
|
27
25
|
:tcp
|
28
26
|
end
|
29
27
|
|
30
|
-
|
31
|
-
|
32
|
-
DEFAULT_OUTBOUND_LIMIT = 2 ** 20
|
33
|
-
# default flush period, if non-zero will cause the connection to be buffered.
|
34
|
-
DEFAULT_FLUSH_PERIOD = 10
|
35
|
-
# defaults for buffered connections.
|
36
|
-
# maximum amount of events to buffer up.
|
37
|
-
DEFAULT_EVENT_LIMIT = 1000
|
38
|
-
# maximum amount of metrics to buffer up.
|
39
|
-
DEFAULT_METRIC_LIMIT = 10000
|
40
|
-
# percent of maximum events/metrics which will cause a flush.
|
41
|
-
DEFAULT_FLUSH_LIMIT = 0.8
|
42
|
-
# Default initial timeout when binding fails.
|
43
|
-
DEFAULT_REBIND_TIMEOUT = 10
|
28
|
+
class SetupOutput
|
29
|
+
attr_reader :config
|
44
30
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
# :host - The host to connect to.
|
50
|
-
# :port - The port to connect to.
|
51
|
-
# :outbound_limit - The amount of bytes that are allowed to be pending in
|
52
|
-
# the application level buffer for the connection to be considered
|
53
|
-
# 'writable'.
|
54
|
-
# :flush_period - The period in which outgoing data is buffered. If this
|
55
|
-
# is 0 no buffering will occur.
|
56
|
-
# Reads the following keys if the connection is buffered.
|
57
|
-
# :event_limit - The maximum amount of events the connection is allowed
|
58
|
-
# to buffer up.
|
59
|
-
# :metric_limimt - The maximum amount of metrics the connection is
|
60
|
-
# allowed to buffer up.
|
61
|
-
# :flush_limit - A percentage (0.0 - 1.0) indicating 'the percentage of
|
62
|
-
# events/metrics' that is required for a flush to be forced.
|
63
|
-
# If this percentage is reached, the connection will attempt to forcibly
|
64
|
-
# flush all buffered events and metrics prior to the end of the flushing
|
65
|
-
# period.
|
66
|
-
# core - The core interface associated with this connection.
|
67
|
-
# log - The logger to use for this connection.
|
68
|
-
# handler - An implementation of FFWD::Handler containing the connection
|
69
|
-
# logic.
|
70
|
-
# args - Arguments passed to the handler when a new instance is created.
|
71
|
-
def self.connect opts, core, log, handler, *args
|
72
|
-
raise "Missing required option :host" if (host = opts[:host]).nil?
|
73
|
-
raise "Missing required option :port" if (port = opts[:port]).nil?
|
31
|
+
def initialize config, log, handler
|
32
|
+
@config = Hash[config]
|
33
|
+
@log = log
|
34
|
+
@handler = handler
|
74
35
|
|
75
|
-
|
76
|
-
flush_period = opts[:flush_period] || DEFAULT_FLUSH_PERIOD
|
36
|
+
@config = Connection.prepare @config
|
77
37
|
|
78
|
-
|
38
|
+
if @config[:flush_period] == 0
|
39
|
+
@config = PlainConnect.prepare @config
|
40
|
+
@type = PlainConnect
|
41
|
+
else
|
42
|
+
@config = FlushingConnect.prepare @config
|
43
|
+
@type = FlushingConnect
|
44
|
+
end
|
45
|
+
end
|
79
46
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
event_limit = opts[:event_limit] || DEFAULT_EVENT_LIMIT
|
84
|
-
metric_limit = opts[:metric_limit] || DEFAULT_METRIC_LIMIT
|
85
|
-
flush_limit = opts[:flush_limit] || DEFAULT_FLUSH_LIMIT
|
47
|
+
def connect core
|
48
|
+
raise "Missing required option :host" if (host = @config[:host]).nil?
|
49
|
+
raise "Missing required option :port" if (port = @config[:port]).nil?
|
86
50
|
|
87
|
-
|
88
|
-
|
89
|
-
flush_period, event_limit, metric_limit, flush_limit
|
90
|
-
)
|
51
|
+
c = Connection.new @log, host, port, @handler, @config
|
52
|
+
@type.new core, @log, c, @config
|
91
53
|
end
|
92
54
|
end
|
93
55
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
56
|
+
def self.connect config, log, handler
|
57
|
+
SetupOutput.new config, log, handler
|
58
|
+
end
|
59
|
+
|
60
|
+
class SetupInput
|
61
|
+
attr_reader :config
|
62
|
+
|
63
|
+
def initialize config, log, connection
|
64
|
+
@config = Hash[config]
|
65
|
+
@log = log
|
66
|
+
@connection = connection
|
67
|
+
end
|
68
|
+
|
69
|
+
def bind core
|
70
|
+
raise "Missing required option :host" if (host = @config[:host]).nil?
|
71
|
+
raise "Missing required option :port" if (port = @config[:port]).nil?
|
72
|
+
@config = Bind.prepare Hash[@config]
|
73
|
+
Bind.new core, @log, host, port, @connection, @config
|
74
|
+
end
|
75
|
+
|
76
|
+
def tunnel core, plugin
|
77
|
+
raise "Missing required option :port" if (port = @config[:port]).nil?
|
78
|
+
FFWD::Tunnel::TCP.new port, core, plugin, @log, @connection
|
79
|
+
end
|
111
80
|
end
|
112
81
|
|
113
|
-
|
114
|
-
|
115
|
-
# opts - Option hash.
|
116
|
-
# :port - The port to bind to on the remote side.
|
117
|
-
# core - The core interface associated with this connection.
|
118
|
-
# log - The logger to use for this connection.
|
119
|
-
# connection - An implementation of FFWD::Connection containing the
|
120
|
-
# connection logic.
|
121
|
-
# args - Arguments passed to the connection when a new instance is created.
|
122
|
-
def self.tunnel opts, core, plugin, log, connection, *args
|
123
|
-
raise "Missing required option :port" if (port = opts[:port]).nil?
|
124
|
-
FFWD.tunnel self.family, port, core, plugin, log, connection, args
|
82
|
+
def self.bind config, log, connection
|
83
|
+
SetupInput.new config, log, connection
|
125
84
|
end
|
126
85
|
end
|
@@ -22,42 +22,54 @@ module FFWD::TCP
|
|
22
22
|
class Bind
|
23
23
|
include FFWD::Reporter
|
24
24
|
|
25
|
+
# Default initial timeout when binding fails.
|
26
|
+
DEFAULT_REBIND_TIMEOUT = 10
|
27
|
+
|
28
|
+
def self.prepare opts
|
29
|
+
opts[:rebind_timeout] ||= DEFAULT_REBIND_TIMEOUT
|
30
|
+
opts
|
31
|
+
end
|
32
|
+
|
25
33
|
setup_reporter :keys => [
|
26
34
|
:received_events, :received_metrics,
|
27
35
|
:failed_events, :failed_metrics
|
28
36
|
]
|
29
37
|
|
30
|
-
attr_reader :reporter_meta
|
38
|
+
attr_reader :log, :reporter_meta
|
31
39
|
|
32
|
-
def initialize core, log, host, port, connection,
|
40
|
+
def initialize core, log, host, port, connection, config
|
41
|
+
@log = log
|
33
42
|
@peer = "#{host}:#{port}"
|
34
43
|
@reporter_meta = {
|
35
44
|
:type => connection.plugin_type,
|
36
45
|
:listen => @peer, :family => 'tcp'
|
37
46
|
}
|
38
47
|
|
39
|
-
@
|
48
|
+
@server = nil
|
40
49
|
|
41
50
|
info = "tcp://#{@peer}"
|
51
|
+
rebind_timeout = config[:rebind_timeout]
|
42
52
|
|
43
53
|
r = FFWD.retry :timeout => rebind_timeout do |a|
|
44
|
-
@
|
54
|
+
@server = EM.start_server host, port, connection, self, core, config
|
55
|
+
|
45
56
|
log.info "Bind on #{info} (attempt #{a})"
|
57
|
+
log.info " config: #{config.inspect}"
|
46
58
|
end
|
47
59
|
|
48
60
|
r.error do |a, t, e|
|
49
|
-
log.
|
61
|
+
log.warning "Bind on #{info} failed, retry ##{a} in #{t}s: #{e}"
|
50
62
|
end
|
51
63
|
|
52
64
|
r.depend_on core
|
53
65
|
|
54
66
|
core.stopping do
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
EM.stop_server @sig
|
59
|
-
@sig = nil
|
67
|
+
if @server
|
68
|
+
EM.stop_server @server
|
69
|
+
@server = nil
|
60
70
|
end
|
71
|
+
|
72
|
+
log.info "Unbound #{info}"
|
61
73
|
end
|
62
74
|
end
|
63
75
|
end
|