ffwd 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,7 +19,9 @@ require 'ffwd/logging'
19
19
  require 'ffwd/connection'
20
20
 
21
21
  module FFWD::Plugin::JSON
22
- module Connection
22
+ class Connection < FFWD::Connection
23
+ include FFWD::Logging
24
+
23
25
  EVENT_FIELDS = [
24
26
  ["key", :key],
25
27
  ["value", :value],
@@ -39,7 +41,7 @@ module FFWD::Plugin::JSON
39
41
  ["attributes", :attributes]
40
42
  ]
41
43
 
42
- def initialize bind, core
44
+ def initialize bind, core, config
43
45
  @bind = bind
44
46
  @core = core
45
47
  end
@@ -25,6 +25,8 @@ module FFWD::Plugin
25
25
  include FFWD::Plugin
26
26
  include FFWD::Logging
27
27
 
28
+ DEFAULT_PREFIX = ''
29
+
28
30
  register_plugin "log",
29
31
  :description => "A simple plugin that outputs to the primary log.",
30
32
  :options => [
@@ -33,8 +35,21 @@ module FFWD::Plugin
33
35
  ]),
34
36
  ]
35
37
 
36
- def self.setup_output opts, core
37
- Writer.new core, opts[:prefix]
38
+ class Setup
39
+ attr_reader :config
40
+
41
+ def initialize config
42
+ @config = config
43
+ @config[:prefix] ||= DEFAULT_PREFIX
44
+ end
45
+
46
+ def connect core
47
+ Writer.new core, @config[:prefix]
48
+ end
49
+ end
50
+
51
+ def self.setup_output config
52
+ Setup.new config
38
53
  end
39
54
  end
40
55
  end
@@ -25,6 +25,8 @@ module FFWD::Plugin::Log
25
25
  subs = []
26
26
 
27
27
  core.output.starting do
28
+ log.info "Started (prefix: '#{@p}')"
29
+
28
30
  subs << core.output.event_subscribe do |e|
29
31
  log.info "Event: #{@p}#{e.to_h}"
30
32
  end
@@ -35,6 +37,8 @@ module FFWD::Plugin::Log
35
37
  end
36
38
 
37
39
  core.output.stopping do
40
+ log.info "Stopped"
41
+
38
42
  subs.each(&:unsubscribe).clear
39
43
  end
40
44
  end
@@ -28,19 +28,19 @@ module FFWD
28
28
 
29
29
  setup_reporter :keys => [:metrics, :events]
30
30
 
31
- attr_reader :reporter_meta, :events, :metrics, :name
31
+ attr_reader :id, :events, :metrics, :reporter_meta
32
32
 
33
- def self.build name
34
- events = FFWD::Channel.new log, "#{name}.events"
35
- metrics = FFWD::Channel.new log, "#{name}.metrics"
36
- new name, metrics, events
33
+ def self.build id
34
+ events = FFWD::Channel.new log, "#{id}.events"
35
+ metrics = FFWD::Channel.new log, "#{id}.metrics"
36
+ new id, metrics, events
37
37
  end
38
38
 
39
- def initialize name, events, metrics
40
- @name = name
39
+ def initialize id, events, metrics
40
+ @id = id
41
41
  @events = events
42
42
  @metrics = metrics
43
- @reporter_meta = {:plugin_channel => @name, :type => "plugin_channel"}
43
+ @reporter_meta = {:plugin_channel => @id, :type => "plugin_channel"}
44
44
  end
45
45
 
46
46
  def event_subscribe &block
@@ -23,14 +23,14 @@ module FFWD::Processor
23
23
  class Setup
24
24
  attr_reader :name
25
25
 
26
- def initialize name, klass, options
26
+ def initialize name, factory, options
27
27
  @name = name
28
- @klass = klass
28
+ @factory = factory
29
29
  @options = options
30
30
  end
31
31
 
32
32
  def setup emitter
33
- @klass.new emitter, @options
33
+ @factory.new emitter, @options
34
34
  end
35
35
  end
36
36
 
@@ -91,6 +91,13 @@ module FFWD::Processor
91
91
 
92
92
  # setup hash of processor setup classes.
93
93
  def self.load_processors config
94
- registry.map{|name, klass| Setup.new name, klass, config[name] || {}}
94
+ registry.map do |name, klass|
95
+ unless klass.respond_to? :prepare
96
+ raise "Processor #{klass} does not have a 'prepare' method"
97
+ end
98
+
99
+ config = klass.prepare Hash[config[name] || {}]
100
+ Setup.new name, klass, config
101
+ end
95
102
  end
96
103
  end
@@ -34,16 +34,26 @@ module FFWD::Processor
34
34
  :keys => [:dropped, :received]
35
35
  )
36
36
 
37
- def initialize emitter, opts={}
37
+ def self.prepare config
38
+ config[:cache_limit] ||= 1000
39
+ config[:timeout] ||= 300
40
+ config[:window] ||= 30
41
+ config
42
+ end
43
+
44
+ def initialize emitter, config={}
38
45
  @emitter = emitter
39
- @cache_limit = opts[:cache_limit] || 1000
40
- @timeout = opts[:timeout] || 300
41
- @period = opts[:period] || 30
46
+
47
+ @cache_limit = config[:cache_limit]
48
+ @timeout = config[:timeout]
49
+ @window = config[:window]
50
+
42
51
  @cache = {}
43
52
  @timer = nil
44
53
 
45
54
  starting do
46
- log.info "Starting count processor on a window of #{@period}s"
55
+ log.info "Started"
56
+ log.info " config: #{config.inspect}"
47
57
  end
48
58
 
49
59
  stopping do
@@ -81,7 +91,7 @@ module FFWD::Processor
81
91
 
82
92
  log.debug "Starting timer"
83
93
 
84
- @timer = EM::Timer.new(@period) do
94
+ @timer = EM::Timer.new(@window) do
85
95
  @timer = nil
86
96
  digest! Time.now
87
97
  end
@@ -49,13 +49,23 @@ module FFWD::Processor
49
49
  DEFAULT_MISSING = 0
50
50
 
51
51
  DEFAULT_PERCENTILES = {
52
- :p50 => {:percentage => 0.50, :info => "50th"},
53
- :p75 => {:percentage => 0.75, :info => "75th"},
54
- :p95 => {:percentage => 0.95, :info => "95th"},
55
- :p99 => {:percentage => 0.99, :info => "99th"},
56
- :p999 => {:percentage => 0.999, :info => "99.9th"},
52
+ :p50 => {:q => 0.50, :info => "50th"},
53
+ :p75 => {:q => 0.75, :info => "75th"},
54
+ :p95 => {:q => 0.95, :info => "95th"},
55
+ :p99 => {:q => 0.99, :info => "99th"},
56
+ :p999 => {:q => 0.999, :info => "99.9th"},
57
57
  }
58
58
 
59
+ def self.prepare config
60
+ config[:window] ||= 10
61
+ config[:cache_limit] ||= 1000
62
+ config[:bucket_limit] ||= 10000
63
+ config[:precision] ||= 3
64
+ config[:missing] ||= DEFAULT_MISSING
65
+ config[:percentiles] ||= DEFAULT_PERCENTILES
66
+ config
67
+ end
68
+
59
69
  #
60
70
  # Options:
61
71
  #
@@ -66,22 +76,23 @@ module FFWD::Processor
66
76
  # :precision - Precision of emitted metrics.
67
77
  # :percentiles - Configuration hash of percentile metrics.
68
78
  # Structure:
69
- # {:p10 => {:info => "Some description", :percentage => 0.1}, ...}
70
- def initialize emitter, opts={}
79
+ # {:p10 => {:info => "Some description", :q => 0.1}, ...}
80
+ def initialize emitter, config={}
71
81
  @emitter = emitter
72
82
 
73
- @window = opts[:window] || 10
74
- @cache_limit = opts[:cache_limit] || 1000
75
- @bucket_limit = opts[:bucket_limit] || 10000
76
- @precision = opts[:precision] || 3
77
- @missing = opts[:missing] || DEFAULT_MISSING
78
- @percentiles = opts[:percentiles] || DEFAULT_PERCENTILES
83
+ @window = config[:window]
84
+ @cache_limit = config[:cache_limit]
85
+ @bucket_limit = config[:bucket_limit]
86
+ @precision = config[:precision]
87
+ @missing = config[:missing]
88
+ @percentiles = config[:percentiles]
79
89
 
80
90
  # Dropped values that would have gone into a bucket.
81
91
  @cache = {}
82
92
 
83
93
  starting do
84
- log.info "Starting histogram processor on a window of #{@window}s"
94
+ log.info "Started"
95
+ log.info " config: #{config.inspect}"
85
96
  end
86
97
 
87
98
  stopping do
@@ -130,7 +141,7 @@ module FFWD::Processor
130
141
  map = {}
131
142
 
132
143
  @percentiles.each do |k, v|
133
- index = (total * v[:percentage]).ceil - 1
144
+ index = (total * v[:q]).ceil - 1
134
145
 
135
146
  if (c = map[index]).nil?
136
147
  info = "#{v[:info]} percentile"
@@ -44,13 +44,22 @@ module FFWD::Processor
44
44
  # :ttl - Allowed age of items in cache in seconds.
45
45
  # If this is nil, items will never expire, so old elements will not be
46
46
  # expunged until data type is restarted.
47
- def initialize emitter, opts={}
47
+ def self.prepare config={}
48
+ config[:precision] ||= 3
49
+ config[:cache_limit] ||= 10000
50
+ config[:min_age] ||= 0.5
51
+ config[:ttl] ||= 600
52
+ config
53
+ end
54
+
55
+ def initialize emitter, config={}
48
56
  @emitter = emitter
49
57
 
50
- @precision = opts[:precision] || 3
51
- @limit = opts[:cache_limit] || 10000
52
- @min_age = opts[:min_age] || 0.5
53
- @ttl = opts[:ttl] || 600
58
+ @precision = config[:precision]
59
+ @limit = config[:cache_limit]
60
+ @min_age = config[:min_age]
61
+ @ttl = config[:ttl]
62
+
54
63
  # keep a reference to the expire cache to prevent having to allocate it
55
64
  # all the time.
56
65
  @expire = Hash.new
@@ -58,8 +67,9 @@ module FFWD::Processor
58
67
  @cache = Hash.new
59
68
 
60
69
  starting do
61
- log.info "Starting rate processor (ttl: #{@ttl})"
62
70
  @timer = EM.add_periodic_timer(@ttl){expire!} unless @ttl.nil?
71
+ log.info "Started"
72
+ log.info " config: #{config.inspect}"
63
73
  end
64
74
 
65
75
  stopping do
@@ -13,9 +13,7 @@
13
13
  # License for the specific language governing permissions and limitations under
14
14
  # the License.
15
15
 
16
- require 'eventmachine'
17
-
18
- require_relative '../tunnel'
16
+ require_relative '../tunnel/tcp'
19
17
 
20
18
  require_relative 'tcp/bind'
21
19
  require_relative 'tcp/plain_connect'
@@ -27,100 +25,61 @@ module FFWD::TCP
27
25
  :tcp
28
26
  end
29
27
 
30
- # default amount of bytes that the outbound connection will allow in its
31
- # application-level buffer.
32
- DEFAULT_OUTBOUND_LIMIT = 2 ** 20
33
- # default flush period, if non-zero will cause the connection to be buffered.
34
- DEFAULT_FLUSH_PERIOD = 10
35
- # defaults for buffered connections.
36
- # maximum amount of events to buffer up.
37
- DEFAULT_EVENT_LIMIT = 1000
38
- # maximum amount of metrics to buffer up.
39
- DEFAULT_METRIC_LIMIT = 10000
40
- # percent of maximum events/metrics which will cause a flush.
41
- DEFAULT_FLUSH_LIMIT = 0.8
42
- # Default initial timeout when binding fails.
43
- DEFAULT_REBIND_TIMEOUT = 10
28
+ class SetupOutput
29
+ attr_reader :config
44
30
 
45
- # Establish an outbound tcp connection.
46
- #
47
- # opts - Option hash.
48
- # Expects the following keys.
49
- # :host - The host to connect to.
50
- # :port - The port to connect to.
51
- # :outbound_limit - The amount of bytes that are allowed to be pending in
52
- # the application level buffer for the connection to be considered
53
- # 'writable'.
54
- # :flush_period - The period in which outgoing data is buffered. If this
55
- # is 0 no buffering will occur.
56
- # Reads the following keys if the connection is buffered.
57
- # :event_limit - The maximum amount of events the connection is allowed
58
- # to buffer up.
59
- # :metric_limimt - The maximum amount of metrics the connection is
60
- # allowed to buffer up.
61
- # :flush_limit - A percentage (0.0 - 1.0) indicating 'the percentage of
62
- # events/metrics' that is required for a flush to be forced.
63
- # If this percentage is reached, the connection will attempt to forcibly
64
- # flush all buffered events and metrics prior to the end of the flushing
65
- # period.
66
- # core - The core interface associated with this connection.
67
- # log - The logger to use for this connection.
68
- # handler - An implementation of FFWD::Handler containing the connection
69
- # logic.
70
- # args - Arguments passed to the handler when a new instance is created.
71
- def self.connect opts, core, log, handler, *args
72
- raise "Missing required option :host" if (host = opts[:host]).nil?
73
- raise "Missing required option :port" if (port = opts[:port]).nil?
31
+ def initialize config, log, handler
32
+ @config = Hash[config]
33
+ @log = log
34
+ @handler = handler
74
35
 
75
- outbound_limit = opts[:outbound_limit] || DEFAULT_OUTBOUND_LIMIT
76
- flush_period = opts[:flush_period] || DEFAULT_FLUSH_PERIOD
36
+ @config = Connection.prepare @config
77
37
 
78
- connection = Connection.new log, host, port, handler, args, outbound_limit
38
+ if @config[:flush_period] == 0
39
+ @config = PlainConnect.prepare @config
40
+ @type = PlainConnect
41
+ else
42
+ @config = FlushingConnect.prepare @config
43
+ @type = FlushingConnect
44
+ end
45
+ end
79
46
 
80
- if flush_period == 0
81
- PlainConnect.new core, log, connection
82
- else
83
- event_limit = opts[:event_limit] || DEFAULT_EVENT_LIMIT
84
- metric_limit = opts[:metric_limit] || DEFAULT_METRIC_LIMIT
85
- flush_limit = opts[:flush_limit] || DEFAULT_FLUSH_LIMIT
47
+ def connect core
48
+ raise "Missing required option :host" if (host = @config[:host]).nil?
49
+ raise "Missing required option :port" if (port = @config[:port]).nil?
86
50
 
87
- FlushingConnect.new(
88
- core, log, connection,
89
- flush_period, event_limit, metric_limit, flush_limit
90
- )
51
+ c = Connection.new @log, host, port, @handler, @config
52
+ @type.new core, @log, c, @config
91
53
  end
92
54
  end
93
55
 
94
- # Bind and listen for a TCP connection.
95
- #
96
- # opts - Option hash.
97
- # :host - The host to bind to.
98
- # :port - The port to bind to.
99
- # :rebind_timeout - The initial timeout to use when rebinding the
100
- # connection.
101
- # core - The core interface associated with this connection.
102
- # log - The logger to use for this connection.
103
- # connection - An implementation of FFWD::Connection containing the
104
- # connection logic.
105
- # args - Arguments passed to the connection when a new instance is created.
106
- def self.bind opts, core, log, connection, *args
107
- raise "Missing required option :host" if (host = opts[:host]).nil?
108
- raise "Missing required option :port" if (port = opts[:port]).nil?
109
- rebind_timeout = opts[:rebind_timeout] || DEFAULT_REBIND_TIMEOUT
110
- Bind.new core, log, host, port, connection, args, rebind_timeout
56
+ def self.connect config, log, handler
57
+ SetupOutput.new config, log, handler
58
+ end
59
+
60
+ class SetupInput
61
+ attr_reader :config
62
+
63
+ def initialize config, log, connection
64
+ @config = Hash[config]
65
+ @log = log
66
+ @connection = connection
67
+ end
68
+
69
+ def bind core
70
+ raise "Missing required option :host" if (host = @config[:host]).nil?
71
+ raise "Missing required option :port" if (port = @config[:port]).nil?
72
+ @config = Bind.prepare Hash[@config]
73
+ Bind.new core, @log, host, port, @connection, @config
74
+ end
75
+
76
+ def tunnel core, plugin
77
+ raise "Missing required option :port" if (port = @config[:port]).nil?
78
+ FFWD::Tunnel::TCP.new port, core, plugin, @log, @connection
79
+ end
111
80
  end
112
81
 
113
- # Set up a TCP tunnel.
114
- #
115
- # opts - Option hash.
116
- # :port - The port to bind to on the remote side.
117
- # core - The core interface associated with this connection.
118
- # log - The logger to use for this connection.
119
- # connection - An implementation of FFWD::Connection containing the
120
- # connection logic.
121
- # args - Arguments passed to the connection when a new instance is created.
122
- def self.tunnel opts, core, plugin, log, connection, *args
123
- raise "Missing required option :port" if (port = opts[:port]).nil?
124
- FFWD.tunnel self.family, port, core, plugin, log, connection, args
82
+ def self.bind config, log, connection
83
+ SetupInput.new config, log, connection
125
84
  end
126
85
  end
@@ -22,42 +22,54 @@ module FFWD::TCP
22
22
  class Bind
23
23
  include FFWD::Reporter
24
24
 
25
+ # Default initial timeout when binding fails.
26
+ DEFAULT_REBIND_TIMEOUT = 10
27
+
28
+ def self.prepare opts
29
+ opts[:rebind_timeout] ||= DEFAULT_REBIND_TIMEOUT
30
+ opts
31
+ end
32
+
25
33
  setup_reporter :keys => [
26
34
  :received_events, :received_metrics,
27
35
  :failed_events, :failed_metrics
28
36
  ]
29
37
 
30
- attr_reader :reporter_meta
38
+ attr_reader :log, :reporter_meta
31
39
 
32
- def initialize core, log, host, port, connection, args, rebind_timeout
40
+ def initialize core, log, host, port, connection, config
41
+ @log = log
33
42
  @peer = "#{host}:#{port}"
34
43
  @reporter_meta = {
35
44
  :type => connection.plugin_type,
36
45
  :listen => @peer, :family => 'tcp'
37
46
  }
38
47
 
39
- @sig = nil
48
+ @server = nil
40
49
 
41
50
  info = "tcp://#{@peer}"
51
+ rebind_timeout = config[:rebind_timeout]
42
52
 
43
53
  r = FFWD.retry :timeout => rebind_timeout do |a|
44
- @sig = EM.start_server host, port, connection, self, core, *args
54
+ @server = EM.start_server host, port, connection, self, core, config
55
+
45
56
  log.info "Bind on #{info} (attempt #{a})"
57
+ log.info " config: #{config.inspect}"
46
58
  end
47
59
 
48
60
  r.error do |a, t, e|
49
- log.error "Failed to bind #{info} (attempt #{a}), retry in #{t}s", e
61
+ log.warning "Bind on #{info} failed, retry ##{a} in #{t}s: #{e}"
50
62
  end
51
63
 
52
64
  r.depend_on core
53
65
 
54
66
  core.stopping do
55
- log.info "Unbinding #{info}"
56
-
57
- if @sig
58
- EM.stop_server @sig
59
- @sig = nil
67
+ if @server
68
+ EM.stop_server @server
69
+ @server = nil
60
70
  end
71
+
72
+ log.info "Unbound #{info}"
61
73
  end
62
74
  end
63
75
  end