ffwd 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/bin/ffwd +9 -0
  3. data/bin/fwc +15 -0
  4. data/lib/em/all.rb +68 -0
  5. data/lib/ffwd.rb +250 -0
  6. data/lib/ffwd/channel.rb +62 -0
  7. data/lib/ffwd/circular_buffer.rb +78 -0
  8. data/lib/ffwd/connection.rb +40 -0
  9. data/lib/ffwd/core.rb +173 -0
  10. data/lib/ffwd/core/emitter.rb +38 -0
  11. data/lib/ffwd/core/interface.rb +47 -0
  12. data/lib/ffwd/core/processor.rb +92 -0
  13. data/lib/ffwd/core/reporter.rb +32 -0
  14. data/lib/ffwd/debug.rb +76 -0
  15. data/lib/ffwd/debug/connection.rb +48 -0
  16. data/lib/ffwd/debug/monitor_session.rb +71 -0
  17. data/lib/ffwd/debug/tcp.rb +82 -0
  18. data/lib/ffwd/event.rb +65 -0
  19. data/lib/ffwd/event_emitter.rb +57 -0
  20. data/lib/ffwd/handler.rb +43 -0
  21. data/lib/ffwd/lifecycle.rb +92 -0
  22. data/lib/ffwd/logging.rb +139 -0
  23. data/lib/ffwd/metric.rb +55 -0
  24. data/lib/ffwd/metric_emitter.rb +50 -0
  25. data/lib/ffwd/plugin.rb +149 -0
  26. data/lib/ffwd/plugin/json_line.rb +47 -0
  27. data/lib/ffwd/plugin/json_line/connection.rb +118 -0
  28. data/lib/ffwd/plugin/log.rb +35 -0
  29. data/lib/ffwd/plugin/log/writer.rb +42 -0
  30. data/lib/ffwd/plugin_channel.rb +64 -0
  31. data/lib/ffwd/plugin_loader.rb +121 -0
  32. data/lib/ffwd/processor.rb +96 -0
  33. data/lib/ffwd/processor/count.rb +109 -0
  34. data/lib/ffwd/processor/histogram.rb +200 -0
  35. data/lib/ffwd/processor/rate.rb +116 -0
  36. data/lib/ffwd/producing_client.rb +181 -0
  37. data/lib/ffwd/protocol.rb +28 -0
  38. data/lib/ffwd/protocol/tcp.rb +126 -0
  39. data/lib/ffwd/protocol/tcp/bind.rb +64 -0
  40. data/lib/ffwd/protocol/tcp/connection.rb +107 -0
  41. data/lib/ffwd/protocol/tcp/flushing_connect.rb +135 -0
  42. data/lib/ffwd/protocol/tcp/plain_connect.rb +74 -0
  43. data/lib/ffwd/protocol/udp.rb +48 -0
  44. data/lib/ffwd/protocol/udp/bind.rb +64 -0
  45. data/lib/ffwd/protocol/udp/connect.rb +110 -0
  46. data/lib/ffwd/reporter.rb +65 -0
  47. data/lib/ffwd/retrier.rb +72 -0
  48. data/lib/ffwd/schema.rb +92 -0
  49. data/lib/ffwd/schema/default.rb +36 -0
  50. data/lib/ffwd/schema/spotify100.rb +58 -0
  51. data/lib/ffwd/statistics.rb +29 -0
  52. data/lib/ffwd/statistics/collector.rb +99 -0
  53. data/lib/ffwd/statistics/system_statistics.rb +255 -0
  54. data/lib/ffwd/tunnel.rb +27 -0
  55. data/lib/ffwd/tunnel/plugin.rb +47 -0
  56. data/lib/ffwd/tunnel/tcp.rb +60 -0
  57. data/lib/ffwd/tunnel/udp.rb +61 -0
  58. data/lib/ffwd/utils.rb +46 -0
  59. data/lib/ffwd/version.rb +18 -0
  60. data/lib/fwc.rb +206 -0
  61. metadata +163 -0
@@ -0,0 +1,200 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require 'ffwd/event'
17
+ require 'ffwd/logging'
18
+ require 'ffwd/processor'
19
+ require 'ffwd/reporter'
20
+ require 'ffwd/utils'
21
+
22
+ module FFWD::Processor
23
+ #
24
+ # Implements histogram statistics over a tumbling time window.
25
+ #
26
+ # Histogram received metrics continiuosly and regularly flushes out the
27
+ # following statistics.
28
+ #
29
+ # <key>.min - Min value collected.
30
+ # <key>.max - Max value collected.
31
+ # <key>.mean - Mean value collected.
32
+ # <key>.p50 - The 50th percentile value collected.
33
+ # <key>.p75 - The 75th percentile value collected.
34
+ # <key>.p95 - The 95th percentile value collected.
35
+ # <key>.p99 - The 99th percentile value collected.
36
+ # <key>.p999 - The 99.9th percentile value collected.
37
+ #
38
+ class HistogramProcessor
39
+ include FFWD::Processor
40
+ include FFWD::Logging
41
+ include FFWD::Reporter
42
+
43
+ register_processor "histogram"
44
+ setup_reporter(
45
+ :reporter_meta => {:processor => "histogram"},
46
+ :keys => [:dropped, :bucket_dropped, :received]
47
+ )
48
+
49
+ DEFAULT_MISSING = 0
50
+
51
+ DEFAULT_PERCENTILES = {
52
+ :p50 => {:percentage => 0.50, :info => "50th"},
53
+ :p75 => {:percentage => 0.75, :info => "75th"},
54
+ :p95 => {:percentage => 0.95, :info => "95th"},
55
+ :p99 => {:percentage => 0.99, :info => "99th"},
56
+ :p999 => {:percentage => 0.999, :info => "99.9th"},
57
+ }
58
+
59
+ #
60
+ # Options:
61
+ #
62
+ # :window - Define at what period the cache is flushed and generates
63
+ # metrics.
64
+ # :cache_limit - Limit the amount of cache entries (by key).
65
+ # :bucket_limit - Limit the amount of limits for each cache entry.
66
+ # :precision - Precision of emitted metrics.
67
+ # :percentiles - Configuration hash of percentile metrics.
68
+ # Structure:
69
+ # {:p10 => {:info => "Some description", :percentage => 0.1}, ...}
70
+ def initialize emitter, opts={}
71
+ @emitter = emitter
72
+
73
+ @window = opts[:window] || 10
74
+ @cache_limit = opts[:cache_limit] || 1000
75
+ @bucket_limit = opts[:bucket_limit] || 10000
76
+ @precision = opts[:precision] || 3
77
+ @missing = opts[:missing] || DEFAULT_MISSING
78
+ @percentiles = opts[:percentiles] || DEFAULT_PERCENTILES
79
+
80
+ # Dropped values that would have gone into a bucket.
81
+ @cache = {}
82
+
83
+ starting do
84
+ log.info "Starting histogram processor on a window of #{@window}s"
85
+ end
86
+
87
+ stopping do
88
+ log.info "Stopping histogram processor"
89
+ @timer.cancel if @timer
90
+ @timer = nil
91
+ digest!
92
+ end
93
+ end
94
+
95
+ def check_timer
96
+ return if @timer
97
+
98
+ log.debug "Starting timer"
99
+
100
+ @timer = EM::Timer.new(@window) do
101
+ @timer = nil
102
+ digest!
103
+ end
104
+ end
105
+
106
+ # Digest the cache.
107
+ def digest!
108
+ if @cache.empty?
109
+ return
110
+ end
111
+
112
+ ms = FFWD.timing do
113
+ @cache.each do |key, bucket|
114
+ calculate(bucket) do |p, info, value|
115
+ @emitter.metric.emit(
116
+ :key => "#{key}.#{p}", :source => key,
117
+ :value => value, :description => "#{info} of #{key}")
118
+ end
119
+ end
120
+
121
+ @cache = {}
122
+ end
123
+
124
+ log.debug "Digest took #{ms}ms"
125
+ end
126
+
127
+ def calculate bucket
128
+ total = bucket.size
129
+
130
+ map = {}
131
+
132
+ @percentiles.each do |k, v|
133
+ index = (total * v[:percentage]).ceil - 1
134
+
135
+ if (c = map[index]).nil?
136
+ info = "#{v[:info]} percentile"
137
+ c = map[index] = {:info => info, :values => []}
138
+ end
139
+
140
+ c[:values] << {:name => k, :value => nil}
141
+ end
142
+
143
+ max = nil
144
+ min = nil
145
+ sum = 0.0
146
+ mean = nil
147
+
148
+ bucket.sort.each_with_index do |t, index|
149
+ max = t if max.nil? or t > max
150
+ min = t if min.nil? or t < min
151
+ sum += t
152
+
153
+ unless (c = map[index]).nil?
154
+ c[:values].each{|d| d[:value] = t}
155
+ end
156
+ end
157
+
158
+ mean = sum / total
159
+
160
+ unless @precision.nil?
161
+ max = max.round(@precision)
162
+ min = min.round(@precision)
163
+ sum = sum.round(@precision)
164
+ mean = mean.round(@precision)
165
+
166
+ map.each do |index, c|
167
+ c[:values].each{|d| d[:value] = d[:value].round(@precision)}
168
+ end
169
+ end
170
+
171
+ yield "max", "Max", max
172
+ yield "min", "Min", min
173
+ yield "sum", "Sum", sum
174
+ yield "mean", "Mean", mean
175
+
176
+ map.each do |index, c|
177
+ c[:values].each do |d|
178
+ yield d[:name], c[:info], d[:value]
179
+ end
180
+ end
181
+ end
182
+
183
+ def process m
184
+ key = m[:key]
185
+ value = m[:value] || @missing
186
+
187
+ if (bucket = @cache[key]).nil?
188
+ return increment :dropped if @cache.size >= @cache_limit
189
+ @cache[key] = bucket = []
190
+ end
191
+
192
+ return increment :bucket_dropped if bucket.size >= @bucket_limit
193
+ return increment :dropped if stopped?
194
+ increment :received
195
+
196
+ bucket << value
197
+ check_timer
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,116 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require 'ffwd/processor'
17
+ require 'ffwd/logging'
18
+ require 'ffwd/reporter'
19
+
20
+ module FFWD::Processor
21
+ #
22
+ # Implements rate statistics (similar to derive in collectd).
23
+ #
24
+ class RateProcessor
25
+ include FFWD::Logging
26
+ include FFWD::Processor
27
+ include FFWD::Reporter
28
+
29
+ register_processor "rate"
30
+ setup_reporter(
31
+ :reporter_meta => {:processor => "rate"},
32
+ :keys => [:dropped, :expired, :received]
33
+ )
34
+
35
+ # Options:
36
+ #
37
+ # :precision - The precision to round to for emitted values.
38
+ # :cache_limit - Maxiumum amount of items allowed in the cache.
39
+ # :min_age - Minimum age that an item has to have in the cache to calculate
40
+ # rates.
41
+ # This exists to prevent too frequent updates to the cache which would
42
+ # yield very static or jumpy rates.
43
+ # Can be set to null to disable.
44
+ # :ttl - Allowed age of items in cache in seconds.
45
+ # If this is nil, items will never expire, so old elements will not be
46
+ # expunged until data type is restarted.
47
+ def initialize emitter, opts={}
48
+ @emitter = emitter
49
+
50
+ @precision = opts[:precision] || 3
51
+ @limit = opts[:cache_limit] || 10000
52
+ @min_age = opts[:min_age] || 0.5
53
+ @ttl = opts[:ttl] || 600
54
+ # keep a reference to the expire cache to prevent having to allocate it
55
+ # all the time.
56
+ @expire = Hash.new
57
+ # Cache of active events.
58
+ @cache = Hash.new
59
+
60
+ starting do
61
+ log.info "Starting rate processor (ttl: #{@ttl})"
62
+ @timer = EM.add_periodic_timer(@ttl){expire!} unless @ttl.nil?
63
+ end
64
+
65
+ stopping do
66
+ log.info "Stopping rate processor"
67
+ @timer.cancel if @timer
68
+ end
69
+ end
70
+
71
+ # Expire cached events that are too old.
72
+ def expire!
73
+ now = Time.new
74
+
75
+ @cache.each do |key, value|
76
+ diff = (now - value[:time])
77
+ next if diff < @ttl
78
+ @expire[key] = value
79
+ end
80
+
81
+ unless @expire.empty?
82
+ increment :expired, @cache.size - @expire.size
83
+ @cache = @expire
84
+ @expire = Hash.new
85
+ end
86
+ end
87
+
88
+ def process msg
89
+ key = msg[:key]
90
+ time = msg[:time]
91
+ value = msg[:value] || 0
92
+
93
+ unless (prev = @cache[key]).nil?
94
+ prev_time = prev[:time]
95
+ prev_value = prev[:value]
96
+
97
+ diff = (time - prev_time)
98
+
99
+ valid = @ttl.nil? or diff < @ttl
100
+ aged = @min_age.nil? or diff > @min_age
101
+
102
+ if diff > 0 and valid and aged
103
+ rate = ((value - prev_value) / diff)
104
+ rate = rate.round(@precision) unless @precision.nil?
105
+ @emitter.metric.emit(
106
+ :key => "#{key}.rate", :source => key, :value => rate)
107
+ end
108
+ else
109
+ return increment :dropped if @cache.size >= @limit
110
+ end
111
+
112
+ increment :received
113
+ @cache[key] = {:key => key, :time => time, :value => value}
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,181 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative 'lifecycle'
17
+ require_relative 'reporter'
18
+ require_relative 'logging'
19
+
20
+ module FFWD
21
+ # A client implementation that delegates all work to other threads.
22
+ class ProducingClient
23
+ include FFWD::Reporter
24
+ include FFWD::Logging
25
+
26
+ class Producer
27
+ def setup; raise "not implemented: setup"; end
28
+ def teardown; raise "not implemented: teardown"; end
29
+ def produce events, metrics; raise "not implemented: produce"; end
30
+ end
31
+
32
+ setup_reporter :keys => [
33
+ # number of events/metrics that we attempted to dispatch but failed.
34
+ :failed_events, :failed_metrics,
35
+ # number of events/metrics that were dropped because the output buffers
36
+ # are full.
37
+ :dropped_events, :dropped_metrics,
38
+ # number of events/metrics successfully sent.
39
+ :sent_events, :sent_metrics,
40
+ # number of requests that take longer than the allowed period.
41
+ :slow_requests
42
+ ]
43
+
44
+ def reporter_meta
45
+ @reporter_meta ||= @producer.reporter_meta.merge(
46
+ :type => "producing_client_out")
47
+ end
48
+
49
+ def report!
50
+ super do |m|
51
+ yield m
52
+ end
53
+
54
+ return unless @producer_is_reporter
55
+
56
+ @producer.report! do |m|
57
+ yield m
58
+ end
59
+ end
60
+
61
+ def initialize channel, producer, flush_period, event_limit, metric_limit
62
+ @flush_period = flush_period
63
+ @event_limit = event_limit
64
+ @metric_limit = metric_limit
65
+
66
+ if @flush_period <= 0
67
+ raise "Invalid flush period: #{flush_period}"
68
+ end
69
+
70
+ @producer = producer
71
+ @producer_is_reporter = FFWD.is_reporter? producer
72
+
73
+ @events = []
74
+ @metrics = []
75
+
76
+ # Pending request.
77
+ @request = nil
78
+ @timer = nil
79
+
80
+ @subs = []
81
+
82
+ channel.starting do
83
+ @timer = EM::PeriodicTimer.new(@flush_period){safer_flush!}
84
+
85
+ @subs << channel.event_subscribe do |e|
86
+ if @events.size >= @event_limit
87
+ increment :dropped_events
88
+ next
89
+ end
90
+
91
+ @events << e
92
+ end
93
+
94
+ @subs << channel.metric_subscribe do |m|
95
+ if @metrics.size >= @metric_limit
96
+ increment :dropped_metrics
97
+ next
98
+ end
99
+
100
+ @metrics << m
101
+ end
102
+
103
+ @producer.setup
104
+ end
105
+
106
+ channel.stopping do
107
+ if @timer
108
+ @timer.cancel
109
+ @timer = nil
110
+ end
111
+
112
+ flush!
113
+
114
+ @subs.each(&:unsubscribe).clear
115
+
116
+ @metrics.clear
117
+ @events.clear
118
+
119
+ @producer.teardown
120
+ end
121
+ end
122
+
123
+ # Apply some heuristics to determine if we can 'ignore' the current flush
124
+ # to prevent loss of data.
125
+ #
126
+ # Checks that if a request is pending; we have not breached the limit of
127
+ # allowed events.
128
+ def safer_flush!
129
+ if @request
130
+ increment :slow_requests
131
+
132
+ ignore_flush = (
133
+ @events.size < @event_limit or
134
+ @metrics.size < @metric_limit)
135
+
136
+ return if ignore_flush
137
+ end
138
+
139
+ flush!
140
+ end
141
+
142
+ def flush!
143
+ if @request or not @request = @producer.produce(@events, @metrics)
144
+ increment :dropped_events, @events.size
145
+ increment :dropped_metrics, @metrics.size
146
+ return
147
+ end
148
+
149
+ @request.callback do
150
+ increment :sent_events, @events.size
151
+ increment :sent_metrics, @metrics.size
152
+ @request = nil
153
+ end
154
+
155
+ @request.errback do
156
+ increment :failed_events, @events.size
157
+ increment :failed_metrics, @metrics.size
158
+ @request = nil
159
+ end
160
+ rescue => e
161
+ increment :failed_events, @events.size
162
+ increment :failed_metrics, @metrics.size
163
+ log.error "Failed to produce", e
164
+ ensure
165
+ @events.clear
166
+ @metrics.clear
167
+ end
168
+ end
169
+
170
+ DEFAULT_FLUSH_PERIOD = 10
171
+ DEFAULT_EVENT_LIMIT = 10000
172
+ DEFAULT_METRIC_LIMIT = 10000
173
+ DEFAULT_FLUSH_SIZE = 1000
174
+
175
+ def self.producing_client channel, producer, opts
176
+ flush_period = opts[:flush_period] || DEFAULT_FLUSH_PERIOD
177
+ event_limit = opts[:event_limit] || DEFAULT_EVENT_LIMIT
178
+ metric_limit = opts[:metric_limit] || DEFAULT_METRIC_LIMIT
179
+ ProducingClient.new channel, producer, flush_period, event_limit, metric_limit
180
+ end
181
+ end