ffwd 0.3.7 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,15 +1,7 @@
1
1
  ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- NDAxMmI5NWEwMzNhOTJjNmUyZTJiNDc5MGNhM2I3NWY1NWIwMjI2MQ==
5
- data.tar.gz: !binary |-
6
- ZDFjMDgxMTcyNzdkYmVmYWUxMzZlNDdmZjY1ZjUxYWE4ZTA3YjQwYg==
2
+ SHA1:
3
+ metadata.gz: b64d17883b2270f2881852ba03b835186eb7b5b8
4
+ data.tar.gz: 73fb67085c33020061b6673a38bd253a672de0be
7
5
  SHA512:
8
- metadata.gz: !binary |-
9
- YjJjMTkyNzQ5YjMwMmI5YzZmNWM0ZTgzNDkyYzAxOTgxMzA3NmFjYzkzYzNj
10
- ZTNjMWI4ZjkyYzdlZmNlMWRiMDEyM2Q3YzMyMjJhNGJlNjYyMWI2NWQ0OTQ0
11
- ZjIxYWVhOTczOGQ5ODBiNTk3ZTcyZjgwMGRjNmI0MWEzN2VhYjQ=
12
- data.tar.gz: !binary |-
13
- MzllOTIxOGM5YjJiYjY1NDM4NDA2Zjc0NjBhMjhjY2EyMTAyMTFkY2QxNjg3
14
- YTc2MTdjOWZlYTU4MjZhZjFiNjNlN2M4NGI4NWY3M2M4ZDk5YzI3Y2E3Mjhi
15
- YzgwYjQ2YTcyNzI1ZGRmN2QwNzBhYTNiMWJhYzRiNWZjZTNlMTQ=
6
+ metadata.gz: 06dc1d8011388e4d1a754e70bf5c40ec45b4037cd8c94a3327dec0a26217c65b1607d54b79df50f08f47d8e4b72fe4fb0c10f007f3ebbc0f14b95e7a2fe3aa77
7
+ data.tar.gz: eed054fc3a895e30a5351b3e96e2ecbd679036961b60ced3521c4a7dd4f3c4c81e44b716693e6024b74aa316e8d1a1aaafae1f68465666aa0c189f95974eafcc
@@ -59,9 +59,9 @@ module FFWD
59
59
  end
60
60
  end
61
61
 
62
- def report!
62
+ def report!(diff)
63
63
  @reporters.each do |reporter|
64
- reporter.report! do |d|
64
+ reporter.report!(diff) do |d|
65
65
  yield d
66
66
  end
67
67
  end
@@ -21,9 +21,9 @@ module FFWD
21
21
  @reporters = reporters
22
22
  end
23
23
 
24
- def report!
24
+ def report!(diff)
25
25
  @reporters.each do |reporter|
26
- reporter.report! do |d|
26
+ reporter.report!(diff) do |d|
27
27
  yield d
28
28
  end
29
29
  end
@@ -21,11 +21,11 @@ module FFWD
21
21
  class FlushingOutput
22
22
  include FFWD::Reporter
23
23
 
24
- report_meta :direction => :out
24
+ report_meta :direction => :output
25
25
 
26
- report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
27
- report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
28
- report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
26
+ report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
27
+ report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
28
+ report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
29
29
 
30
30
  attr_reader :log, :reporter_meta
31
31
 
@@ -13,6 +13,8 @@
13
13
  # License for the specific language governing permissions and limitations under
14
14
  # the License.
15
15
 
16
+ require 'json'
17
+
16
18
  require_relative '../../logging'
17
19
 
18
20
  module FFWD::Plugin::Log
@@ -28,11 +30,11 @@ module FFWD::Plugin::Log
28
30
  log.info "Started (prefix: '#{@p}')"
29
31
 
30
32
  subs << core.output.event_subscribe do |e|
31
- log.info "Event: #{@p}#{e.to_h}"
33
+ log.info "Event: #{@p}#{JSON.dump(e.to_h)}"
32
34
  end
33
35
 
34
36
  subs << core.output.metric_subscribe do |m|
35
- log.info "Metric: #{@p}#{m.to_h}"
37
+ log.info "Metric: #{@p}#{JSON.dump(m.to_h)}"
36
38
  end
37
39
  end
38
40
 
@@ -26,9 +26,9 @@ module FFWD
26
26
  include FFWD::Reporter
27
27
  include FFWD::Logging
28
28
 
29
- report_meta :component => :plugin_channel
30
- report_key :metrics, :meta => {:what => :metrics, :unit => :metric}
31
- report_key :events, :meta => {:what => :events, :unit => :event}
29
+ report_meta :component => :channel
30
+ report_key :metrics, :meta => {:what => "seen-metrics", :unit => :metric}
31
+ report_key :events, :meta => {:what => "seen-events", :unit => :event}
32
32
 
33
33
  attr_reader :id, :events, :metrics, :reporter_meta
34
34
 
@@ -42,7 +42,7 @@ module FFWD
42
42
  @id = id
43
43
  @events = events
44
44
  @metrics = metrics
45
- @reporter_meta = {:plugin_channel => @id, :type => "plugin_channel"}
45
+ @reporter_meta = {:channel => @id}
46
46
  end
47
47
 
48
48
  def event_subscribe &block
@@ -32,34 +32,34 @@ module FFWD
32
32
  report_meta :component => :producing_client
33
33
 
34
34
  # number of events/metrics that we attempted to dispatch but failed.
35
- report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
36
- report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
35
+ report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
36
+ report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
37
37
 
38
38
  # number of events/metrics that were dropped because the output buffers
39
39
  # are full.
40
- report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
41
- report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :event}
40
+ report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
41
+ report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :event}
42
42
 
43
43
  # number of events/metrics successfully sent.
44
- report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
45
- report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :event}
44
+ report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
45
+ report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :event}
46
46
 
47
47
  # number of requests that take longer than the allowed period.
48
- report_key :slow_requests, :meta => {:what => :slow_requests, :unit => :request}
48
+ report_key :slow_requests, :meta => {:what => "slow-requests", :unit => :request}
49
49
 
50
50
  def reporter_meta
51
51
  return {} if @producer_is_reporter
52
52
  @producer.class.reporter_meta.merge(@producer.reporter_meta)
53
53
  end
54
54
 
55
- def report!
56
- super do |m|
55
+ def report!(diff)
56
+ super(diff) do |m|
57
57
  yield m
58
58
  end
59
59
 
60
60
  return unless @producer_is_reporter
61
61
 
62
- @producer.report! do |m|
62
+ @producer.report!(diff) do |m|
63
63
  yield m
64
64
  end
65
65
  end
@@ -30,10 +30,10 @@ module FFWD::TCP
30
30
  opts
31
31
  end
32
32
 
33
- report_meta :protocol => :tcp, :direction => :in
33
+ report_meta :protocol => :tcp, :direction => :input
34
34
 
35
- report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
36
- report_key :received_events, :meta => {:what => :received_events, :unit => :event}
35
+ report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
36
+ report_key :received_events, :meta => {:what => "received-events", :unit => :event}
37
37
 
38
38
  report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
39
39
  report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
@@ -29,18 +29,18 @@ module FFWD::TCP
29
29
  # maximum amount of metrics to buffer up.
30
30
  DEFAULT_METRIC_LIMIT = 10000
31
31
 
32
- report_meta :protocol => :tcp, :direction => :out
32
+ report_meta :protocol => :tcp, :direction => :output
33
33
 
34
- report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
35
- report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
34
+ report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
35
+ report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
36
36
 
37
- report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
38
- report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
37
+ report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
38
+ report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
39
39
 
40
- report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
41
- report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
40
+ report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
41
+ report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
42
42
 
43
- report_key :forced_flush, :meta => {:what => :forced_flush, :unit => :flush}
43
+ report_key :forced_flush, :meta => {:what => "forced-flush", :unit => :flush}
44
44
 
45
45
  attr_reader :log
46
46
 
@@ -25,16 +25,16 @@ module FFWD::TCP
25
25
  opts
26
26
  end
27
27
 
28
- report_meta :protocol => :tcp, :direction => :out
28
+ report_meta :protocol => :tcp, :direction => :output
29
29
 
30
- report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
31
- report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
30
+ report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
31
+ report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
32
32
 
33
- report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
34
- report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
33
+ report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
34
+ report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
35
35
 
36
- report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
37
- report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
36
+ report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
37
+ report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
38
38
 
39
39
  def reporter_meta
40
40
  @c.reporter_meta
@@ -32,13 +32,13 @@ module FFWD::UDP
32
32
  opts
33
33
  end
34
34
 
35
- report_meta :protocol => :udp, :direction => :out
35
+ report_meta :protocol => :udp, :direction => :output
36
36
 
37
- report_key :received_events, :meta => {:what => :received_events, :unit => :event}
38
- report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
37
+ report_key :received_events, :meta => {:what => "received-events", :unit => :event}
38
+ report_key :received_metrics, :meta => {:what => "received-metrics", :unit => :metric}
39
39
 
40
- report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
41
- report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
40
+ report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
41
+ report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
42
42
 
43
43
  attr_reader :reporter_meta, :log, :config
44
44
 
@@ -29,13 +29,13 @@ module FFWD::UDP
29
29
 
30
30
  attr_reader :reporter_meta, :log, :config
31
31
 
32
- report_meta :protocol => :udp, :direction => :out
32
+ report_meta :protocol => :udp, :direction => :output
33
33
 
34
- report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
35
- report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
34
+ report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
35
+ report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
36
36
 
37
- report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
38
- report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
37
+ report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
38
+ report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
39
39
 
40
40
  def initialize core, log, host, port, handler, config
41
41
  @log = log
@@ -25,7 +25,9 @@ module FFWD::Reporter
25
25
  meta = meta.merge(instance.send(:reporter_meta))
26
26
  end
27
27
 
28
- return meta.merge(k[:meta])
28
+ meta = meta.merge(k[:meta])
29
+ meta[:unit] = "#{meta[:unit] || 'n'}/s"
30
+ return meta
29
31
  end
30
32
 
31
33
  module ClassMethods
@@ -71,15 +73,16 @@ module FFWD::Reporter
71
73
  reporter_data[n] += c
72
74
  end
73
75
 
74
- def report!
76
+ def report! diff
75
77
  self.class.reporter_keys.each do |key|
76
78
  k = key[:key]
77
79
  v = reporter_data[k]
78
80
  reporter_data[k] = 0
79
81
 
80
82
  meta = ((@_reporter_meta ||= {})[k] ||= FFWD::Reporter.build_meta(self, key))
83
+ rate = (v / diff).round(3)
81
84
 
82
- yield(:key => k, :value => v, :meta => meta)
85
+ yield(:key => k, :value => rate, :meta => meta)
83
86
  end
84
87
  end
85
88
  end
@@ -78,9 +78,12 @@ module FFWD::Statistics
78
78
  end
79
79
 
80
80
  def generate! last, now
81
+ diff = now - last
82
+ return if diff <= 0
83
+
81
84
  if @system
82
- @system.collect @channel do |key, value|
83
- attributes = FFWD.merge_hashes @attributes, {:what => key, :component => :system}
85
+ @system.collect @channel do |key, unit, value|
86
+ attributes = FFWD.merge_hashes @attributes, {:what => key, :unit => unit, :component => :system}
84
87
  @emitter.metric.emit(
85
88
  :key => @prefix, :value => value,
86
89
  :tags => @tags, :attributes => attributes)
@@ -88,7 +91,7 @@ module FFWD::Statistics
88
91
  end
89
92
 
90
93
  @reporters.each do |id, reporter|
91
- reporter.report! do |d|
94
+ reporter.report!(diff) do |d|
92
95
  attributes = FFWD.merge_hashes @attributes, d[:meta]
93
96
  @emitter.metric.emit(
94
97
  :key => @prefix, :value => d[:value],
@@ -99,21 +99,18 @@ module FFWD::Statistics
99
99
  end
100
100
 
101
101
  def collect channel
102
- memory_use = memory_usage
103
- cpu_use = cpu_usage
102
+ cpu = cpu_use
103
+ memory = memory_use
104
104
 
105
- cpu_use.each do |key, value|
106
- yield "statistics.cpu-#{key}", value
105
+ cpu.each do |key, value|
106
+ yield "cpu-#{key}", "%", value
107
107
  end
108
108
 
109
- memory_use.each do |key, value|
110
- yield "statistics.memory-#{key}", value
109
+ memory.each do |key, value|
110
+ yield "memory-#{key}", "B", value
111
111
  end
112
112
 
113
- channel << {
114
- :cpu => cpu_use,
115
- :memory => memory_use,
116
- }
113
+ channel << {:memory => memory}
117
114
  end
118
115
 
119
116
  def check
@@ -140,7 +137,7 @@ module FFWD::Statistics
140
137
  return true
141
138
  end
142
139
 
143
- def memory_usage
140
+ def memory_use
144
141
  result = {:resident => 0, :total => read_total_memory}
145
142
 
146
143
  read_smaps do |smap|
@@ -150,7 +147,7 @@ module FFWD::Statistics
150
147
  result
151
148
  end
152
149
 
153
- def cpu_usage
150
+ def cpu_use
154
151
  stat = read_pid_stat
155
152
 
156
153
  current = {
@@ -167,11 +164,15 @@ module FFWD::Statistics
167
164
  @cpu_prev = current
168
165
  end
169
166
 
170
- return {
171
- :system => current[:system] - prev[:system],
172
- :user => current[:user] - prev[:user],
173
- :total => current[:total] - prev[:total],
174
- }
167
+ total = current[:total] - prev[:total]
168
+ system = current[:system] - prev[:system]
169
+ user = current[:user] - prev[:user]
170
+
171
+ if total == 0
172
+ return {:user => 0, :system => 0}
173
+ end
174
+
175
+ {:user => (user / total).round(3), :system => (system / total).round(3)}
175
176
  end
176
177
 
177
178
  private
@@ -23,11 +23,11 @@ module FFWD::Tunnel
23
23
 
24
24
  report_meta :protocol => :tunnel_tcp
25
25
 
26
- report_key :received_events, :meta => {:what => :received_events, :unit => :event}
27
- report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
26
+ report_key :received_events, :meta => {:what => "received-events", :unit => :event}
27
+ report_key :received_metrics, :meta => {:what => "received-metrics", :unit => :metric}
28
28
 
29
- report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
30
- report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
29
+ report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
30
+ report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
31
31
 
32
32
  attr_reader :log
33
33
 
@@ -23,11 +23,11 @@ module FFWD::Tunnel
23
23
 
24
24
  report_meta :protocol => :tunnel_udp
25
25
 
26
- report_key :received_events, :meta => {:what => :received_events, :unit => :event}
27
- report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
26
+ report_key :received_events, :meta => {:what => "received-events", :unit => :event}
27
+ report_key :received_metrics, :meta => {:what => "received-metrics", :unit => :metric}
28
28
 
29
- report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
30
- report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
29
+ report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
30
+ report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
31
31
 
32
32
  attr_reader :log
33
33
 
@@ -14,5 +14,5 @@
14
14
  # the License.
15
15
 
16
16
  module FFWD
17
- VERSION = "0.3.7"
17
+ VERSION = "0.4.0"
18
18
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ffwd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.7
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - John-John Tedro
@@ -9,62 +9,62 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-01-17 00:00:00.000000000 Z
12
+ date: 2015-03-10 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: eventmachine
16
16
  requirement: !ruby/object:Gem::Requirement
17
17
  requirements:
18
- - - ! '>='
18
+ - - '='
19
19
  - !ruby/object:Gem::Version
20
- version: '0'
20
+ version: 1.0.4
21
21
  type: :runtime
22
22
  prerelease: false
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
- - - ! '>='
25
+ - - '='
26
26
  - !ruby/object:Gem::Version
27
- version: '0'
27
+ version: 1.0.4
28
28
  - !ruby/object:Gem::Dependency
29
29
  name: rspec
30
30
  requirement: !ruby/object:Gem::Requirement
31
31
  requirements:
32
- - - ! '>='
32
+ - - '>='
33
33
  - !ruby/object:Gem::Version
34
34
  version: '0'
35
35
  type: :development
36
36
  prerelease: false
37
37
  version_requirements: !ruby/object:Gem::Requirement
38
38
  requirements:
39
- - - ! '>='
39
+ - - '>='
40
40
  - !ruby/object:Gem::Version
41
41
  version: '0'
42
42
  - !ruby/object:Gem::Dependency
43
43
  name: rspec-mocks
44
44
  requirement: !ruby/object:Gem::Requirement
45
45
  requirements:
46
- - - ! '>='
46
+ - - '>='
47
47
  - !ruby/object:Gem::Version
48
48
  version: '0'
49
49
  type: :development
50
50
  prerelease: false
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
- - - ! '>='
53
+ - - '>='
54
54
  - !ruby/object:Gem::Version
55
55
  version: '0'
56
56
  - !ruby/object:Gem::Dependency
57
57
  name: autoversion
58
58
  requirement: !ruby/object:Gem::Requirement
59
59
  requirements:
60
- - - ! '>='
60
+ - - '>='
61
61
  - !ruby/object:Gem::Version
62
62
  version: '0'
63
63
  type: :development
64
64
  prerelease: false
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
- - - ! '>='
67
+ - - '>='
68
68
  - !ruby/object:Gem::Version
69
69
  version: '0'
70
70
  description:
@@ -81,18 +81,17 @@ files:
81
81
  - bin/fwc
82
82
  - lib/em/all.rb
83
83
  - lib/em/protocols/frame_object_protocol.rb
84
- - lib/ffwd.rb
85
84
  - lib/ffwd/channel.rb
86
85
  - lib/ffwd/connection.rb
87
- - lib/ffwd/core.rb
88
86
  - lib/ffwd/core/emitter.rb
89
87
  - lib/ffwd/core/interface.rb
90
88
  - lib/ffwd/core/processor.rb
91
89
  - lib/ffwd/core/reporter.rb
92
- - lib/ffwd/debug.rb
90
+ - lib/ffwd/core.rb
93
91
  - lib/ffwd/debug/connection.rb
94
92
  - lib/ffwd/debug/monitor_session.rb
95
93
  - lib/ffwd/debug/tcp.rb
94
+ - lib/ffwd/debug.rb
96
95
  - lib/ffwd/event.rb
97
96
  - lib/ffwd/event_emitter.rb
98
97
  - lib/ffwd/flushing_output.rb
@@ -102,32 +101,29 @@ files:
102
101
  - lib/ffwd/logging.rb
103
102
  - lib/ffwd/metric.rb
104
103
  - lib/ffwd/metric_emitter.rb
105
- - lib/ffwd/plugin.rb
106
- - lib/ffwd/plugin/json.rb
107
104
  - lib/ffwd/plugin/json/connection.rb
108
- - lib/ffwd/plugin/log.rb
105
+ - lib/ffwd/plugin/json.rb
109
106
  - lib/ffwd/plugin/log/writer.rb
107
+ - lib/ffwd/plugin/log.rb
108
+ - lib/ffwd/plugin.rb
110
109
  - lib/ffwd/plugin_channel.rb
111
110
  - lib/ffwd/plugin_loader.rb
112
111
  - lib/ffwd/processor.rb
113
- - lib/ffwd/processor/count.rb
114
- - lib/ffwd/processor/histogram.rb
115
- - lib/ffwd/processor/rate.rb
116
112
  - lib/ffwd/producing_client.rb
117
- - lib/ffwd/protocol.rb
118
- - lib/ffwd/protocol/tcp.rb
119
113
  - lib/ffwd/protocol/tcp/bind.rb
120
114
  - lib/ffwd/protocol/tcp/connection.rb
121
115
  - lib/ffwd/protocol/tcp/flushing_connect.rb
122
116
  - lib/ffwd/protocol/tcp/plain_connect.rb
123
- - lib/ffwd/protocol/udp.rb
117
+ - lib/ffwd/protocol/tcp.rb
124
118
  - lib/ffwd/protocol/udp/bind.rb
125
119
  - lib/ffwd/protocol/udp/connect.rb
120
+ - lib/ffwd/protocol/udp.rb
121
+ - lib/ffwd/protocol.rb
126
122
  - lib/ffwd/reporter.rb
127
123
  - lib/ffwd/retrier.rb
128
- - lib/ffwd/schema.rb
129
124
  - lib/ffwd/schema/default.rb
130
125
  - lib/ffwd/schema/spotify100.rb
126
+ - lib/ffwd/schema.rb
131
127
  - lib/ffwd/statistics/collector.rb
132
128
  - lib/ffwd/statistics/system_statistics.rb
133
129
  - lib/ffwd/test/protocol.rb
@@ -136,6 +132,7 @@ files:
136
132
  - lib/ffwd/tunnel/udp.rb
137
133
  - lib/ffwd/utils.rb
138
134
  - lib/ffwd/version.rb
135
+ - lib/ffwd.rb
139
136
  - lib/fwc.rb
140
137
  homepage: https://github.com/spotify/ffwd
141
138
  licenses:
@@ -147,17 +144,17 @@ require_paths:
147
144
  - lib
148
145
  required_ruby_version: !ruby/object:Gem::Requirement
149
146
  requirements:
150
- - - ! '>='
147
+ - - '>='
151
148
  - !ruby/object:Gem::Version
152
149
  version: '0'
153
150
  required_rubygems_version: !ruby/object:Gem::Requirement
154
151
  requirements:
155
- - - ! '>='
152
+ - - '>='
156
153
  - !ruby/object:Gem::Version
157
154
  version: '0'
158
155
  requirements: []
159
156
  rubyforge_project:
160
- rubygems_version: 2.2.2
157
+ rubygems_version: 2.0.14
161
158
  signing_key:
162
159
  specification_version: 4
163
160
  summary: Core framework for the FastForward Daemon.
@@ -1,130 +0,0 @@
1
- # $LICENSE
2
- # Copyright 2013-2014 Spotify AB. All rights reserved.
3
- #
4
- # The contents of this file are licensed under the Apache License, Version 2.0
5
- # (the "License"); you may not use this file except in compliance with the
6
- # License. You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
- # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
- # License for the specific language governing permissions and limitations under
14
- # the License.
15
-
16
- require_relative '../processor'
17
- require_relative '../logging'
18
- require_relative '../event'
19
- require_relative '../utils'
20
- require_relative '../reporter'
21
-
22
- module FFWD::Processor
23
- #
24
- # Implements counting statistics (similar to statsd).
25
- #
26
- class CountProcessor
27
- include FFWD::Logging
28
- include FFWD::Processor
29
- include FFWD::Reporter
30
-
31
- register_processor "count"
32
-
33
- report_meta :component => :processor, :processor => :count
34
- report_key :dropped, :meta => {:what => :count_dropped, :unit => :value}
35
- report_key :received, :meta => {:what => :count_received, :unit => :value}
36
-
37
- def self.prepare config
38
- config[:cache_limit] ||= 1000
39
- config[:timeout] ||= 300
40
- config[:window] ||= 30
41
- config
42
- end
43
-
44
- def initialize emitter, config={}
45
- @emitter = emitter
46
-
47
- @cache_limit = config[:cache_limit]
48
- @timeout = config[:timeout]
49
- @window = config[:window]
50
-
51
- @cache = {}
52
- @timer = nil
53
-
54
- starting do
55
- log.info "Started"
56
- log.info " config: #{config.inspect}"
57
- end
58
-
59
- stopping do
60
- log.info "Stopping count processor"
61
- @timer.cancel if @timer
62
- @timer = nil
63
- end
64
- end
65
-
66
- def flush_caches! now
67
- @cache.each do |cache_key, entry|
68
- key = entry[:key]
69
- count = entry[:count]
70
- last = entry[:last]
71
-
72
- # OK to modify the hash being iterated over.
73
- if now - last > @timeout
74
- @cache.delete cache_key
75
- next
76
- end
77
-
78
- attributes = entry[:attributes]
79
- tags = entry[:tags]
80
-
81
- entry[:count] = 0
82
-
83
- @emitter.metric.emit(
84
- :key => key, :value => count, :source => key,
85
- :attributes => attributes, :tags => tags)
86
- end
87
- end
88
-
89
- def check_timer
90
- return if @timer
91
-
92
- log.debug "Starting timer"
93
-
94
- @timer = EM::Timer.new(@window) do
95
- @timer = nil
96
- digest! Time.now
97
- end
98
- end
99
-
100
- def digest! now
101
- ms = FFWD.timing do
102
- flush_caches! now
103
- end
104
-
105
- log.debug "Digest took #{ms}ms"
106
- end
107
-
108
- def process m
109
- key = m[:key]
110
- value = m[:value]
111
-
112
- now = Time.now
113
-
114
- cache_key = [key, (m[:attributes] || {}), (m[:tags] || [])].hash
115
-
116
- unless (entry = @cache[cache_key])
117
- return increment :dropped if @cache.size >= @cache_limit
118
- entry = @cache[cache_key] = {
119
- :key => key,
120
- :count => 0, :last => now, :tags => m[:tags],
121
- :attributes => m[:attributes]}
122
- end
123
-
124
- increment :received
125
- entry[:count] += value
126
- entry[:last] = now
127
- check_timer
128
- end
129
- end
130
- end
@@ -1,212 +0,0 @@
1
- # $LICENSE
2
- # Copyright 2013-2014 Spotify AB. All rights reserved.
3
- #
4
- # The contents of this file are licensed under the Apache License, Version 2.0
5
- # (the "License"); you may not use this file except in compliance with the
6
- # License. You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
- # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
- # License for the specific language governing permissions and limitations under
14
- # the License.
15
-
16
- require 'ffwd/event'
17
- require 'ffwd/logging'
18
- require 'ffwd/processor'
19
- require 'ffwd/reporter'
20
- require 'ffwd/utils'
21
-
22
- module FFWD::Processor
23
- #
24
- # Implements histogram statistics over a tumbling time window.
25
- #
26
- # Histogram received metrics continiuosly and regularly flushes out the
27
- # following statistics.
28
- #
29
- # <key>.min - Min value collected.
30
- # <key>.max - Max value collected.
31
- # <key>.mean - Mean value collected.
32
- # <key>.p50 - The 50th percentile value collected.
33
- # <key>.p75 - The 75th percentile value collected.
34
- # <key>.p95 - The 95th percentile value collected.
35
- # <key>.p99 - The 99th percentile value collected.
36
- # <key>.p999 - The 99.9th percentile value collected.
37
- #
38
- class HistogramProcessor
39
- include FFWD::Processor
40
- include FFWD::Logging
41
- include FFWD::Reporter
42
-
43
- register_processor "histogram"
44
-
45
- report_meta :component => :processor, :processor => :histogram
46
- report_key :dropped, {:meta => {:what => :dropped, :unit => :value}}
47
- report_key :bucket_dropped, {:meta => {:what => :bucket_dropped, :unit => :value}}
48
- report_key :received, {:meta => {:what => :received, :unit => :value}}
49
-
50
- DEFAULT_MISSING = 0
51
-
52
- DEFAULT_PERCENTILES = {
53
- :p50 => {:q => 0.50, :info => "50th"},
54
- :p75 => {:q => 0.75, :info => "75th"},
55
- :p95 => {:q => 0.95, :info => "95th"},
56
- :p99 => {:q => 0.99, :info => "99th"},
57
- :p999 => {:q => 0.999, :info => "99.9th"},
58
- }
59
-
60
- def self.prepare config
61
- config[:window] ||= 10
62
- config[:cache_limit] ||= 1000
63
- config[:bucket_limit] ||= 10000
64
- config[:precision] ||= 3
65
- config[:missing] ||= DEFAULT_MISSING
66
- config[:percentiles] ||= DEFAULT_PERCENTILES
67
- config
68
- end
69
-
70
- #
71
- # Options:
72
- #
73
- # :window - Define at what period the cache is flushed and generates
74
- # metrics.
75
- # :cache_limit - Limit the amount of cache entries (by key).
76
- # :bucket_limit - Limit the amount of limits for each cache entry.
77
- # :precision - Precision of emitted metrics.
78
- # :percentiles - Configuration hash of percentile metrics.
79
- # Structure:
80
- # {:p10 => {:info => "Some description", :q => 0.1}, ...}
81
- def initialize emitter, config={}
82
- @emitter = emitter
83
-
84
- @window = config[:window]
85
- @cache_limit = config[:cache_limit]
86
- @bucket_limit = config[:bucket_limit]
87
- @precision = config[:precision]
88
- @missing = config[:missing]
89
- @percentiles = config[:percentiles]
90
-
91
- # Dropped values that would have gone into a bucket.
92
- @cache = {}
93
-
94
- starting do
95
- log.info "Started"
96
- log.info " config: #{config.inspect}"
97
- end
98
-
99
- stopping do
100
- log.info "Stopping histogram processor"
101
- @timer.cancel if @timer
102
- @timer = nil
103
- digest!
104
- end
105
- end
106
-
107
- def check_timer
108
- return if @timer
109
-
110
- log.debug "Starting timer"
111
-
112
- @timer = EM::Timer.new(@window) do
113
- @timer = nil
114
- digest!
115
- end
116
- end
117
-
118
- # Digest the cache.
119
- def digest!
120
- if @cache.empty?
121
- return
122
- end
123
-
124
- ms = FFWD.timing do
125
- @cache.each do |key, bucket|
126
- calculate(bucket) do |p, info, value|
127
- @emitter.metric.emit(
128
- :key => "#{key}.#{p}", :source => key,
129
- :value => value, :description => "#{info} of #{key}")
130
- end
131
- end
132
-
133
- @cache = {}
134
- end
135
-
136
- log.debug "Digest took #{ms}ms"
137
- end
138
-
139
- def calculate bucket
140
- total = bucket.size
141
-
142
- map = {}
143
-
144
- @percentiles.each do |k, v|
145
- index = (total * v[:q]).ceil - 1
146
-
147
- if (c = map[index]).nil?
148
- info = "#{v[:info]} percentile"
149
- c = map[index] = {:info => info, :values => []}
150
- end
151
-
152
- c[:values] << {:name => k, :value => nil}
153
- end
154
-
155
- max = nil
156
- min = nil
157
- sum = 0.0
158
- mean = nil
159
-
160
- bucket.sort.each_with_index do |t, index|
161
- max = t if max.nil? or t > max
162
- min = t if min.nil? or t < min
163
- sum += t
164
-
165
- unless (c = map[index]).nil?
166
- c[:values].each{|d| d[:value] = t}
167
- end
168
- end
169
-
170
- mean = sum / total
171
-
172
- unless @precision.nil?
173
- max = max.round(@precision)
174
- min = min.round(@precision)
175
- sum = sum.round(@precision)
176
- mean = mean.round(@precision)
177
-
178
- map.each do |index, c|
179
- c[:values].each{|d| d[:value] = d[:value].round(@precision)}
180
- end
181
- end
182
-
183
- yield "max", "Max", max
184
- yield "min", "Min", min
185
- yield "sum", "Sum", sum
186
- yield "mean", "Mean", mean
187
-
188
- map.each do |index, c|
189
- c[:values].each do |d|
190
- yield d[:name], c[:info], d[:value]
191
- end
192
- end
193
- end
194
-
195
- def process m
196
- key = m[:key]
197
- value = m[:value] || @missing
198
-
199
- if (bucket = @cache[key]).nil?
200
- return increment :dropped if @cache.size >= @cache_limit
201
- @cache[key] = bucket = []
202
- end
203
-
204
- return increment :bucket_dropped if bucket.size >= @bucket_limit
205
- return increment :dropped if stopped?
206
- increment :received
207
-
208
- bucket << value
209
- check_timer
210
- end
211
- end
212
- end
@@ -1,127 +0,0 @@
1
- # $LICENSE
2
- # Copyright 2013-2014 Spotify AB. All rights reserved.
3
- #
4
- # The contents of this file are licensed under the Apache License, Version 2.0
5
- # (the "License"); you may not use this file except in compliance with the
6
- # License. You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
- # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
- # License for the specific language governing permissions and limitations under
14
- # the License.
15
-
16
- require 'ffwd/processor'
17
- require 'ffwd/logging'
18
- require 'ffwd/reporter'
19
-
20
- module FFWD::Processor
21
- #
22
- # Implements rate statistics (similar to derive in collectd).
23
- #
24
- class RateProcessor
25
- include FFWD::Logging
26
- include FFWD::Processor
27
- include FFWD::Reporter
28
-
29
- register_processor "rate"
30
-
31
- report_meta :component => :processor, :processor => :rate
32
- report_key :dropped, {:meta => {:what => :dropped, :unit => :value}}
33
- report_key :expired, {:meta => {:what => :expired, :unit => :value}}
34
- report_key :received, {:meta => {:what => :received, :unit => :value}}
35
-
36
- # Options:
37
- #
38
- # :precision - The precision to round to for emitted values.
39
- # :cache_limit - Maxiumum amount of items allowed in the cache.
40
- # :min_age - Minimum age that an item has to have in the cache to calculate
41
- # rates.
42
- # This exists to prevent too frequent updates to the cache which would
43
- # yield very static or jumpy rates.
44
- # Can be set to null to disable.
45
- # :ttl - Allowed age of items in cache in seconds.
46
- # If this is nil, items will never expire, so old elements will not be
47
- # expunged until data type is restarted.
48
- def self.prepare config={}
49
- config[:precision] ||= 3
50
- config[:cache_limit] ||= 10000
51
- config[:min_age] ||= 0.5
52
- config[:ttl] ||= 600
53
- config
54
- end
55
-
56
- def initialize emitter, config={}
57
- @emitter = emitter
58
-
59
- @precision = config[:precision]
60
- @limit = config[:cache_limit]
61
- @min_age = config[:min_age]
62
- @ttl = config[:ttl]
63
-
64
- # keep a reference to the expire cache to prevent having to allocate it
65
- # all the time.
66
- @expire = Hash.new
67
- # Cache of active events.
68
- @cache = Hash.new
69
-
70
- starting do
71
- @timer = EM.add_periodic_timer(@ttl){expire!} unless @ttl.nil?
72
- log.info "Started"
73
- log.info " config: #{config.inspect}"
74
- end
75
-
76
- stopping do
77
- log.info "Stopping rate processor"
78
- @timer.cancel if @timer
79
- end
80
- end
81
-
82
- # Expire cached events that are too old.
83
- def expire!
84
- now = Time.new
85
-
86
- @cache.each do |key, value|
87
- diff = (now - value[:time])
88
- next if diff < @ttl
89
- @expire[key] = value
90
- end
91
-
92
- unless @expire.empty?
93
- increment :expired, @cache.size - @expire.size
94
- @cache = @expire
95
- @expire = Hash.new
96
- end
97
- end
98
-
99
- def process msg
100
- key = msg[:key]
101
- time = msg[:time]
102
- value = msg[:value] || 0
103
-
104
- unless (prev = @cache[key]).nil?
105
- prev_time = prev[:time]
106
- prev_value = prev[:value]
107
-
108
- diff = (time - prev_time)
109
-
110
- valid = @ttl.nil? or diff < @ttl
111
- aged = @min_age.nil? or diff > @min_age
112
-
113
- if diff > 0 and valid and aged
114
- rate = ((value - prev_value) / diff)
115
- rate = rate.round(@precision) unless @precision.nil?
116
- @emitter.metric.emit(
117
- :key => "#{key}.rate", :source => key, :value => rate)
118
- end
119
- else
120
- return increment :dropped if @cache.size >= @limit
121
- end
122
-
123
- increment :received
124
- @cache[key] = {:key => key, :time => time, :value => value}
125
- end
126
- end
127
- end