ffwd 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/bin/ffwd +9 -0
  3. data/bin/fwc +15 -0
  4. data/lib/em/all.rb +68 -0
  5. data/lib/ffwd.rb +250 -0
  6. data/lib/ffwd/channel.rb +62 -0
  7. data/lib/ffwd/circular_buffer.rb +78 -0
  8. data/lib/ffwd/connection.rb +40 -0
  9. data/lib/ffwd/core.rb +173 -0
  10. data/lib/ffwd/core/emitter.rb +38 -0
  11. data/lib/ffwd/core/interface.rb +47 -0
  12. data/lib/ffwd/core/processor.rb +92 -0
  13. data/lib/ffwd/core/reporter.rb +32 -0
  14. data/lib/ffwd/debug.rb +76 -0
  15. data/lib/ffwd/debug/connection.rb +48 -0
  16. data/lib/ffwd/debug/monitor_session.rb +71 -0
  17. data/lib/ffwd/debug/tcp.rb +82 -0
  18. data/lib/ffwd/event.rb +65 -0
  19. data/lib/ffwd/event_emitter.rb +57 -0
  20. data/lib/ffwd/handler.rb +43 -0
  21. data/lib/ffwd/lifecycle.rb +92 -0
  22. data/lib/ffwd/logging.rb +139 -0
  23. data/lib/ffwd/metric.rb +55 -0
  24. data/lib/ffwd/metric_emitter.rb +50 -0
  25. data/lib/ffwd/plugin.rb +149 -0
  26. data/lib/ffwd/plugin/json_line.rb +47 -0
  27. data/lib/ffwd/plugin/json_line/connection.rb +118 -0
  28. data/lib/ffwd/plugin/log.rb +35 -0
  29. data/lib/ffwd/plugin/log/writer.rb +42 -0
  30. data/lib/ffwd/plugin_channel.rb +64 -0
  31. data/lib/ffwd/plugin_loader.rb +121 -0
  32. data/lib/ffwd/processor.rb +96 -0
  33. data/lib/ffwd/processor/count.rb +109 -0
  34. data/lib/ffwd/processor/histogram.rb +200 -0
  35. data/lib/ffwd/processor/rate.rb +116 -0
  36. data/lib/ffwd/producing_client.rb +181 -0
  37. data/lib/ffwd/protocol.rb +28 -0
  38. data/lib/ffwd/protocol/tcp.rb +126 -0
  39. data/lib/ffwd/protocol/tcp/bind.rb +64 -0
  40. data/lib/ffwd/protocol/tcp/connection.rb +107 -0
  41. data/lib/ffwd/protocol/tcp/flushing_connect.rb +135 -0
  42. data/lib/ffwd/protocol/tcp/plain_connect.rb +74 -0
  43. data/lib/ffwd/protocol/udp.rb +48 -0
  44. data/lib/ffwd/protocol/udp/bind.rb +64 -0
  45. data/lib/ffwd/protocol/udp/connect.rb +110 -0
  46. data/lib/ffwd/reporter.rb +65 -0
  47. data/lib/ffwd/retrier.rb +72 -0
  48. data/lib/ffwd/schema.rb +92 -0
  49. data/lib/ffwd/schema/default.rb +36 -0
  50. data/lib/ffwd/schema/spotify100.rb +58 -0
  51. data/lib/ffwd/statistics.rb +29 -0
  52. data/lib/ffwd/statistics/collector.rb +99 -0
  53. data/lib/ffwd/statistics/system_statistics.rb +255 -0
  54. data/lib/ffwd/tunnel.rb +27 -0
  55. data/lib/ffwd/tunnel/plugin.rb +47 -0
  56. data/lib/ffwd/tunnel/tcp.rb +60 -0
  57. data/lib/ffwd/tunnel/udp.rb +61 -0
  58. data/lib/ffwd/utils.rb +46 -0
  59. data/lib/ffwd/version.rb +18 -0
  60. data/lib/fwc.rb +206 -0
  61. metadata +163 -0
@@ -0,0 +1,74 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative '../../reporter'
17
+
18
+ module FFWD::TCP
19
+ class PlainConnect
20
+ include FFWD::Reporter
21
+
22
+ setup_reporter :keys => [
23
+ :dropped_events, :dropped_metrics,
24
+ :sent_events, :sent_metrics,
25
+ :failed_events, :failed_metrics
26
+ ]
27
+
28
+ attr_reader :log
29
+
30
+ def reporter_meta
31
+ @c.reporter_meta
32
+ end
33
+
34
+ INITIAL_TIMEOUT = 2
35
+
36
+ def initialize core, log, connection
37
+ @log = log
38
+ @c = connection
39
+
40
+ subs = []
41
+
42
+ core.starting do
43
+ @c.connect
44
+ subs << core.output.event_subscribe{|e| handle_event e}
45
+ subs << core.output.metric_subscribe{|e| handle_metric e}
46
+ end
47
+
48
+ core.stopping do
49
+ @c.disconnect
50
+ subs.each(&:unsubscribe).clear
51
+ end
52
+ end
53
+
54
+ def handle_event event
55
+ return increment :dropped_events, 1 unless @c.writable?
56
+ @c.send_event event
57
+ increment :sent_events, 1
58
+ rescue => e
59
+ log.error "Failed to handle event", e
60
+ log.error "The following event could not be flushed: #{event.to_h}"
61
+ increment :failed_events, 1
62
+ end
63
+
64
+ def handle_metric metric
65
+ return increment :dropped_metrics, 1 unless @c.writable?
66
+ @c.send_metric metric
67
+ increment :sent_metrics, 1
68
+ rescue => e
69
+ log.error "Failed to handle metric", e
70
+ log.error "The following metric could not be flushed: #{metric.to_h}"
71
+ increment :failed_metrics, 1
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,48 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require 'eventmachine'
17
+
18
+ require_relative 'udp/connect'
19
+ require_relative 'udp/bind'
20
+
21
+ require_relative '../tunnel'
22
+
23
+ module FFWD::UDP
24
+ def self.family
25
+ :udp
26
+ end
27
+
28
+ DEFAULT_REBIND_TIMEOUT = 10
29
+
30
+ def self.connect opts, core, log, handler
31
+ raise "Missing required key :host" if (host = opts[:host]).nil?
32
+ raise "Missing required key :port" if (port = opts[:port]).nil?
33
+ Connect.new core, log, host, port, handler
34
+ end
35
+
36
+ def self.bind opts, core, log, connection, *args
37
+ raise "Missing required key :host" if (host = opts[:host]).nil?
38
+ raise "Missing required key :port" if (port = opts[:port]).nil?
39
+ rebind_timeout = opts[:rebind_timeout] || DEFAULT_REBIND_TIMEOUT
40
+ Bind.new core, log, host, port, connection, args, rebind_timeout
41
+ end
42
+
43
+ def self.tunnel opts, core, plugin, log, connection, *args
44
+ raise "Missing required key :port" if (port = opts[:port]).nil?
45
+ FFWD.tunnel self.family, port, core, plugin, log, connection, args
46
+ end
47
+ end
48
+
@@ -0,0 +1,64 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require 'eventmachine'
17
+
18
+ require_relative '../../reporter'
19
+ require_relative '../../retrier'
20
+
21
+ module FFWD::UDP
22
+ class Bind
23
+ include FFWD::Reporter
24
+
25
+ setup_reporter :keys => [
26
+ :received_events, :received_metrics,
27
+ :failed_events, :failed_metrics
28
+ ]
29
+
30
+ attr_reader :reporter_meta
31
+
32
+ def initialize core, log, host, port, connection, args, rebind_timeout
33
+ @peer = "#{host}:#{port}"
34
+ @reporter_meta = {
35
+ :type => connection.plugin_type,
36
+ :listen => @peer, :family => 'udp'
37
+ }
38
+
39
+ @sig = nil
40
+
41
+ info = "udp://#{@peer}"
42
+
43
+ r = FFWD.retry :timeout => rebind_timeout do |a|
44
+ @sig = EM.open_datagram_socket host, port, connection, self, core, *args
45
+ log.info "Bind on #{info} (attempt #{a})"
46
+ end
47
+
48
+ r.error do |a, t, e|
49
+ log.error "Failed to bind #{info} (attempt #{a}), retry in #{t}s", e
50
+ end
51
+
52
+ r.depend_on core
53
+
54
+ core.stopping do
55
+ log.info "Unbinding #{info}"
56
+
57
+ if @sig
58
+ @sig.unbind
59
+ @sig = nil
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,110 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative '../../reporter'
17
+ require_relative '../../retrier'
18
+
19
+ module FFWD::UDP
20
+ class Connect
21
+ include FFWD::Reporter
22
+
23
+ attr_reader :reporter_meta, :log
24
+
25
+ setup_reporter :keys => [
26
+ :dropped_events, :dropped_metrics,
27
+ :sent_events, :sent_metrics
28
+ ]
29
+
30
+ def initialize core, log, host, port, handler
31
+ @log = log
32
+ @host = host
33
+ @port = port
34
+ @handler = handler
35
+
36
+ @bind_host = "0.0.0.0"
37
+ @host_ip = nil
38
+ @c = nil
39
+ @peer = "#{host}:#{port}"
40
+ @reporter_meta = {
41
+ :type => @handler.name, :peer => peer
42
+ }
43
+
44
+ info = "udp://#{@peer}"
45
+
46
+ subs = []
47
+
48
+ r = FFWD.retry :timeout => resolve_timeout do |a|
49
+ unless @host_ip
50
+ @host_ip = resolve_ip @host
51
+ raise "Could not resolve: #{@host}" if @host_ip.nil?
52
+ end
53
+
54
+ @c = EM.open_datagram_socket(@bind_host, nil)
55
+ log.info "Setup of output to #{info} successful"
56
+
57
+ subs << core.output.event_subscribe{|e| handle_event e}
58
+ subs << core.output.metric_subscribe{|m| handle_metric m}
59
+ end
60
+
61
+ r.error do |a, t, e|
62
+ log.error "Setup of output to #{info} failed (attempt #{a}), retry in #{t}s", e
63
+ end
64
+
65
+ r.depend_on core
66
+
67
+ core.stopping do
68
+ if @c
69
+ @c.close
70
+ @c = nil
71
+ end
72
+
73
+ subs.each(&:unsubscribe).clear
74
+ end
75
+ end
76
+
77
+ private
78
+
79
+ def handle_event event
80
+ unless @c
81
+ increment :dropped_events, 1
82
+ return
83
+ end
84
+
85
+ data = @handler.serialize_event event
86
+ @c.send_datagram data, @host_ip, @port
87
+ increment :sent_events, 1
88
+ end
89
+
90
+ def handle_metric metric
91
+ unless @c
92
+ increment :dropped_metrics, 1
93
+ return
94
+ end
95
+
96
+ data = @handler.serialize_metric metric
97
+ @c.send_datagram data, @host_ip, @port
98
+ increment :sent_metrics, 1
99
+ end
100
+
101
+ def resolve_ip host
102
+ Socket.getaddrinfo(@host, nil, nil, :DGRAM).each do |item|
103
+ next if item[0] != "AF_INET"
104
+ return item[3]
105
+ end
106
+
107
+ return nil
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,65 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ module FFWD::Reporter
17
+ def self.map_meta meta
18
+ Hash[meta.map{|k, v| [k.to_s, v]}]
19
+ end
20
+
21
+ module ClassMethods
22
+ def reporter_keys
23
+ @_reporter_keys ||= [:total]
24
+ end
25
+
26
+ def reporter_meta
27
+ @_reporter_meta ||= nil
28
+ end
29
+
30
+ def reporter_meta_method
31
+ @_reporter_meta_method ||= :reporter_meta
32
+ end
33
+
34
+ def setup_reporter opts={}
35
+ @_reporter_keys = [:total] + (opts[:keys] || [])
36
+ @_reporter_meta = opts[:reporter_meta]
37
+ @_reporter_meta_method = opts[:id_method] || :reporter_meta
38
+ end
39
+ end
40
+
41
+ def self.included mod
42
+ mod.extend ClassMethods
43
+ end
44
+
45
+ def reporter_data
46
+ @_reporter_keys ||= self.class.reporter_keys
47
+ @_reporter_data ||= Hash[@_reporter_keys.map{|k| [k, 0]}]
48
+ end
49
+
50
+ def increment n, c=1
51
+ reporter_data[n] += c
52
+ reporter_data[:total] += c
53
+ end
54
+
55
+ def report!
56
+ @_reporter_meta ||= FFWD::Reporter.map_meta(
57
+ self.class.reporter_meta || send(self.class.reporter_meta_method))
58
+
59
+ reporter_data.each do |k, v|
60
+ yield(:key => k, :value => v,
61
+ :meta => @_reporter_meta)
62
+ reporter_data[k] = 0
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,72 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative 'logging'
17
+ require_relative 'lifecycle'
18
+
19
+ module FFWD
20
+ # Try to execute a block on an exponential timer until it no longer throws
21
+ # an exception.
22
+ class Retrier
23
+ include FFWD::Lifecycle
24
+
25
+ def initialize timeout, &block
26
+ @block = block
27
+ @timer = nil
28
+ @timeout = timeout
29
+ @current_timeout = @timeout
30
+ @attempt = 0
31
+ @error_callbacks = []
32
+
33
+ starting do
34
+ try_block
35
+ end
36
+
37
+ stopping do
38
+ if @timer
39
+ @timer.cancel
40
+ @timer = nil
41
+ end
42
+ end
43
+ end
44
+
45
+ def error &block
46
+ @error_callbacks << block
47
+ end
48
+
49
+ def try_block
50
+ @attempt += 1
51
+ @block.call @attempt
52
+ @current_timeout = @timeout
53
+ rescue => e
54
+ @error_callbacks.each do |block|
55
+ block.call @attempt, @current_timeout, e
56
+ end
57
+
58
+ @timer = EM::Timer.new(@current_timeout) do
59
+ @current_timeout *= 2
60
+ @timer = nil
61
+ try_block
62
+ end
63
+ end
64
+ end
65
+
66
+ DEFAULT_TIMEOUT = 10
67
+
68
+ def self.retry opts={}, &block
69
+ timeout = opts[:timeout] || DEFAULT_TIMEOUT
70
+ Retrier.new(timeout, &block)
71
+ end
72
+ end
@@ -0,0 +1,92 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ module FFWD
17
+ SCHEMA_DEFAULT_SUPPORT = [
18
+ :dump_metric,
19
+ :dump_event
20
+ ]
21
+
22
+ DEFUALT_SCHEMA = 'default'
23
+ DEFAULT_CONTENT_TYPE = 'application/json'
24
+
25
+ def self.parse_schema opts, support=SCHEMA_DEFAULT_SUPPORT
26
+ name = opts[:schema] || DEFUALT_SCHEMA
27
+ content_type = opts[:content_type] || DEFAULT_CONTENT_TYPE
28
+ key = [name, content_type]
29
+
30
+ schema = FFWD::Schema.loaded[key]
31
+
32
+ if schema.nil?
33
+ raise "No schema '#{name}' for content type '#{content_type}'"
34
+ end
35
+
36
+ unless schema.support? support
37
+ raise "Schema #{schema} does not support all of: #{support}"
38
+ end
39
+
40
+ return schema.mod
41
+ end
42
+
43
+ module Schema
44
+ class Loaded
45
+ attr_reader :source, :mod
46
+
47
+ def initialize source, mod
48
+ @source = source
49
+ @mod = mod
50
+ end
51
+
52
+ def support? support
53
+ support.each do |m|
54
+ return false unless @mod.respond_to? m
55
+ end
56
+
57
+ return true
58
+ end
59
+ end
60
+
61
+ def self.loaded
62
+ @@loaded ||= {}
63
+ end
64
+
65
+ def self.discovered
66
+ @@discovered ||= {}
67
+ end
68
+
69
+ module ClassMethods
70
+ def register_schema name, content_type, impl
71
+ key = [name, content_type]
72
+ FFWD::Schema.discovered[key] = impl
73
+ end
74
+ end
75
+
76
+ def self.included mod
77
+ mod.extend ClassMethods
78
+ end
79
+
80
+ def self.category
81
+ 'schema'
82
+ end
83
+
84
+ def self.load_discovered source
85
+ FFWD::Schema.discovered.each do |key, mod|
86
+ FFWD::Schema.loaded[key] = Loaded.new source, mod
87
+ end
88
+
89
+ FFWD::Schema.discovered.clear
90
+ end
91
+ end
92
+ end