ffwd 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/bin/ffwd +9 -0
  3. data/bin/fwc +15 -0
  4. data/lib/em/all.rb +68 -0
  5. data/lib/ffwd.rb +250 -0
  6. data/lib/ffwd/channel.rb +62 -0
  7. data/lib/ffwd/circular_buffer.rb +78 -0
  8. data/lib/ffwd/connection.rb +40 -0
  9. data/lib/ffwd/core.rb +173 -0
  10. data/lib/ffwd/core/emitter.rb +38 -0
  11. data/lib/ffwd/core/interface.rb +47 -0
  12. data/lib/ffwd/core/processor.rb +92 -0
  13. data/lib/ffwd/core/reporter.rb +32 -0
  14. data/lib/ffwd/debug.rb +76 -0
  15. data/lib/ffwd/debug/connection.rb +48 -0
  16. data/lib/ffwd/debug/monitor_session.rb +71 -0
  17. data/lib/ffwd/debug/tcp.rb +82 -0
  18. data/lib/ffwd/event.rb +65 -0
  19. data/lib/ffwd/event_emitter.rb +57 -0
  20. data/lib/ffwd/handler.rb +43 -0
  21. data/lib/ffwd/lifecycle.rb +92 -0
  22. data/lib/ffwd/logging.rb +139 -0
  23. data/lib/ffwd/metric.rb +55 -0
  24. data/lib/ffwd/metric_emitter.rb +50 -0
  25. data/lib/ffwd/plugin.rb +149 -0
  26. data/lib/ffwd/plugin/json_line.rb +47 -0
  27. data/lib/ffwd/plugin/json_line/connection.rb +118 -0
  28. data/lib/ffwd/plugin/log.rb +35 -0
  29. data/lib/ffwd/plugin/log/writer.rb +42 -0
  30. data/lib/ffwd/plugin_channel.rb +64 -0
  31. data/lib/ffwd/plugin_loader.rb +121 -0
  32. data/lib/ffwd/processor.rb +96 -0
  33. data/lib/ffwd/processor/count.rb +109 -0
  34. data/lib/ffwd/processor/histogram.rb +200 -0
  35. data/lib/ffwd/processor/rate.rb +116 -0
  36. data/lib/ffwd/producing_client.rb +181 -0
  37. data/lib/ffwd/protocol.rb +28 -0
  38. data/lib/ffwd/protocol/tcp.rb +126 -0
  39. data/lib/ffwd/protocol/tcp/bind.rb +64 -0
  40. data/lib/ffwd/protocol/tcp/connection.rb +107 -0
  41. data/lib/ffwd/protocol/tcp/flushing_connect.rb +135 -0
  42. data/lib/ffwd/protocol/tcp/plain_connect.rb +74 -0
  43. data/lib/ffwd/protocol/udp.rb +48 -0
  44. data/lib/ffwd/protocol/udp/bind.rb +64 -0
  45. data/lib/ffwd/protocol/udp/connect.rb +110 -0
  46. data/lib/ffwd/reporter.rb +65 -0
  47. data/lib/ffwd/retrier.rb +72 -0
  48. data/lib/ffwd/schema.rb +92 -0
  49. data/lib/ffwd/schema/default.rb +36 -0
  50. data/lib/ffwd/schema/spotify100.rb +58 -0
  51. data/lib/ffwd/statistics.rb +29 -0
  52. data/lib/ffwd/statistics/collector.rb +99 -0
  53. data/lib/ffwd/statistics/system_statistics.rb +255 -0
  54. data/lib/ffwd/tunnel.rb +27 -0
  55. data/lib/ffwd/tunnel/plugin.rb +47 -0
  56. data/lib/ffwd/tunnel/tcp.rb +60 -0
  57. data/lib/ffwd/tunnel/udp.rb +61 -0
  58. data/lib/ffwd/utils.rb +46 -0
  59. data/lib/ffwd/version.rb +18 -0
  60. data/lib/fwc.rb +206 -0
  61. metadata +163 -0
@@ -0,0 +1,35 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require 'ffwd/event'
17
+ require 'ffwd/logging'
18
+ require 'ffwd/metric'
19
+ require 'ffwd/plugin'
20
+
21
+ require_relative 'log/writer'
22
+
23
+ module FFWD::Plugin
24
+ module Log
25
+ include FFWD::Plugin
26
+ include FFWD::Logging
27
+
28
+ register_plugin "log"
29
+
30
+ def self.setup_output opts, core
31
+ prefix = opts[:prefix]
32
+ Writer.new core, prefix
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,42 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative '../../logging'
17
+
18
+ module FFWD::Plugin::Log
19
+ class Writer
20
+ include FFWD::Logging
21
+
22
+ def initialize core, prefix
23
+ @p = prefix ? "#{prefix} " : ""
24
+
25
+ subs = []
26
+
27
+ core.output.starting do
28
+ subs << core.output.event_subscribe do |e|
29
+ log.info "Event: #{@p}#{e.to_h}"
30
+ end
31
+
32
+ subs << core.output.metric_subscribe do |m|
33
+ log.info "Metric: #{@p}#{m.to_h}"
34
+ end
35
+ end
36
+
37
+ core.output.stopping do
38
+ subs.each(&:unsubscribe).clear
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,64 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative 'lifecycle'
17
+ require_relative 'logging'
18
+ require_relative 'reporter'
19
+
20
+ module FFWD
21
+ # A set of channels, one for metrics and one for events.
22
+ # This is simply a convenience class to group the channel that are available
23
+ # to a plugin in one direction (usually either input or output).
24
+ class PluginChannel
25
+ include FFWD::Lifecycle
26
+ include FFWD::Reporter
27
+ include FFWD::Logging
28
+
29
+ setup_reporter :keys => [:metrics, :events]
30
+
31
+ attr_reader :reporter_meta, :events, :metrics, :name
32
+
33
+ def self.build name
34
+ events = FFWD::Channel.new log, "#{name}.events"
35
+ metrics = FFWD::Channel.new log, "#{name}.metrics"
36
+ new name, metrics, events
37
+ end
38
+
39
+ def initialize name, events, metrics
40
+ @name = name
41
+ @events = events
42
+ @metrics = metrics
43
+ @reporter_meta = {:plugin_channel => @name, :type => "plugin_channel"}
44
+ end
45
+
46
+ def event_subscribe &block
47
+ @events.subscribe(&block)
48
+ end
49
+
50
+ def event event
51
+ @events << event
52
+ increment :events
53
+ end
54
+
55
+ def metric_subscribe &block
56
+ @metrics.subscribe(&block)
57
+ end
58
+
59
+ def metric metric
60
+ @metrics << metric
61
+ increment :metrics
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,121 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative 'logging'
17
+ require_relative 'plugin'
18
+
19
+ # Some crazy code to load modules from a specific directory structure.
20
+ module FFWD::PluginLoader
21
+ include FFWD::Logging
22
+
23
+ MODULE_NAME = 'ffwd'
24
+
25
+ def self.plugin_directories= directories
26
+ @plugin_directories = directories
27
+ end
28
+
29
+ def self.plugin_directories
30
+ @plugin_directories || []
31
+ end
32
+
33
+ # Discover plugins in the specified directory that are prefixed with 'ffwd-'.
34
+ def self.discover_plugins dir
35
+ return [] unless File.directory? dir
36
+
37
+ Dir.foreach(dir).map do |entity|
38
+ next if entity.start_with? "."
39
+ next unless entity.start_with? "#{MODULE_NAME}-"
40
+ full_path = File.join dir, entity, 'lib'
41
+ next unless File.directory? full_path
42
+ yield full_path
43
+ end
44
+ end
45
+
46
+ def self.plugin_paths
47
+ return @plugin_paths if @plugin_paths
48
+
49
+ @plugin_paths = []
50
+
51
+ plugin_directories.map do |dir|
52
+ discover_plugins(dir) do |path|
53
+ @plugin_paths << path
54
+ end
55
+ end
56
+
57
+ return @plugin_paths
58
+ end
59
+
60
+ def self.load_paths
61
+ return @load_paths if @load_paths
62
+
63
+ @load_paths = []
64
+
65
+ Gem::Specification.latest_specs(true).collect do |spec|
66
+ @load_paths << ["from gem: #{spec.full_name}",
67
+ File.join(spec.full_gem_path, 'lib')]
68
+ end
69
+
70
+ plugin_paths.each do |path|
71
+ @load_paths << ["from plugin directory: #{path}", path]
72
+ end
73
+
74
+ $LOAD_PATH.each do |path|
75
+ @load_paths << ["from $LOAD_PATH: #{path}", path]
76
+ end
77
+
78
+ unless plugin_paths.empty?
79
+ $LOAD_PATH.concat plugin_paths
80
+ end
81
+
82
+ return @load_paths
83
+ end
84
+
85
+ def self.list_modules module_category, blacklist, &block
86
+ load_paths.each do |source, path|
87
+ dir = File.join(path, MODULE_NAME, module_category)
88
+
89
+ next unless File.directory? dir
90
+
91
+ Dir.foreach dir do |entity|
92
+ next if entity.start_with? "."
93
+ next unless entity.end_with? ".rb"
94
+ next unless File.file? File.join(dir, entity)
95
+
96
+ base = entity.slice(0, entity.size - 3)
97
+
98
+ if blacklist.include? base
99
+ log.warning "Ignoring blacklisted module: #{base}"
100
+ next
101
+ end
102
+
103
+ path = [MODULE_NAME, module_category, base].join('/')
104
+ yield source, path
105
+ end
106
+ end
107
+ end
108
+
109
+ def self.load mod, blacklist
110
+ self.list_modules(mod.category, blacklist) do |source, m|
111
+ begin
112
+ require m
113
+ rescue LoadError => e
114
+ log.error "Failed to require '#{m}'", e
115
+ end
116
+
117
+ # Initialize all newly discovered plugins.
118
+ mod.load_discovered source
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,96 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative 'logging'
17
+ require_relative 'lifecycle'
18
+
19
+ module FFWD::Processor
20
+ include FFWD::Lifecycle
21
+ include FFWD::Logging
22
+
23
+ class Setup
24
+ attr_reader :name
25
+
26
+ def initialize name, klass, options
27
+ @name = name
28
+ @klass = klass
29
+ @options = options
30
+ end
31
+
32
+ def setup emitter
33
+ @klass.new emitter, @options
34
+ end
35
+ end
36
+
37
+ # Module to include for processors.
38
+ #
39
+ # Usage:
40
+ #
41
+ # class MyProcessor
42
+ # include FFWD::Processor
43
+ #
44
+ # register_processor "my_processor"
45
+ #
46
+ # def initialize opts
47
+ # .. read options ..
48
+ # end
49
+ #
50
+ # def start emitter
51
+ # ... setup EventMachine tasks ...
52
+ # end
53
+ #
54
+ # def process emitter, m
55
+ # ... process a single metric ...
56
+ # emitter.metrics.emit ...
57
+ # end
58
+ # end
59
+ def process m
60
+ raise Exception.new("process: Not Implemented")
61
+ end
62
+
63
+ module ClassMethods
64
+ def register_processor(name)
65
+ unless FFWD::Processor.registry[name].nil?
66
+ raise "Already registered '#{name}'"
67
+ end
68
+
69
+ FFWD::Processor.registry[name] = self
70
+ end
71
+ end
72
+
73
+ def name
74
+ self.class.name
75
+ end
76
+
77
+ def self.registry
78
+ @@registry ||= {}
79
+ end
80
+
81
+ def self.category
82
+ 'processor'
83
+ end
84
+
85
+ def self.included(mod)
86
+ mod.extend ClassMethods
87
+ end
88
+
89
+ def self.load_discovered source
90
+ end
91
+
92
+ # setup hash of processor setup classes.
93
+ def self.load_processors config
94
+ registry.map{|name, klass| Setup.new name, klass, config[name] || {}}
95
+ end
96
+ end
@@ -0,0 +1,109 @@
1
+ # $LICENSE
2
+ # Copyright 2013-2014 Spotify AB. All rights reserved.
3
+ #
4
+ # The contents of this file are licensed under the Apache License, Version 2.0
5
+ # (the "License"); you may not use this file except in compliance with the
6
+ # License. You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+ # License for the specific language governing permissions and limitations under
14
+ # the License.
15
+
16
+ require_relative '../processor'
17
+ require_relative '../logging'
18
+ require_relative '../event'
19
+ require_relative '../utils'
20
+ require_relative '../reporter'
21
+
22
+ module FFWD::Processor
23
+ #
24
+ # Implements counting statistics (similar to statsd).
25
+ #
26
+ class CountProcessor
27
+ include FFWD::Logging
28
+ include FFWD::Processor
29
+ include FFWD::Reporter
30
+
31
+ register_processor "count"
32
+ setup_reporter(
33
+ :reporter_meta => {:processor => "count"},
34
+ :keys => [:dropped, :received]
35
+ )
36
+
37
+ def initialize emitter, opts={}
38
+ @emitter = emitter
39
+ @cache_limit = opts[:cache_limit] || 1000
40
+ @timeout = opts[:timeout] || 300
41
+ @period = opts[:period] || 10
42
+ @cache = {}
43
+ @timer = nil
44
+
45
+ starting do
46
+ log.info "Starting count processor on a window of #{@period}s"
47
+ end
48
+
49
+ stopping do
50
+ log.info "Stopping count processor"
51
+ @timer.cancel if @timer
52
+ @timer = nil
53
+ end
54
+ end
55
+
56
+ def flush_caches! now
57
+ @cache.each do |key, entry|
58
+ count = entry[:count]
59
+ last = entry[:last]
60
+
61
+ if now - last > @timeout
62
+ @cache.delete key
63
+ next
64
+ end
65
+
66
+ entry[:count] = 0
67
+
68
+ @emitter.metric.emit(
69
+ :key => "#{key}.sum", :value => count, :source => key)
70
+ end
71
+ end
72
+
73
+ def check_timer
74
+ return if @timer
75
+
76
+ log.debug "Starting timer"
77
+
78
+ @timer = EM::Timer.new(@period) do
79
+ @timer = nil
80
+ digest!
81
+ end
82
+ end
83
+
84
+ def digest! now
85
+ ms = FFWD.timing do
86
+ flush_caches! now
87
+ end
88
+
89
+ log.debug "Digest took #{ms}ms"
90
+ end
91
+
92
+ def process m
93
+ key = m[:key]
94
+ value = m[:value]
95
+
96
+ now = Time.now
97
+
98
+ unless (entry = @cache[key])
99
+ return increment :dropped if @cache.size >= @cache_limit
100
+ entry = @cache[key] = {:count => 0, :last => now}
101
+ end
102
+
103
+ increment :received
104
+ entry[:count] += value
105
+ entry[:last] = now
106
+ check_timer
107
+ end
108
+ end
109
+ end