ffwd 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/ffwd +9 -0
- data/bin/fwc +15 -0
- data/lib/em/all.rb +68 -0
- data/lib/ffwd.rb +250 -0
- data/lib/ffwd/channel.rb +62 -0
- data/lib/ffwd/circular_buffer.rb +78 -0
- data/lib/ffwd/connection.rb +40 -0
- data/lib/ffwd/core.rb +173 -0
- data/lib/ffwd/core/emitter.rb +38 -0
- data/lib/ffwd/core/interface.rb +47 -0
- data/lib/ffwd/core/processor.rb +92 -0
- data/lib/ffwd/core/reporter.rb +32 -0
- data/lib/ffwd/debug.rb +76 -0
- data/lib/ffwd/debug/connection.rb +48 -0
- data/lib/ffwd/debug/monitor_session.rb +71 -0
- data/lib/ffwd/debug/tcp.rb +82 -0
- data/lib/ffwd/event.rb +65 -0
- data/lib/ffwd/event_emitter.rb +57 -0
- data/lib/ffwd/handler.rb +43 -0
- data/lib/ffwd/lifecycle.rb +92 -0
- data/lib/ffwd/logging.rb +139 -0
- data/lib/ffwd/metric.rb +55 -0
- data/lib/ffwd/metric_emitter.rb +50 -0
- data/lib/ffwd/plugin.rb +149 -0
- data/lib/ffwd/plugin/json_line.rb +47 -0
- data/lib/ffwd/plugin/json_line/connection.rb +118 -0
- data/lib/ffwd/plugin/log.rb +35 -0
- data/lib/ffwd/plugin/log/writer.rb +42 -0
- data/lib/ffwd/plugin_channel.rb +64 -0
- data/lib/ffwd/plugin_loader.rb +121 -0
- data/lib/ffwd/processor.rb +96 -0
- data/lib/ffwd/processor/count.rb +109 -0
- data/lib/ffwd/processor/histogram.rb +200 -0
- data/lib/ffwd/processor/rate.rb +116 -0
- data/lib/ffwd/producing_client.rb +181 -0
- data/lib/ffwd/protocol.rb +28 -0
- data/lib/ffwd/protocol/tcp.rb +126 -0
- data/lib/ffwd/protocol/tcp/bind.rb +64 -0
- data/lib/ffwd/protocol/tcp/connection.rb +107 -0
- data/lib/ffwd/protocol/tcp/flushing_connect.rb +135 -0
- data/lib/ffwd/protocol/tcp/plain_connect.rb +74 -0
- data/lib/ffwd/protocol/udp.rb +48 -0
- data/lib/ffwd/protocol/udp/bind.rb +64 -0
- data/lib/ffwd/protocol/udp/connect.rb +110 -0
- data/lib/ffwd/reporter.rb +65 -0
- data/lib/ffwd/retrier.rb +72 -0
- data/lib/ffwd/schema.rb +92 -0
- data/lib/ffwd/schema/default.rb +36 -0
- data/lib/ffwd/schema/spotify100.rb +58 -0
- data/lib/ffwd/statistics.rb +29 -0
- data/lib/ffwd/statistics/collector.rb +99 -0
- data/lib/ffwd/statistics/system_statistics.rb +255 -0
- data/lib/ffwd/tunnel.rb +27 -0
- data/lib/ffwd/tunnel/plugin.rb +47 -0
- data/lib/ffwd/tunnel/tcp.rb +60 -0
- data/lib/ffwd/tunnel/udp.rb +61 -0
- data/lib/ffwd/utils.rb +46 -0
- data/lib/ffwd/version.rb +18 -0
- data/lib/fwc.rb +206 -0
- metadata +163 -0
@@ -0,0 +1,35 @@
|
|
1
|
+
# $LICENSE
|
2
|
+
# Copyright 2013-2014 Spotify AB. All rights reserved.
|
3
|
+
#
|
4
|
+
# The contents of this file are licensed under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
13
|
+
# License for the specific language governing permissions and limitations under
|
14
|
+
# the License.
|
15
|
+
|
16
|
+
require 'ffwd/event'
|
17
|
+
require 'ffwd/logging'
|
18
|
+
require 'ffwd/metric'
|
19
|
+
require 'ffwd/plugin'
|
20
|
+
|
21
|
+
require_relative 'log/writer'
|
22
|
+
|
23
|
+
module FFWD::Plugin
|
24
|
+
module Log
|
25
|
+
include FFWD::Plugin
|
26
|
+
include FFWD::Logging
|
27
|
+
|
28
|
+
register_plugin "log"
|
29
|
+
|
30
|
+
def self.setup_output opts, core
|
31
|
+
prefix = opts[:prefix]
|
32
|
+
Writer.new core, prefix
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# $LICENSE
|
2
|
+
# Copyright 2013-2014 Spotify AB. All rights reserved.
|
3
|
+
#
|
4
|
+
# The contents of this file are licensed under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
13
|
+
# License for the specific language governing permissions and limitations under
|
14
|
+
# the License.
|
15
|
+
|
16
|
+
require_relative '../../logging'
|
17
|
+
|
18
|
+
module FFWD::Plugin::Log
|
19
|
+
class Writer
|
20
|
+
include FFWD::Logging
|
21
|
+
|
22
|
+
def initialize core, prefix
|
23
|
+
@p = prefix ? "#{prefix} " : ""
|
24
|
+
|
25
|
+
subs = []
|
26
|
+
|
27
|
+
core.output.starting do
|
28
|
+
subs << core.output.event_subscribe do |e|
|
29
|
+
log.info "Event: #{@p}#{e.to_h}"
|
30
|
+
end
|
31
|
+
|
32
|
+
subs << core.output.metric_subscribe do |m|
|
33
|
+
log.info "Metric: #{@p}#{m.to_h}"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
core.output.stopping do
|
38
|
+
subs.each(&:unsubscribe).clear
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# $LICENSE
|
2
|
+
# Copyright 2013-2014 Spotify AB. All rights reserved.
|
3
|
+
#
|
4
|
+
# The contents of this file are licensed under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
13
|
+
# License for the specific language governing permissions and limitations under
|
14
|
+
# the License.
|
15
|
+
|
16
|
+
require_relative 'lifecycle'
|
17
|
+
require_relative 'logging'
|
18
|
+
require_relative 'reporter'
|
19
|
+
|
20
|
+
module FFWD
|
21
|
+
# A set of channels, one for metrics and one for events.
|
22
|
+
# This is simply a convenience class to group the channel that are available
|
23
|
+
# to a plugin in one direction (usually either input or output).
|
24
|
+
class PluginChannel
|
25
|
+
include FFWD::Lifecycle
|
26
|
+
include FFWD::Reporter
|
27
|
+
include FFWD::Logging
|
28
|
+
|
29
|
+
setup_reporter :keys => [:metrics, :events]
|
30
|
+
|
31
|
+
attr_reader :reporter_meta, :events, :metrics, :name
|
32
|
+
|
33
|
+
def self.build name
|
34
|
+
events = FFWD::Channel.new log, "#{name}.events"
|
35
|
+
metrics = FFWD::Channel.new log, "#{name}.metrics"
|
36
|
+
new name, metrics, events
|
37
|
+
end
|
38
|
+
|
39
|
+
def initialize name, events, metrics
|
40
|
+
@name = name
|
41
|
+
@events = events
|
42
|
+
@metrics = metrics
|
43
|
+
@reporter_meta = {:plugin_channel => @name, :type => "plugin_channel"}
|
44
|
+
end
|
45
|
+
|
46
|
+
def event_subscribe &block
|
47
|
+
@events.subscribe(&block)
|
48
|
+
end
|
49
|
+
|
50
|
+
def event event
|
51
|
+
@events << event
|
52
|
+
increment :events
|
53
|
+
end
|
54
|
+
|
55
|
+
def metric_subscribe &block
|
56
|
+
@metrics.subscribe(&block)
|
57
|
+
end
|
58
|
+
|
59
|
+
def metric metric
|
60
|
+
@metrics << metric
|
61
|
+
increment :metrics
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# $LICENSE
|
2
|
+
# Copyright 2013-2014 Spotify AB. All rights reserved.
|
3
|
+
#
|
4
|
+
# The contents of this file are licensed under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
13
|
+
# License for the specific language governing permissions and limitations under
|
14
|
+
# the License.
|
15
|
+
|
16
|
+
require_relative 'logging'
|
17
|
+
require_relative 'plugin'
|
18
|
+
|
19
|
+
# Some crazy code to load modules from a specific directory structure.
|
20
|
+
module FFWD::PluginLoader
|
21
|
+
include FFWD::Logging
|
22
|
+
|
23
|
+
MODULE_NAME = 'ffwd'
|
24
|
+
|
25
|
+
def self.plugin_directories= directories
|
26
|
+
@plugin_directories = directories
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.plugin_directories
|
30
|
+
@plugin_directories || []
|
31
|
+
end
|
32
|
+
|
33
|
+
# Discover plugins in the specified directory that are prefixed with 'ffwd-'.
|
34
|
+
def self.discover_plugins dir
|
35
|
+
return [] unless File.directory? dir
|
36
|
+
|
37
|
+
Dir.foreach(dir).map do |entity|
|
38
|
+
next if entity.start_with? "."
|
39
|
+
next unless entity.start_with? "#{MODULE_NAME}-"
|
40
|
+
full_path = File.join dir, entity, 'lib'
|
41
|
+
next unless File.directory? full_path
|
42
|
+
yield full_path
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.plugin_paths
|
47
|
+
return @plugin_paths if @plugin_paths
|
48
|
+
|
49
|
+
@plugin_paths = []
|
50
|
+
|
51
|
+
plugin_directories.map do |dir|
|
52
|
+
discover_plugins(dir) do |path|
|
53
|
+
@plugin_paths << path
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
return @plugin_paths
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.load_paths
|
61
|
+
return @load_paths if @load_paths
|
62
|
+
|
63
|
+
@load_paths = []
|
64
|
+
|
65
|
+
Gem::Specification.latest_specs(true).collect do |spec|
|
66
|
+
@load_paths << ["from gem: #{spec.full_name}",
|
67
|
+
File.join(spec.full_gem_path, 'lib')]
|
68
|
+
end
|
69
|
+
|
70
|
+
plugin_paths.each do |path|
|
71
|
+
@load_paths << ["from plugin directory: #{path}", path]
|
72
|
+
end
|
73
|
+
|
74
|
+
$LOAD_PATH.each do |path|
|
75
|
+
@load_paths << ["from $LOAD_PATH: #{path}", path]
|
76
|
+
end
|
77
|
+
|
78
|
+
unless plugin_paths.empty?
|
79
|
+
$LOAD_PATH.concat plugin_paths
|
80
|
+
end
|
81
|
+
|
82
|
+
return @load_paths
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.list_modules module_category, blacklist, &block
|
86
|
+
load_paths.each do |source, path|
|
87
|
+
dir = File.join(path, MODULE_NAME, module_category)
|
88
|
+
|
89
|
+
next unless File.directory? dir
|
90
|
+
|
91
|
+
Dir.foreach dir do |entity|
|
92
|
+
next if entity.start_with? "."
|
93
|
+
next unless entity.end_with? ".rb"
|
94
|
+
next unless File.file? File.join(dir, entity)
|
95
|
+
|
96
|
+
base = entity.slice(0, entity.size - 3)
|
97
|
+
|
98
|
+
if blacklist.include? base
|
99
|
+
log.warning "Ignoring blacklisted module: #{base}"
|
100
|
+
next
|
101
|
+
end
|
102
|
+
|
103
|
+
path = [MODULE_NAME, module_category, base].join('/')
|
104
|
+
yield source, path
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.load mod, blacklist
|
110
|
+
self.list_modules(mod.category, blacklist) do |source, m|
|
111
|
+
begin
|
112
|
+
require m
|
113
|
+
rescue LoadError => e
|
114
|
+
log.error "Failed to require '#{m}'", e
|
115
|
+
end
|
116
|
+
|
117
|
+
# Initialize all newly discovered plugins.
|
118
|
+
mod.load_discovered source
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
# $LICENSE
|
2
|
+
# Copyright 2013-2014 Spotify AB. All rights reserved.
|
3
|
+
#
|
4
|
+
# The contents of this file are licensed under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
13
|
+
# License for the specific language governing permissions and limitations under
|
14
|
+
# the License.
|
15
|
+
|
16
|
+
require_relative 'logging'
|
17
|
+
require_relative 'lifecycle'
|
18
|
+
|
19
|
+
module FFWD::Processor
|
20
|
+
include FFWD::Lifecycle
|
21
|
+
include FFWD::Logging
|
22
|
+
|
23
|
+
class Setup
|
24
|
+
attr_reader :name
|
25
|
+
|
26
|
+
def initialize name, klass, options
|
27
|
+
@name = name
|
28
|
+
@klass = klass
|
29
|
+
@options = options
|
30
|
+
end
|
31
|
+
|
32
|
+
def setup emitter
|
33
|
+
@klass.new emitter, @options
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Module to include for processors.
|
38
|
+
#
|
39
|
+
# Usage:
|
40
|
+
#
|
41
|
+
# class MyProcessor
|
42
|
+
# include FFWD::Processor
|
43
|
+
#
|
44
|
+
# register_processor "my_processor"
|
45
|
+
#
|
46
|
+
# def initialize opts
|
47
|
+
# .. read options ..
|
48
|
+
# end
|
49
|
+
#
|
50
|
+
# def start emitter
|
51
|
+
# ... setup EventMachine tasks ...
|
52
|
+
# end
|
53
|
+
#
|
54
|
+
# def process emitter, m
|
55
|
+
# ... process a single metric ...
|
56
|
+
# emitter.metrics.emit ...
|
57
|
+
# end
|
58
|
+
# end
|
59
|
+
def process m
|
60
|
+
raise Exception.new("process: Not Implemented")
|
61
|
+
end
|
62
|
+
|
63
|
+
module ClassMethods
|
64
|
+
def register_processor(name)
|
65
|
+
unless FFWD::Processor.registry[name].nil?
|
66
|
+
raise "Already registered '#{name}'"
|
67
|
+
end
|
68
|
+
|
69
|
+
FFWD::Processor.registry[name] = self
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def name
|
74
|
+
self.class.name
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.registry
|
78
|
+
@@registry ||= {}
|
79
|
+
end
|
80
|
+
|
81
|
+
def self.category
|
82
|
+
'processor'
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.included(mod)
|
86
|
+
mod.extend ClassMethods
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.load_discovered source
|
90
|
+
end
|
91
|
+
|
92
|
+
# setup hash of processor setup classes.
|
93
|
+
def self.load_processors config
|
94
|
+
registry.map{|name, klass| Setup.new name, klass, config[name] || {}}
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# $LICENSE
|
2
|
+
# Copyright 2013-2014 Spotify AB. All rights reserved.
|
3
|
+
#
|
4
|
+
# The contents of this file are licensed under the Apache License, Version 2.0
|
5
|
+
# (the "License"); you may not use this file except in compliance with the
|
6
|
+
# License. You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
12
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
13
|
+
# License for the specific language governing permissions and limitations under
|
14
|
+
# the License.
|
15
|
+
|
16
|
+
require_relative '../processor'
|
17
|
+
require_relative '../logging'
|
18
|
+
require_relative '../event'
|
19
|
+
require_relative '../utils'
|
20
|
+
require_relative '../reporter'
|
21
|
+
|
22
|
+
module FFWD::Processor
|
23
|
+
#
|
24
|
+
# Implements counting statistics (similar to statsd).
|
25
|
+
#
|
26
|
+
class CountProcessor
|
27
|
+
include FFWD::Logging
|
28
|
+
include FFWD::Processor
|
29
|
+
include FFWD::Reporter
|
30
|
+
|
31
|
+
register_processor "count"
|
32
|
+
setup_reporter(
|
33
|
+
:reporter_meta => {:processor => "count"},
|
34
|
+
:keys => [:dropped, :received]
|
35
|
+
)
|
36
|
+
|
37
|
+
def initialize emitter, opts={}
|
38
|
+
@emitter = emitter
|
39
|
+
@cache_limit = opts[:cache_limit] || 1000
|
40
|
+
@timeout = opts[:timeout] || 300
|
41
|
+
@period = opts[:period] || 10
|
42
|
+
@cache = {}
|
43
|
+
@timer = nil
|
44
|
+
|
45
|
+
starting do
|
46
|
+
log.info "Starting count processor on a window of #{@period}s"
|
47
|
+
end
|
48
|
+
|
49
|
+
stopping do
|
50
|
+
log.info "Stopping count processor"
|
51
|
+
@timer.cancel if @timer
|
52
|
+
@timer = nil
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def flush_caches! now
|
57
|
+
@cache.each do |key, entry|
|
58
|
+
count = entry[:count]
|
59
|
+
last = entry[:last]
|
60
|
+
|
61
|
+
if now - last > @timeout
|
62
|
+
@cache.delete key
|
63
|
+
next
|
64
|
+
end
|
65
|
+
|
66
|
+
entry[:count] = 0
|
67
|
+
|
68
|
+
@emitter.metric.emit(
|
69
|
+
:key => "#{key}.sum", :value => count, :source => key)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def check_timer
|
74
|
+
return if @timer
|
75
|
+
|
76
|
+
log.debug "Starting timer"
|
77
|
+
|
78
|
+
@timer = EM::Timer.new(@period) do
|
79
|
+
@timer = nil
|
80
|
+
digest!
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def digest! now
|
85
|
+
ms = FFWD.timing do
|
86
|
+
flush_caches! now
|
87
|
+
end
|
88
|
+
|
89
|
+
log.debug "Digest took #{ms}ms"
|
90
|
+
end
|
91
|
+
|
92
|
+
def process m
|
93
|
+
key = m[:key]
|
94
|
+
value = m[:value]
|
95
|
+
|
96
|
+
now = Time.now
|
97
|
+
|
98
|
+
unless (entry = @cache[key])
|
99
|
+
return increment :dropped if @cache.size >= @cache_limit
|
100
|
+
entry = @cache[key] = {:count => 0, :last => now}
|
101
|
+
end
|
102
|
+
|
103
|
+
increment :received
|
104
|
+
entry[:count] += value
|
105
|
+
entry[:last] = now
|
106
|
+
check_timer
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|