ffwd 0.1.7 → 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4b630458a10baaf2b380c2fea5d264c0b5899dfb
+  data.tar.gz: de1dbbce8ec3bbd5a4d8acd85dc2b7f6c5ef9cfa
+SHA512:
+  metadata.gz: 1d32fad236f6af767352403f042f9cf03a5e7133a41081a0d390715467339ed579e0993b853a433dbbc5cae02c433ad44b4bcadea58228e2c68a9983e6f88219
+  data.tar.gz: b60bf9c734110e417fd45c9bb8282e9224780146bb2bd403b62d5a315c2ff26d061b8eaa303a266a1fd61f2ecc2dfcdc19ae99b1cf9562b8b222000191e509a0

data/lib/ffwd.rb

@@ -25,9 +25,8 @@ require_relative 'ffwd/schema'
 require_relative 'ffwd/version'
 
 module FFWD
-  DEFAULT_PLUGIN_DIRECTORIES = [
-    './plugins'
-  ]
+  DEFAULT_PLUGIN_DIRECTORIES = ['./plugins' ]
+  DEFAULT_CONFIG_PREFIXES = ['./ffwd.d']
 
   def self.load_yaml path
     return YAML.load_file path
@@ -70,7 +69,7 @@ module FFWD
       c = load_yaml entry_path
 
       if c.nil?
-        log.warn "Ignoring: #{entry_path} (invalid yaml)"
+        log.warning "Ignoring: #{entry_path} (invalid yaml)"
         next
       end
 
@@ -82,6 +81,7 @@ module FFWD
     @@opts ||= {:debug => false, :config => nil, :config_dir => nil,
                 :list_plugins => false, :list_schemas => false,
                 :dump_config => false, :show_version => false,
+                :config_prefixes => DEFAULT_CONFIG_PREFIXES,
                 :config_paths => [],
                 :plugin_directories => DEFAULT_PLUGIN_DIRECTORIES}
   end
@@ -95,9 +95,15 @@ module FFWD
       end
 
       o.on "-c", "--config <path>", "Load the specified configuration file." do |path|
+        puts "WARNING: Deprecated option '-c #{path}'."
+        puts "         Use positional argument to specify configuration!"
         opts[:config_paths] << path
       end
 
+      o.on "-D", "--config-prefix <path>", "Scan for configuration files relative to this directory." do |path|
+        opts[:config_prefixes] << path
+      end
+
       o.on "-d", "--config-directory <path>", "Load configuration files from the specified directory." do |path|
         opts[:config_dir] = path
       end
@@ -132,13 +138,13 @@ module FFWD
     plugins = {}
 
     plugins[:tunnel] = FFWD::Plugin.load_plugins(
-      log, "Tunnel", config[:tunnel], :tunnel)
+      log, "Tunnel", config[:tunnel], :input, :tunnel)
 
     plugins[:input] = FFWD::Plugin.load_plugins(
-      log, "Input", config[:input], :input)
+      log, "Input", config[:input], :input, :bind)
 
     plugins[:output] = FFWD::Plugin.load_plugins(
-      log, "Output", config[:output], :output)
+      log, "Output", config[:output], :output, :connect)
 
     plugins
   end
@@ -219,8 +225,24 @@ module FFWD
     puts "    https://github.com/spotify/ffwd"
   end
 
+  def self.match_any_config_path prefixes, path
+    return path if File.file? path
+
+    prefixes.each do |prefix|
+      full = File.join prefix, path
+      next unless File.file? full
+      return full
+    end
+
+    return nil
+  end
+
   def self.main args
-    parse_options args
+    positional = parse_options(args)
+
+    positional.each do |path|
+      opts[:config_paths] << path
+    end
 
     if opts[:show_version]
       puts "ffwd version: #{FFWD::VERSION}"
@@ -232,7 +254,9 @@ module FFWD
     config = {}
 
     opts[:config_paths].each do |path|
-      unless File.file? path
+      path = match_any_config_path opts[:config_prefixes], path
+
+      unless path
         puts "Configuration path does not exist: #{path}"
         puts ""
         puts parser.help
@@ -240,7 +264,7 @@ module FFWD
       end
 
       return 0 unless source = load_yaml(path)
-
+      puts "Loaded: #{path}"
       merge_configurations config, source
     end
 

data/lib/ffwd/core.rb

@@ -23,7 +23,6 @@ require_relative 'logging'
 require_relative 'plugin_channel'
 require_relative 'processor'
 require_relative 'protocol'
-require_relative 'statistics'
 require_relative 'utils'
 
 require_relative 'core/emitter'
@@ -31,6 +30,8 @@ require_relative 'core/interface'
 require_relative 'core/processor'
 require_relative 'core/reporter'
 
+require_relative 'statistics/collector'
+
 module FFWD
   class Core
     include FFWD::Lifecycle
@@ -46,8 +47,8 @@ module FFWD
       @core_opts = opts[:core] || {}
       @processors = FFWD::Processor.load_processors(opts[:processor] || {})
 
-      @output_channel = FFWD::PluginChannel.build 'output'
-      @input_channel = FFWD::PluginChannel.build 'input'
+      @output_channel = FFWD::PluginChannel.build 'core.output'
+      @input_channel = FFWD::PluginChannel.build 'core.input'
 
       @system_channel = Channel.new log, "system_channel"
 
@@ -66,8 +67,8 @@ module FFWD
 
       if @debug_opts
         @debug = FFWD::Debug.setup @debug_opts
-        @debug.monitor "core.input", @input_channel, FFWD::Debug::Input
-        @debug.monitor "core.output", @output_channel, FFWD::Debug::Output
+        @debug.monitor @input_channel, FFWD::Debug::Input
+        @debug.monitor @output_channel, FFWD::Debug::Output
         @debug.depend_on self
       end
 
@@ -75,7 +76,8 @@ module FFWD
       @statistics = nil
 
       if config = @statistics_opts
-        @statistics = FFWD::Statistics.setup @emitter, @system_channel, config
+        @statistics = FFWD::Statistics::Collector.build(
+          @emitter, @system_channel, config)
         @statistics.depend_on self
       end
 
@@ -86,19 +88,19 @@ module FFWD
 
       @interface.depend_on self
 
-      @input_instances = @input_plugins.map do |plugin|
-        plugin.setup @interface
+      @input_instances = @input_plugins.map do |factory|
+        factory.call @interface
       end
 
-      @output_instances = @output_plugins.map do |plugin|
-        plugin.setup @interface
+      @output_instances = @output_plugins.map do |factory|
+        factory.call @interface
       end
 
       unless @statistics.nil?
         reporters = [@input_channel, @output_channel, @processor]
         reporters += @input_instances.select{|i| FFWD.is_reporter?(i)}
         reporters += @output_instances.select{|i| FFWD.is_reporter?(i)}
-        @statistics.register "core", Core::Reporter.new(reporters)
+        @statistics.register self, "core", Core::Reporter.new(reporters)
       end
 
       # Make the core-related channels depend on core.

data/lib/ffwd/core/emitter.rb

@@ -13,8 +13,6 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-require_relative '../logging'
-require_relative '../utils'
 require_relative '../metric_emitter'
 require_relative '../event_emitter'
 

data/lib/ffwd/core/interface.rb

@@ -25,8 +25,11 @@ module FFWD
     attr_reader :tunnel_plugins, :statistics, :debug, :processors
     attr_reader :tags, :attributes
 
-    def initialize(input, output, tunnel_plugins, statistics, debug,
-                   processors, opts)
+    def initialize(
+      input, output,
+      tunnel_plugins, statistics, debug,
+      processors, opts
+    )
       @input = input
       @output = output
       @tunnel_plugins = tunnel_plugins

data/lib/ffwd/core/processor.rb

@@ -38,25 +38,24 @@ module FFWD
       @emitter = emitter
       @processors = processors
       @reporters = reporters
-
-      subs = []
+      @subs = []
 
       @processors.each do |name, p|
         p.depend_on input
       end
 
       input.starting do
-        subs << input.metric_subscribe do |m|
+        @subs << input.metric_subscribe do |m|
           process_metric m
         end
 
-        subs << input.event_subscribe do |e|
+        @subs << input.event_subscribe do |e|
           process_event e
         end
       end
 
       input.stopping do
-        subs.each(&:unsubscribe).clear
+        @subs.each(&:unsubscribe).clear
       end
     end
 

data/lib/ffwd/debug/monitor_session.rb

@@ -17,10 +17,7 @@ require_relative '../lifecycle'
 
 module FFWD::Debug
   class MonitorSession
-    attr_reader :id
-
-    def initialize id, channel, type
-      @id = id
+    def initialize channel, type
       @type = type
       @clients = {}
 
@@ -31,7 +28,7 @@ module FFWD::Debug
           data = @type.serialize_event event
 
           begin
-            send JSON.dump(:id => @id, :type => :event, :data => data)
+            send JSON.dump(:id => channel.id, :type => :event, :data => data)
           rescue => e
             log.error "Failed to serialize event", e
             return
@@ -42,7 +39,7 @@ module FFWD::Debug
           data = @type.serialize_metric metric
 
           begin
-            send JSON.dump(:id => @id, :type => :metric, :data => data)
+            send JSON.dump(:id => channel.id, :type => :metric, :data => data)
           rescue => e
             log.error "Failed to serialize metric", e
             return

data/lib/ffwd/debug/tcp.rb

@@ -31,14 +31,15 @@ module FFWD::Debug
       @host = host
       @port = port
       @peer = "#{@host}:#{@port}"
+      info = "tcp://#{@peer}"
 
       r = FFWD.retry :timeout => rebind_timeout do |attempt|
         EM.start_server @host, @port, Connection, self
-        log.info "Bind on tcp://#{@peer} (attempt #{attempt})"
+        log.info "Bind on #{info} (attempt #{attempt})"
       end
 
       r.error do |a, t, e|
-        log.error "Failed to bind tcp://#{@peer} (attempt #{a}), retry in #{t}s", e
+        log.warning "Bind on #{info} failed, retry ##{a} in #{t}s: #{e}"
       end
 
       r.depend_on self
@@ -61,21 +62,25 @@ module FFWD::Debug
     end
 
     # Setup monitor hooks for the specified input and output channel.
-    def monitor id, channel, type
-      if session = @sessions[id]
-        log.error "Session already monitored: #{id}"
-        return
-      end
+    def monitor channel, type
+      channel.starting do
+        if session = @sessions[channel.id]
+          log.error "Session already monitored: #{channel.id}"
+          return
+        end
 
-      session = @sessions[id] = MonitorSession.new id, channel, type
+        session = MonitorSession.new channel, type
 
-      # provide the session to the already connected clients.
-      @clients.each do |peer, client|
-        session.register peer, client
+        # provide the session to any already connected clients.
+        @clients.each do |peer, client|
+          session.register peer, client
+        end
+
+        @sessions[channel.id] = session
       end
 
       channel.stopping do
-        @sessions.delete id
+        @sessions.delete channel.id
       end
     end
   end

data/lib/ffwd/handler.rb

@@ -21,7 +21,7 @@ module FFWD
     attr_accessor :parent
 
     def self.new signature, parent, *args
-      instance = super(signature, *args)
+      instance = super(signature, parent, *args)
 
       instance.instance_eval do
         @parent = parent

data/lib/ffwd/plugin.rb

@@ -28,7 +28,6 @@ module FFWD
         @options = config[:options]
         @setup_input_method = load_method @mod, config[:setup_input_method_name]
         @setup_output_method = load_method @mod, config[:setup_output_method_name]
-        @setup_tunnel_method = load_method @mod, config[:setup_tunnel_method_name]
       end
 
       def capabilities
@@ -42,10 +41,6 @@ module FFWD
           capabilities << "output"
         end
 
-        if not @setup_tunnel_method.nil?
-          capabilities << "tunnel"
-        end
-
         return capabilities
       end
 
@@ -56,7 +51,6 @@ module FFWD
       def get(kind)
         return @setup_input_method if kind == :input
         return @setup_output_method if kind == :output
-        return @setup_tunnel_method if kind == :tunnel
         return nil
       end
 
@@ -69,16 +63,16 @@ module FFWD
     end
 
     class Setup
-      attr_reader :name, :config
+      attr_reader :config, :name
 
-      def initialize name, config, setup
-        @name = name
+      def initialize method, config, name
+        @method = method
         @config = config
-        @setup = setup
+        @name = name
       end
 
-      def setup *args
-        @setup.call @config, *args
+      def call *args
+        @method.call(*args)
       end
     end
 
@@ -100,7 +94,6 @@ module FFWD
 
         config[:setup_input_method_name] = (opts[:setup_input_method] || :setup_input)
         config[:setup_output_method_name] = (opts[:setup_output_method] || :setup_output)
-        config[:setup_tunnel_method_name] = (opts[:setup_tunnel_method] || :setup_tunnel)
 
         FFWD::Plugin.discovered[name] = config
       end
@@ -122,7 +115,7 @@ module FFWD
       FFWD::Plugin.discovered.clear
     end
 
-    def self.load_plugins log, kind_name, config, kind
+    def self.load_plugins log, kind_name, config, kind, m
       result = []
 
       return result if config.nil?
@@ -145,7 +138,19 @@ module FFWD
           next
         end
 
-        result << Setup.new(name, plugin_config, setup)
+        factory = setup.call Hash[plugin_config]
+
+        unless factory.respond_to? m
+          log.error "#{d}: Plugin '#{name}' does not support '#{m.to_s}'"
+          next
+        end
+
+        unless factory.respond_to? :config
+          log.error "#{d}: Plugin '#{name}' does not support 'config'"
+          next
+        end
+
+        result << Setup.new(factory.method(m), factory.config, name)
       end
 
       return result

data/lib/ffwd/plugin/json.rb

@@ -50,9 +50,7 @@ module FFWD::Plugin::JSON
         ])
     ]
 
-  class LineConnection < FFWD::Connection
-    include FFWD::Logging
-    include FFWD::Plugin::JSON::Connection
+  class LineConnection < FFWD::Plugin::JSON::Connection
     include EM::Protocols::LineText2
 
     def self.plugin_type
@@ -64,10 +62,7 @@ module FFWD::Plugin::JSON
     end
   end
 
-  class FrameConnection < FFWD::Connection
-    include FFWD::Logging
-    include FFWD::Plugin::JSON::Connection
-
+  class FrameConnection < FFWD::Plugin::JSON::Connection
     def self.plugin_type
       "json_frame_in"
     end
@@ -79,13 +74,18 @@ module FFWD::Plugin::JSON
 
   KINDS = {"frame" => FrameConnection, "line" => LineConnection}
 
-  def self.setup_input opts, core
-    kind = (opts[:kind] || DEFAULT_KIND).to_s
-    raise "No such kind: #{kind}" unless connection = KINDS[kind]
-    protocol = FFWD.parse_protocol opts[:protocol] || DEFAULT_PROTOCOL[kind]
+  def self.setup_input opts
     opts[:host] ||= DEFAULT_HOST
     opts[:port] ||= DEFAULT_PORT
 
+    kind = (opts[:kind] || DEFAULT_KIND).to_s
+
+    unless connection = KINDS[kind]
+      raise "No such protocol kind: #{kind}"
+    end
+
+    protocol = FFWD.parse_protocol opts[:protocol] || DEFAULT_PROTOCOL[kind]
+
     if connection == FrameConnection and protocol != FFWD::UDP
       log.warning "When using :frame kind, you should use the UDP protocol." +
                   " Not #{protocol.family.to_s.upcase}"
@@ -96,14 +96,6 @@ module FFWD::Plugin::JSON
                   "Not #{protocol.family.to_s.upcase}"
     end
 
-    protocol.bind opts, core, log, connection
-  end
-
-  def self.setup_tunnel opts, core, tunnel
-    protocol = FFWD.parse_protocol opts[:protocol] || "tcp"
-    kind = (opts[:kind] || DEFAULT_KIND).to_s
-    raise "No such kind: #{kind}" unless connection = KINDS[kind]
-    opts[:port] ||= DEFAULT_PORT
-    protocol.tunnel opts, core, tunnel, log, connection
+    protocol.bind opts, log, connection
   end
 end