ffwd 0.1.7 → 0.2.0

data/lib/ffwd/plugin/json/connection.rb

@@ -19,7 +19,9 @@ require 'ffwd/logging'
 require 'ffwd/connection'
 
 module FFWD::Plugin::JSON
-  module Connection
+  class Connection < FFWD::Connection
+    include FFWD::Logging
+
     EVENT_FIELDS = [
       ["key", :key],
       ["value", :value],
@@ -39,7 +41,7 @@ module FFWD::Plugin::JSON
       ["attributes", :attributes]
     ]
 
-    def initialize bind, core
+    def initialize bind, core, config
       @bind = bind
       @core = core
     end

data/lib/ffwd/plugin/log.rb

@@ -25,6 +25,8 @@ module FFWD::Plugin
     include FFWD::Plugin
     include FFWD::Logging
 
+    DEFAULT_PREFIX = ''
+
     register_plugin "log",
       :description => "A simple plugin that outputs to the primary log.",
       :options => [
@@ -33,8 +35,21 @@ module FFWD::Plugin
         ]),
       ]
 
-    def self.setup_output opts, core
-      Writer.new core, opts[:prefix]
+    class Setup
+      attr_reader :config
+
+      def initialize config
+        @config = config
+        @config[:prefix] ||= DEFAULT_PREFIX
+      end
+
+      def connect core
+        Writer.new core, @config[:prefix]
+      end
+    end
+
+    def self.setup_output config
+      Setup.new config
     end
   end
 end

data/lib/ffwd/plugin/log/writer.rb

@@ -25,6 +25,8 @@ module FFWD::Plugin::Log
       subs = []
 
       core.output.starting do
+        log.info "Started (prefix: '#{@p}')"
+
         subs << core.output.event_subscribe do |e|
           log.info "Event: #{@p}#{e.to_h}"
         end
@@ -35,6 +37,8 @@ module FFWD::Plugin::Log
       end
 
       core.output.stopping do
+        log.info "Stopped"
+
         subs.each(&:unsubscribe).clear
       end
     end

data/lib/ffwd/plugin_channel.rb

@@ -28,19 +28,19 @@ module FFWD
 
     setup_reporter :keys => [:metrics, :events]
 
-    attr_reader :reporter_meta, :events, :metrics, :name
+    attr_reader :id, :events, :metrics, :reporter_meta
 
-    def self.build name
-      events = FFWD::Channel.new log, "#{name}.events"
-      metrics = FFWD::Channel.new log, "#{name}.metrics"
-      new name, metrics, events
+    def self.build id
+      events = FFWD::Channel.new log, "#{id}.events"
+      metrics = FFWD::Channel.new log, "#{id}.metrics"
+      new id, metrics, events
     end
 
-    def initialize name, events, metrics
-      @name = name
+    def initialize id, events, metrics
+      @id = id
       @events = events
       @metrics = metrics
-      @reporter_meta = {:plugin_channel => @name, :type => "plugin_channel"}
+      @reporter_meta = {:plugin_channel => @id, :type => "plugin_channel"}
     end
 
     def event_subscribe &block

data/lib/ffwd/processor.rb

@@ -23,14 +23,14 @@ module FFWD::Processor
   class Setup
     attr_reader :name
 
-    def initialize name, klass, options
+    def initialize name, factory, options
       @name = name
-      @klass = klass
+      @factory = factory
       @options = options
     end
 
     def setup emitter
-      @klass.new emitter, @options
+      @factory.new emitter, @options
     end
   end
 
@@ -91,6 +91,13 @@ module FFWD::Processor
 
   # setup hash of processor setup classes.
   def self.load_processors config
-    registry.map{|name, klass| Setup.new name, klass, config[name] || {}}
+    registry.map do |name, klass|
+      unless klass.respond_to? :prepare
+        raise "Processor #{klass} does not have a 'prepare' method"
+      end
+
+      config = klass.prepare Hash[config[name] || {}]
+      Setup.new name, klass, config
+    end
   end
 end

data/lib/ffwd/processor/count.rb

@@ -34,16 +34,26 @@ module FFWD::Processor
       :keys => [:dropped, :received]
     )
 
-    def initialize emitter, opts={}
+    def self.prepare config
+      config[:cache_limit] ||= 1000
+      config[:timeout] ||= 300
+      config[:window] ||= 30
+      config
+    end
+
+    def initialize emitter, config={}
       @emitter = emitter
-      @cache_limit = opts[:cache_limit] || 1000
-      @timeout = opts[:timeout] || 300
-      @period = opts[:period] || 30
+
+      @cache_limit = config[:cache_limit]
+      @timeout = config[:timeout]
+      @window = config[:window]
+
       @cache = {}
       @timer = nil
 
       starting do
-        log.info "Starting count processor on a window of #{@period}s"
+        log.info "Started"
+        log.info "  config: #{config.inspect}"
       end
 
       stopping do
@@ -81,7 +91,7 @@ module FFWD::Processor
 
       log.debug "Starting timer"
 
-      @timer = EM::Timer.new(@period) do
+      @timer = EM::Timer.new(@window) do
         @timer = nil
         digest! Time.now
       end

data/lib/ffwd/processor/histogram.rb

@@ -49,13 +49,23 @@ module FFWD::Processor
     DEFAULT_MISSING = 0
 
     DEFAULT_PERCENTILES = {
-      :p50 => {:percentage => 0.50, :info => "50th"},
-      :p75 => {:percentage => 0.75, :info => "75th"},
-      :p95 => {:percentage => 0.95, :info => "95th"},
-      :p99 => {:percentage => 0.99, :info => "99th"},
-      :p999 => {:percentage => 0.999, :info => "99.9th"},
+      :p50 => {:q => 0.50, :info => "50th"},
+      :p75 => {:q => 0.75, :info => "75th"},
+      :p95 => {:q => 0.95, :info => "95th"},
+      :p99 => {:q => 0.99, :info => "99th"},
+      :p999 => {:q => 0.999, :info => "99.9th"},
     }
 
+    def self.prepare config
+      config[:window] ||= 10
+      config[:cache_limit] ||= 1000
+      config[:bucket_limit] ||= 10000
+      config[:precision] ||= 3
+      config[:missing] ||= DEFAULT_MISSING
+      config[:percentiles] ||= DEFAULT_PERCENTILES
+      config
+    end
+
     #
     # Options:
     #
@@ -66,22 +76,23 @@ module FFWD::Processor
     # :precision - Precision of emitted metrics.
     # :percentiles - Configuration hash of percentile metrics.
     # Structure:
-    #   {:p10 => {:info => "Some description", :percentage => 0.1}, ...}
-    def initialize emitter, opts={}
+    #   {:p10 => {:info => "Some description", :q => 0.1}, ...}
+    def initialize emitter, config={}
       @emitter = emitter
 
-      @window = opts[:window] || 10
-      @cache_limit = opts[:cache_limit] || 1000
-      @bucket_limit = opts[:bucket_limit] || 10000
-      @precision = opts[:precision] || 3
-      @missing = opts[:missing] || DEFAULT_MISSING
-      @percentiles = opts[:percentiles] || DEFAULT_PERCENTILES
+      @window = config[:window]
+      @cache_limit = config[:cache_limit]
+      @bucket_limit = config[:bucket_limit]
+      @precision = config[:precision]
+      @missing = config[:missing]
+      @percentiles = config[:percentiles]
 
       # Dropped values that would have gone into a bucket.
       @cache = {}
 
       starting do
-        log.info "Starting histogram processor on a window of #{@window}s"
+        log.info "Started"
+        log.info "  config: #{config.inspect}"
       end
 
       stopping do
@@ -130,7 +141,7 @@ module FFWD::Processor
       map = {}
 
       @percentiles.each do |k, v|
-        index = (total * v[:percentage]).ceil - 1
+        index = (total * v[:q]).ceil - 1
 
         if (c = map[index]).nil?
           info = "#{v[:info]} percentile"

data/lib/ffwd/processor/rate.rb

@@ -44,13 +44,22 @@ module FFWD::Processor
     # :ttl - Allowed age of items in cache in seconds.
     # If this is nil, items will never expire, so old elements will not be
     # expunged until data type is restarted.
-    def initialize emitter, opts={}
+    def self.prepare config={}
+      config[:precision] ||= 3
+      config[:cache_limit] ||= 10000
+      config[:min_age] ||= 0.5
+      config[:ttl] ||= 600
+      config
+    end
+
+    def initialize emitter, config={}
       @emitter = emitter
 
-      @precision = opts[:precision] || 3
-      @limit = opts[:cache_limit] || 10000
-      @min_age = opts[:min_age] || 0.5
-      @ttl = opts[:ttl] || 600
+      @precision = config[:precision]
+      @limit = config[:cache_limit]
+      @min_age = config[:min_age]
+      @ttl = config[:ttl]
+
       # keep a reference to the expire cache to prevent having to allocate it
       # all the time.
       @expire = Hash.new
@@ -58,8 +67,9 @@ module FFWD::Processor
       @cache = Hash.new
 
       starting do
-        log.info "Starting rate processor (ttl: #{@ttl})"
         @timer = EM.add_periodic_timer(@ttl){expire!} unless @ttl.nil?
+        log.info "Started"
+        log.info "  config: #{config.inspect}"
       end
 
       stopping do

data/lib/ffwd/protocol/tcp.rb

@@ -13,9 +13,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-require 'eventmachine'
-
-require_relative '../tunnel'
+require_relative '../tunnel/tcp'
 
 require_relative 'tcp/bind'
 require_relative 'tcp/plain_connect'
@@ -27,100 +25,61 @@ module FFWD::TCP
     :tcp
   end
 
-  # default amount of bytes that the outbound connection will allow in its
-  # application-level buffer.
-  DEFAULT_OUTBOUND_LIMIT = 2 ** 20
-  # default flush period, if non-zero will cause the connection to be buffered.
-  DEFAULT_FLUSH_PERIOD = 10
-  # defaults for buffered connections.
-  # maximum amount of events to buffer up.
-  DEFAULT_EVENT_LIMIT = 1000
-  # maximum amount of metrics to buffer up.
-  DEFAULT_METRIC_LIMIT = 10000
-  # percent of maximum events/metrics which will cause a flush.
-  DEFAULT_FLUSH_LIMIT = 0.8
-  # Default initial timeout when binding fails.
-  DEFAULT_REBIND_TIMEOUT = 10
+  class SetupOutput
+    attr_reader :config
 
-  # Establish an outbound tcp connection.
-  #
-  # opts - Option hash.
-  #   Expects the following keys.
-  #     :host - The host to connect to.
-  #     :port - The port to connect to.
-  #     :outbound_limit - The amount of bytes that are allowed to be pending in
-  #     the application level buffer for the connection to be considered
-  #     'writable'.
-  #     :flush_period - The period in which outgoing data is buffered. If this
-  #     is 0 no buffering will occur.
-  #   Reads the following keys if the connection is buffered.
-  #     :event_limit - The maximum amount of events the connection is allowed
-  #     to buffer up.
-  #     :metric_limimt - The maximum amount of metrics the connection is
-  #     allowed to buffer up.
-  #     :flush_limit - A percentage (0.0 - 1.0) indicating 'the percentage of
-  #     events/metrics' that is required for a flush to be forced.
-  #     If this percentage is reached, the connection will attempt to forcibly
-  #     flush all buffered events and metrics prior to the end of the flushing
-  #     period.
-  # core - The core interface associated with this connection.
-  # log - The logger to use for this connection.
-  # handler - An implementation of FFWD::Handler containing the connection
-  # logic.
-  # args - Arguments passed to the handler when a new instance is created.
-  def self.connect opts, core, log, handler, *args
-    raise "Missing required option :host" if (host = opts[:host]).nil?
-    raise "Missing required option :port" if (port = opts[:port]).nil?
+    def initialize config, log, handler
+      @config = Hash[config]
+      @log = log
+      @handler = handler
 
-    outbound_limit = opts[:outbound_limit] || DEFAULT_OUTBOUND_LIMIT
-    flush_period = opts[:flush_period] || DEFAULT_FLUSH_PERIOD
+      @config = Connection.prepare @config
 
-    connection = Connection.new log, host, port, handler, args, outbound_limit
+      if @config[:flush_period] == 0
+        @config = PlainConnect.prepare @config
+        @type = PlainConnect
+      else
+        @config = FlushingConnect.prepare @config
+        @type = FlushingConnect
+      end
+    end
 
-    if flush_period == 0
-      PlainConnect.new core, log, connection
-    else
-      event_limit = opts[:event_limit] || DEFAULT_EVENT_LIMIT
-      metric_limit = opts[:metric_limit] || DEFAULT_METRIC_LIMIT
-      flush_limit = opts[:flush_limit] || DEFAULT_FLUSH_LIMIT
+    def connect core
+      raise "Missing required option :host" if (host = @config[:host]).nil?
+      raise "Missing required option :port" if (port = @config[:port]).nil?
 
-      FlushingConnect.new(
-        core, log, connection,
-        flush_period, event_limit, metric_limit, flush_limit
-      )
+      c = Connection.new @log, host, port, @handler, @config
+      @type.new core, @log, c, @config
     end
   end
 
-  # Bind and listen for a TCP connection.
-  #
-  # opts - Option hash.
-  #   :host - The host to bind to.
-  #   :port - The port to bind to.
-  #   :rebind_timeout - The initial timeout to use when rebinding the
-  #   connection.
-  # core - The core interface associated with this connection.
-  # log - The logger to use for this connection.
-  # connection - An implementation of FFWD::Connection containing the
-  # connection logic.
-  # args - Arguments passed to the connection when a new instance is created.
-  def self.bind opts, core, log, connection, *args
-    raise "Missing required option :host" if (host = opts[:host]).nil?
-    raise "Missing required option :port" if (port = opts[:port]).nil?
-    rebind_timeout = opts[:rebind_timeout] || DEFAULT_REBIND_TIMEOUT
-    Bind.new core, log, host, port, connection, args, rebind_timeout
+  def self.connect config, log, handler
+    SetupOutput.new config, log, handler
+  end
+
+  class SetupInput
+    attr_reader :config
+
+    def initialize config, log, connection
+      @config = Hash[config]
+      @log = log
+      @connection = connection
+    end
+
+    def bind core
+      raise "Missing required option :host" if (host = @config[:host]).nil?
+      raise "Missing required option :port" if (port = @config[:port]).nil?
+      @config = Bind.prepare Hash[@config]
+      Bind.new core, @log, host, port, @connection, @config
+    end
+
+    def tunnel core, plugin
+      raise "Missing required option :port" if (port = @config[:port]).nil?
+      FFWD::Tunnel::TCP.new port, core, plugin, @log, @connection
+    end
   end
 
-  # Set up a TCP tunnel.
-  #
-  # opts - Option hash.
-  #   :port - The port to bind to on the remote side.
-  # core - The core interface associated with this connection.
-  # log - The logger to use for this connection.
-  # connection - An implementation of FFWD::Connection containing the
-  # connection logic.
-  # args - Arguments passed to the connection when a new instance is created.
-  def self.tunnel opts, core, plugin, log, connection, *args
-    raise "Missing required option :port" if (port = opts[:port]).nil?
-    FFWD.tunnel self.family, port, core, plugin, log, connection, args
+  def self.bind config, log, connection
+    SetupInput.new config, log, connection
   end
 end

data/lib/ffwd/protocol/tcp/bind.rb

@@ -22,42 +22,54 @@ module FFWD::TCP
   class Bind
     include FFWD::Reporter
 
+    # Default initial timeout when binding fails.
+    DEFAULT_REBIND_TIMEOUT = 10
+
+    def self.prepare opts
+      opts[:rebind_timeout] ||= DEFAULT_REBIND_TIMEOUT
+      opts
+    end
+
     setup_reporter :keys => [
       :received_events, :received_metrics,
       :failed_events, :failed_metrics
     ]
 
-    attr_reader :reporter_meta
+    attr_reader :log, :reporter_meta
 
-    def initialize core, log, host, port, connection, args, rebind_timeout
+    def initialize core, log, host, port, connection, config
+      @log = log
       @peer = "#{host}:#{port}"
       @reporter_meta = {
         :type => connection.plugin_type,
         :listen => @peer, :family => 'tcp'
       }
 
-      @sig = nil
+      @server = nil
 
       info = "tcp://#{@peer}"
+      rebind_timeout = config[:rebind_timeout]
 
       r = FFWD.retry :timeout => rebind_timeout do |a|
-        @sig = EM.start_server host, port, connection, self, core, *args
+        @server = EM.start_server host, port, connection, self, core, config
+
         log.info "Bind on #{info} (attempt #{a})"
+        log.info "  config: #{config.inspect}"
       end
 
       r.error do |a, t, e|
-        log.error "Failed to bind #{info} (attempt #{a}), retry in #{t}s", e
+        log.warning "Bind on #{info} failed, retry ##{a} in #{t}s: #{e}"
       end
 
       r.depend_on core
 
       core.stopping do
-        log.info "Unbinding #{info}"
-
-        if @sig
-          EM.stop_server @sig
-          @sig = nil
+        if @server
+          EM.stop_server @server
+          @server = nil
         end
+
+        log.info "Unbound #{info}"
       end
     end
   end