ffwd 0.1.7 → 0.2.0

data/lib/ffwd/protocol/tcp/connection.rb

@@ -13,19 +13,35 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
+require_relative '../../utils'
+
 module FFWD::TCP
   class Connection
     INITIAL_TIMEOUT = 2
 
+    # default flush period, if non-zero will cause the connection to be buffered.
+    DEFAULT_FLUSH_PERIOD = 10
+    # default amount of bytes that the outbound connection will allow in its
+    # application-level buffer.
+    DEFAULT_TCP_OUTBOUND_LIMIT = 2 ** 20
+
     attr_reader :log, :peer, :reporter_meta
 
-    def initialize log, host, port, handler, args, outbound_limit
+    def self.prepare opts
+      opts[:flush_period] ||= DEFAULT_FLUSH_PERIOD
+      opts[:tcp_outbound_limit] ||= DEFAULT_TCP_OUTBOUND_LIMIT
+      opts[:ignored] = (opts[:ignored] || []).map{|v| Utils.check_ignored v}
+      opts
+    end
+
+    def initialize log, host, port, handler, config
       @log = log
       @host = host
       @port = port
       @handler = handler
-      @args = args
-      @outbound_limit = outbound_limit
+      @config = config
+
+      @tcp_outbound_limit = config[:tcp_outbound_limit]
 
       @peer = "#{host}:#{port}"
       @closing = false
@@ -39,8 +55,9 @@ module FFWD::TCP
 
     # Start attempting to connect.
     def connect
-      log.info "Connecting to tcp://#{@host}:#{@port}"
-      @c = EM.connect @host, @port, @handler, self, *@args
+      @c = EM.connect @host, @port, @handler, self, @config
+      log.info "Connect to tcp://#{@host}:#{@port}"
+      log.info "  config: #{@config.inspect}"
     end
 
     # Explicitly disconnect and discard any reconnect attempts..
@@ -101,7 +118,7 @@ module FFWD::TCP
 
     # Check if a connection is writable or not.
     def writable?
-      not @c.nil? and @open and @c.get_outbound_data_size < @outbound_limit
+      not @c.nil? and @open and @c.get_outbound_data_size < @tcp_outbound_limit
     end
   end
 end

data/lib/ffwd/protocol/tcp/flushing_connect.rb

@@ -21,6 +21,14 @@ module FFWD::TCP
   class FlushingConnect
     include FFWD::Reporter
 
+    # percent of maximum events/metrics which will cause a flush.
+    DEFAULT_FORCED_FLUSH_FACTOR = 0.8
+    # defaults for buffered connections.
+    # maximum amount of events to buffer up.
+    DEFAULT_EVENT_LIMIT = 1000
+    # maximum amount of metrics to buffer up.
+    DEFAULT_METRIC_LIMIT = 10000
+
     setup_reporter :keys => [
       :dropped_events, :dropped_metrics,
       :sent_events, :sent_metrics,
@@ -34,49 +42,55 @@ module FFWD::TCP
       @c.reporter_meta
     end
 
-    def initialize(
-      core, log, connection,
-      flush_period, event_limit, metric_limit, flush_limit
-    )
+    def self.prepare opts
+      opts[:forced_flush_factor] ||= DEFAULT_FORCED_FLUSH_FACTOR
+      opts[:event_limit] ||= DEFAULT_EVENT_LIMIT
+      opts[:metric_limit] ||= DEFAULT_METRIC_LIMIT
+      opts
+    end
+
+    def initialize(core, log, connection, config)
       @log = log
       @c = connection
 
-      @flush_period = flush_period
-      @event_limit = event_limit
-      @event_flush_limit = flush_limit * event_limit
-      @metric_limit = metric_limit
-      @metric_flush_limit = flush_limit * metric_limit
+      flush_period = config[:flush_period]
+      ignored = config[:ignored]
+      forced_flush_factor = config[:forced_flush_factor]
+      event_limit = config[:event_limit]
+      metric_limit = config[:metric_limit]
 
       @event_buffer = []
       @metric_buffer = []
       @timer = nil
-
-      subs = []
+      @subs = []
 
       core.starting do
-        log.info "Flushing every #{@flush_period}s"
-        @timer = EM::PeriodicTimer.new(@flush_period){flush!}
+        @c.connect
 
-        event_consumer = setup_consumer(
-          @event_buffer, @event_limit, @event_flush_limit, :dropped_events)
-        metric_consumer = setup_consumer(
-          @metric_buffer, @metric_limit, @metric_flush_limit, :dropped_metrics)
+        @timer = EM::PeriodicTimer.new(flush_period){flush!}
 
-        subs << core.output.event_subscribe(&event_consumer)
-        subs << core.output.metric_subscribe(&metric_consumer)
+        unless ignored.include? :events
+          event_consumer = setup_consumer(
+            @event_buffer, forced_flush_factor, event_limit, :dropped_events)
+          @subs << core.output.event_subscribe(&event_consumer)
+        end
 
-        @c.connect
+        unless ignored.include? :metrics
+          metric_consumer = setup_consumer(
+            @metric_buffer, forced_flush_factor, metric_limit, :dropped_metrics)
+          @subs << core.output.metric_subscribe(&metric_consumer)
+        end
       end
 
       core.stopping do
-        subs.each(&:unsubscribe).clear
+        @c.disconnect
 
         if @timer
           @timer.cancel
           @timer = nil
         end
 
-        @c.disconnect
+        @subs.each(&:unsubscribe).clear
       end
     end
 
@@ -116,16 +130,19 @@ module FFWD::TCP
 
     private
 
-    def setup_consumer buffer, buffer_limit, flush_limit, statistics_key
+    def setup_consumer buffer, drop, flush, statistics_key
+      drop_limit = drop
+      forced_flush_limit = drop * flush
+
       proc do |e|
-        if buffer.size >= buffer_limit
+        if buffer.size >= drop_limit
           increment statistics_key, 1
           next
         end
 
         buffer << e
 
-        if buffer.size >= flush_limit
+        if buffer.size >= forced_flush_limit
           increment :forced_flush, 1
           flush!
         end

data/lib/ffwd/protocol/tcp/plain_connect.rb

@@ -19,6 +19,10 @@ module FFWD::TCP
   class PlainConnect
     include FFWD::Reporter
 
+    def self.prepare opts
+      opts
+    end
+
     setup_reporter :keys => [
       :dropped_events, :dropped_metrics,
       :sent_events, :sent_metrics,
@@ -33,16 +37,24 @@ module FFWD::TCP
 
     INITIAL_TIMEOUT = 2
 
-    def initialize core, log, connection
+    def initialize core, log, connection, config
       @log = log
       @c = connection
 
+      ignored = config[:ignored]
+
       subs = []
 
       core.starting do
         @c.connect
-        subs << core.output.event_subscribe{|e| handle_event e}
-        subs << core.output.metric_subscribe{|e| handle_metric e}
+
+        unless ignored.include? :events
+          subs << core.output.event_subscribe{|e| handle_event e}
+        end
+
+        unless ignored.include? :metrics
+          subs << core.output.metric_subscribe{|e| handle_metric e}
+        end
       end
 
       core.stopping do

data/lib/ffwd/protocol/udp.rb

@@ -13,36 +13,61 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-require 'eventmachine'
+require_relative '../utils'
+require_relative '../tunnel/udp'
 
 require_relative 'udp/connect'
 require_relative 'udp/bind'
 
-require_relative '../tunnel'
-
 module FFWD::UDP
   def self.family
     :udp
   end
 
-  DEFAULT_REBIND_TIMEOUT = 10
+  class SetupOutput
+    attr_reader :config
+
+    def initialize config, log, handler
+      @config = config
+      @log = log
+      @handler = handler
+      @config = Connect.prepare Hash[@config]
+    end
+
+    def connect core
+      raise "Missing required key :host" if (host = @config[:host]).nil?
+      raise "Missing required key :port" if (port = @config[:port]).nil?
+      Connect.new core, @log, host, port, @handler, @config
+    end
+  end
 
-  def self.connect opts, core, log, handler
-    raise "Missing required key :host" if (host = opts[:host]).nil?
-    raise "Missing required key :port" if (port = opts[:port]).nil?
-    Connect.new core, log, host, port, handler
+  def self.connect config, log, handler
+    SetupOutput.new config, log, handler
   end
 
-  def self.bind opts, core, log, connection, *args
-    raise "Missing required key :host" if (host = opts[:host]).nil?
-    raise "Missing required key :port" if (port = opts[:port]).nil?
-    rebind_timeout = opts[:rebind_timeout] || DEFAULT_REBIND_TIMEOUT
-    Bind.new core, log, host, port, connection, args, rebind_timeout
+  class SetupInput
+    attr_reader :config
+
+    def initialize config, log, connection
+      @config = config
+      @log = log
+      @connection = connection
+    end
+
+    def bind core
+      raise "Missing required key :host" if (host = @config[:host]).nil?
+      raise "Missing required key :port" if (port = @config[:port]).nil?
+      @config = Bind.prepare Hash[@config]
+      Bind.new core, @log, host, port, @connection, @config
+    end
+
+    def tunnel core, plugin
+      raise "Missing required key :port" if (port = @config[:port]).nil?
+      FFWD::Tunnel::UDP.new port, core, plugin, @log, @connection, @config
+    end
   end
 
-  def self.tunnel opts, core, plugin, log, connection, *args
-    raise "Missing required key :port" if (port = opts[:port]).nil?
-    FFWD.tunnel self.family, port, core, plugin, log, connection, args
+  def self.bind config, log, connection
+    SetupInput.new config, log, connection
   end
 end
-

data/lib/ffwd/protocol/udp/bind.rb

@@ -22,42 +22,53 @@ module FFWD::UDP
   class Bind
     include FFWD::Reporter
 
+    DEFAULT_REBIND_TIMEOUT = 10
+
+    def self.prepare opts
+      opts[:rebind_timeout] ||= DEFAULT_REBIND_TIMEOUT
+      opts
+    end
+
     setup_reporter :keys => [
       :received_events, :received_metrics,
       :failed_events, :failed_metrics
     ]
 
-    attr_reader :reporter_meta
+    attr_reader :reporter_meta, :log, :config
 
-    def initialize core, log, host, port, connection, args, rebind_timeout
+    def initialize core, log, host, port, connection, config
+      @log = log
       @peer = "#{host}:#{port}"
       @reporter_meta = {
         :type => connection.plugin_type,
         :listen => @peer, :family => 'udp'
       }
 
-      @sig = nil
+      rebind_timeout = config[:rebind_timeout]
+
+      @socket = nil
 
       info = "udp://#{@peer}"
 
       r = FFWD.retry :timeout => rebind_timeout do |a|
-        @sig = EM.open_datagram_socket host, port, connection, self, core, *args
+        @socket = EM.open_datagram_socket host, port, connection, self, core, config
         log.info "Bind on #{info} (attempt #{a})"
+        log.info "  config: #{config.inspect}"
       end
 
       r.error do |a, t, e|
-        log.error "Failed to bind #{info} (attempt #{a}), retry in #{t}s", e
+        log.warning "Bind on #{info} failed, retry ##{a} in #{t}s: #{e}"
       end
 
       r.depend_on core
 
       core.stopping do
-        log.info "Unbinding #{info}"
-
-        if @sig
-          @sig.unbind
-          @sig = nil
+        if @socket
+          @socket.unbind
+          @socket = nil
         end
+
+        log.info "Unbound #{info}"
       end
     end
   end

data/lib/ffwd/protocol/udp/connect.rb

@@ -22,19 +22,26 @@ module FFWD::UDP
 
     RESOLVE_TIMEOUT = 10
 
-    attr_reader :reporter_meta, :log
+    def self.prepare config
+      config[:ignored] = (config[:ignored] || []).map{|v| Utils.check_ignored v}
+      config
+    end
+
+    attr_reader :reporter_meta, :log, :config
 
     setup_reporter :keys => [
       :dropped_events, :dropped_metrics,
       :sent_events, :sent_metrics
     ]
 
-    def initialize core, log, host, port, handler
+    def initialize core, log, host, port, handler, config
       @log = log
       @host = host
       @port = port
       @handler = handler
 
+      ignored = config[:ignored]
+
       @bind_host = "0.0.0.0"
       @host_ip = nil
       @c = nil
@@ -45,7 +52,7 @@ module FFWD::UDP
 
       info = "udp://#{@peer}"
 
-      subs = []
+      @subs = []
 
       r = FFWD.retry :timeout => RESOLVE_TIMEOUT do |a|
         unless @host_ip
@@ -53,16 +60,22 @@ module FFWD::UDP
           raise "Could not resolve: #{@host}" if @host_ip.nil?
         end
 
-        @c = EM.open_datagram_socket(@bind_host, nil, @handler, self)
+        @c = EM.open_datagram_socket @bind_host, nil, @handler, self, config
 
-        log.info "Setup of output to #{info} successful"
+        unless ignored.include? :events
+          @subs << core.output.event_subscribe{|e| handle_event e}
+        end
+
+        unless ignored.include? :metrics
+          @subs << core.output.metric_subscribe{|m| handle_metric m}
+        end
 
-        subs << core.output.event_subscribe{|e| handle_event e}
-        subs << core.output.metric_subscribe{|m| handle_metric m}
+        log.info "Connect to #{info} (attempt #{a})"
+        log.info "  config: #{config.inspect}"
       end
 
       r.error do |a, t, e|
-        log.error "Setup of output to #{info} failed (attempt #{a}), retry in #{t}s", e
+        log.warning "Connect to #{info} failed, retry ##{a} in #{t}s: #{e}"
       end
 
       r.depend_on core
@@ -73,7 +86,7 @@ module FFWD::UDP
           @c = nil
         end
 
-        subs.each(&:unsubscribe).clear
+        @subs.each(&:unsubscribe).clear
       end
     end
 

data/lib/ffwd/retrier.rb

@@ -22,10 +22,13 @@ module FFWD
   class Retrier
     include FFWD::Lifecycle
 
+    MAX_FACTOR = 7
+
     def initialize timeout, &block
       @block = block
       @timer = nil
       @timeout = timeout
+      @max_timeout = timeout * 2**MAX_FACTOR
       @current_timeout = @timeout
       @attempt = 0
       @error_callbacks = []
@@ -56,7 +59,7 @@ module FFWD
       end
 
       @timer = EM::Timer.new(@current_timeout) do
-        @current_timeout *= 2
+        @current_timeout *= 2 unless @current_timeout >= @max_timeout
         @timer = nil
         try_block
       end