ffwd 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c295242d5e946456b427bee9e90460f9ec0dd071
-  data.tar.gz: dc6f89b142d7f5446a01682e347e784ef1641f58
+  metadata.gz: 69e61111c5686efc942cdde2b5478c95b5575fe5
+  data.tar.gz: 038e9a560c4388d42726684edafad3b0f605fd27
 SHA512:
-  metadata.gz: 118a3cadbe8cb5530a869184f55e3787d3247a7a37b34bcb11be05e6dcd45c39505ffa214018a3e9e858c589ce7affa3e102b99e4aa316bfd23ad243a8b37887
-  data.tar.gz: 237ca9680135e557dc107ae47774c55e798a1b0fc4ac351ec3efdb8f3b06b1378239a92d7c41e88b3244333af3528d6d3e919bcb56ef7b171ad3aff67fbd30c7
+  metadata.gz: 4c534a19834fb159a982058bb2f9f8c2b80a409a2e01f385f3ec8c4c86e788b05bff2faa170338e739c49021ad5e5d7179f4f47881aacee8c0415c5099b6497a
+  data.tar.gz: c579ee941ace40659f19faef1e5e911be655746ecace89126f6e268672b1530f7d800337c74547679df6c2a6717105282cb41c76b0f53dd7b81c416f78dfcb6a

data/lib/ffwd/flushing_output.rb

@@ -0,0 +1,149 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/reporter'
+
+require_relative 'utils'
+
+module FFWD
+  class FlushingOutput
+    include FFWD::Reporter
+
+    report_meta :direction => :out
+
+    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
+    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+
+    attr_reader :log, :reporter_meta
+
+    def initialize core, log, hook, config
+      @log = log
+      @flush_interval = config[:flush_interval]
+      @buffer_limit = config[:buffer_limit]
+      @hook = hook
+      @reporter_meta = @hook.reporter_meta
+
+      @buffer = []
+      @pending = nil
+      @c = nil
+
+      @sub = nil
+
+      core.starting do
+        @log.info "Started"
+        @log.info "  config: #{config}"
+
+        @hook.connect
+
+        @sub = core.output.metric_subscribe do |metric|
+          if @buffer.size >= @buffer_limit
+            increment :dropped_metrics, 1
+            next
+          end
+
+          @buffer << metric
+          check_timer!
+        end
+      end
+
+      core.stopping do
+        @log.info "Stopped"
+
+        @hook.close
+
+        if @sub
+          @sub.unsubscribe
+          @sub = nil
+        end
+
+        if @timer
+          @timer.cancel
+          @timer = nil
+        end
+      end
+    end
+
+    def flush!
+      if @timer
+        @timer.cancel
+        @timer = nil
+      end
+
+      if @pending
+        @log.info "Request already in progress, dropping metrics"
+        increment :dropped_metrics, @buffer.size
+        @buffer.clear
+        return
+      end
+
+      unless @hook.active?
+        @log.error "Dropping metrics, no active connection available"
+        increment :dropped_metrics, @buffer.size
+        @buffer.clear
+        return
+      end
+
+      buffer_size = @buffer.size
+
+      @pending = @hook.send @buffer
+
+      @pending.callback do
+        increment :sent_metrics, buffer_size
+        @pending = nil
+      end
+
+      @pending.errback do
+        @log.error "Failed to submit metrics: #{@pending.error}"
+        increment :failed_metrics, buffer_size
+        @pending = nil
+      end
+    rescue => e
+      @log.error "Error during flush", e
+    ensure
+      @buffer.clear
+    end
+
+    def check_timer!
+      return if @timer
+
+      @log.debug "Setting timer to #{@flush_interval}s"
+
+      @timer = EM::Timer.new(@flush_interval) do
+        flush!
+      end
+    end
+
+    class Setup
+      attr_reader :config
+
+      def initialize log, hook, config
+        @log = log
+        @hook = hook
+        @config = config
+      end
+
+      def connect core
+        FlushingOutput.new core, @log, @hook, @config
+      end
+    end
+  end
+
+  def self.flushing_output log, hook, config={}
+    raise "Expected: flush_interval" unless config[:flush_interval]
+    raise "Expected: buffer_limit" unless config[:buffer_limit]
+    FlushingOutput::Setup.new log, hook, config
+  end
+end

data/lib/ffwd/flushing_output_hook.rb

@@ -0,0 +1,48 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module FFWD
+  class FlushingOutputHook
+    # Establish connections.
+    def connect
+      raise "not implemented: connect"
+    end
+
+    # Close any open connections.
+    def close
+      raise "not implemented: close"
+    end
+
+    # Return true if connection is accessible, false otherwise.
+    def active?
+      raise "not implemented: active?"
+    end
+
+    # Send the specified batch of metrics.
+    #
+    # Must return a callback object with the following attributes.
+    # callback - That accepts a block that will be run on successful execution.
+    # errback - That accepts a block that will be run on failed execution.
+    # error - If errback has been triggered, should contain the error that
+    # occured.
+    def send metrics
+      raise "not implemented: send"
+    end
+
+    def reporter_meta
+      {}
+    end
+  end
+end

data/lib/ffwd/plugin_channel.rb

@@ -26,7 +26,9 @@ module FFWD
     include FFWD::Reporter
     include FFWD::Logging
 
-    setup_reporter :keys => [:metrics, :events]
+    report_meta :component => :plugin_channel
+    report_key :metrics, :meta => {:what => :metrics, :unit => :metric}
+    report_key :events, :meta => {:what => :events, :unit => :event}
 
     attr_reader :id, :events, :metrics, :reporter_meta
 

data/lib/ffwd/processor/count.rb

@@ -29,10 +29,10 @@ module FFWD::Processor
     include FFWD::Reporter
 
     register_processor "count"
-    setup_reporter(
-      :reporter_meta => {:processor => "count"},
-      :keys => [:dropped, :received]
-    )
+
+    report_meta :component => :processor, :processor => :count
+    report_key :dropped, :meta => {:what => :count_dropped, :unit => :value}
+    report_key :received, :meta => {:what => :count_received, :unit => :value}
 
     def self.prepare config
       config[:cache_limit] ||= 1000

data/lib/ffwd/processor/histogram.rb

@@ -41,10 +41,11 @@ module FFWD::Processor
     include FFWD::Reporter
 
     register_processor "histogram"
-    setup_reporter(
-      :reporter_meta => {:processor => "histogram"},
-      :keys => [:dropped, :bucket_dropped, :received]
-    )
+
+    report_meta :component => :processor, :processor => :histogram
+    report_key :dropped, {:meta => {:what => :dropped, :unit => :value}}
+    report_key :bucket_dropped, {:meta => {:what => :bucket_dropped, :unit => :value}}
+    report_key :received, {:meta => {:what => :received, :unit => :value}}
 
     DEFAULT_MISSING = 0
 

data/lib/ffwd/processor/rate.rb

@@ -27,10 +27,11 @@ module FFWD::Processor
     include FFWD::Reporter
 
     register_processor "rate"
-    setup_reporter(
-      :reporter_meta => {:processor => "rate"},
-      :keys => [:dropped, :expired, :received]
-    )
+
+    report_meta :component => :processor, :processor => :rate
+    report_key :dropped, {:meta => {:what => :dropped, :unit => :value}}
+    report_key :expired, {:meta => {:what => :expired, :unit => :value}}
+    report_key :received, {:meta => {:what => :received, :unit => :value}}
 
     # Options:
     #

data/lib/ffwd/producing_client.rb

@@ -29,21 +29,27 @@ module FFWD
       def produce events, metrics; raise "not implemented: produce"; end
     end
 
-    setup_reporter :keys => [
-      # number of events/metrics that we attempted to dispatch but failed.
-      :failed_events, :failed_metrics,
-      # number of events/metrics that were dropped because the output buffers
-      # are full.
-      :dropped_events, :dropped_metrics,
-      # number of events/metrics successfully sent.
-      :sent_events, :sent_metrics,
-      # number of requests that take longer than the allowed period.
-      :slow_requests
-    ]
+    report_meta :component => :producing_client
+
+    # number of events/metrics that we attempted to dispatch but failed.
+    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
+    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+
+    # number of events/metrics that were dropped because the output buffers
+    # are full.
+    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :event}
+
+    # number of events/metrics successfully sent.
+    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
+    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :event}
+
+    # number of requests that take longer than the allowed period.
+    report_key :slow_requests, :meta => {:what => :slow_requests, :unit => :request}
 
     def reporter_meta
-      @reporter_meta ||= @producer.reporter_meta.merge(
-        :type => "producing_client_out")
+      return {} if @producer_is_reporter
+      @producer.class.reporter_meta.merge(@producer.reporter_meta)
     end
 
     def report!

data/lib/ffwd/protocol/tcp/bind.rb

@@ -30,20 +30,17 @@ module FFWD::TCP
       opts
     end
 
-    setup_reporter :keys => [
-      :received_events, :received_metrics,
-      :failed_events, :failed_metrics
-    ]
+    report_meta :protocol => :tcp, :direction => :in
+
+    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
+    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
 
     attr_reader :log, :reporter_meta
 
     def initialize core, log, host, port, connection, config
       @log = log
       @peer = "#{host}:#{port}"
-      @reporter_meta = {
-        :type => connection.plugin_type,
-        :listen => @peer, :family => 'tcp'
-      }
+      @reporter_meta = {:component => connection.plugin_type, :listen => @peer}
 
       @server = nil
 

data/lib/ffwd/protocol/tcp/connection.rb

@@ -46,7 +46,7 @@ module FFWD::TCP
       @peer = "#{host}:#{port}"
       @closing = false
       @reconnect_timeout = INITIAL_TIMEOUT
-      @reporter_meta = {:type => @handler.plugin_type, :peer => peer}
+      @reporter_meta = {:component => @handler.plugin_type, :peer => peer}
 
       @timer = nil
       @c = nil

data/lib/ffwd/protocol/tcp/flushing_connect.rb

@@ -29,12 +29,18 @@ module FFWD::TCP
     # maximum amount of metrics to buffer up.
     DEFAULT_METRIC_LIMIT = 10000
 
-    setup_reporter :keys => [
-      :dropped_events, :dropped_metrics,
-      :sent_events, :sent_metrics,
-      :failed_events, :failed_metrics,
-      :forced_flush
-    ]
+    report_meta :protocol => :tcp, :direction => :out
+
+    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
+
+    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
+    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+
+    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
+    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+
+    report_key :forced_flush, :meta => {:what => :forced_flush, :unit => :flush}
 
     attr_reader :log
 

data/lib/ffwd/protocol/tcp/plain_connect.rb

@@ -19,17 +19,22 @@ module FFWD::TCP
   class PlainConnect
     include FFWD::Reporter
 
+    attr_reader :log
+
     def self.prepare opts
       opts
     end
 
-    setup_reporter :keys => [
-      :dropped_events, :dropped_metrics,
-      :sent_events, :sent_metrics,
-      :failed_events, :failed_metrics
-    ]
+    report_meta :protocol => :tcp, :direction => :out
 
-    attr_reader :log
+    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
+
+    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
+    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+
+    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
+    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
 
     def reporter_meta
       @c.reporter_meta

data/lib/ffwd/protocol/udp/bind.rb

@@ -32,20 +32,20 @@ module FFWD::UDP
       opts
     end
 
-    setup_reporter :keys => [
-      :received_events, :received_metrics,
-      :failed_events, :failed_metrics
-    ]
+    report_meta :protocol => :udp, :direction => :out
+
+    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
+    report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
+
+    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
+    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
 
     attr_reader :reporter_meta, :log, :config
 
     def initialize core, log, host, port, connection, config
       @log = log
       @peer = "#{host}:#{port}"
-      @reporter_meta = {
-        :type => connection.plugin_type,
-        :listen => @peer, :family => 'udp'
-      }
+      @reporter_meta = {:type => connection.plugin_type, :listen => @peer}
 
       rebind_timeout = config[:rebind_timeout]
 

data/lib/ffwd/protocol/udp/connect.rb

@@ -29,10 +29,13 @@ module FFWD::UDP
 
     attr_reader :reporter_meta, :log, :config
 
-    setup_reporter :keys => [
-      :dropped_events, :dropped_metrics,
-      :sent_events, :sent_metrics
-    ]
+    report_meta :protocol => :udp, :direction => :out
+
+    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
+    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+
+    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
 
     def initialize core, log, host, port, handler, config
       @log = log
@@ -46,9 +49,7 @@ module FFWD::UDP
       @host_ip = nil
       @c = nil
       @peer = "#{host}:#{port}"
-      @reporter_meta = {
-        :type => @handler.plugin_type, :peer => @peer
-      }
+      @reporter_meta = {:component => @handler.plugin_type, :peer => @peer}
 
       info = "udp://#{@peer}"
 

data/lib/ffwd/reporter.rb

@@ -18,23 +18,44 @@ module FFWD::Reporter
     Hash[meta.map{|k, v| [k.to_s, v]}]
   end
 
+  def self.build_meta instance, k
+    meta = instance.class.reporter_meta || {}
+
+    if instance.respond_to?(:reporter_meta)
+      meta = meta.merge(instance.send(:reporter_meta))
+    end
+
+    return meta.merge(k[:meta])
+  end
+
   module ClassMethods
     def reporter_keys
-      @_reporter_keys ||= [:total]
+      @reporter_keys ||= []
     end
 
+    # Statically configured metadata.
     def reporter_meta
-      @_reporter_meta ||= nil
+      @reporter_meta ||= {}
     end
 
-    def reporter_meta_method
-      @_reporter_meta_method ||= :reporter_meta
+    # Configure either static or dynamic metadata.
+    # If a symbol is provided, it is assumed to be the name of the function
+    # that will be used to fetch metadata.
+    # If a Hash is provided, it will be assumed to be the static metadata.
+    def report_meta meta
+      unless meta.is_a? Hash
+        raise "Invalid meta: #{meta.inspect}"
+      end
+
+      @reporter_meta = meta
+    end
+
+    def report_key key, options={}
+      reporter_keys <<  {:key => key, :meta => options[:meta] || {}}
     end
 
     def setup_reporter opts={}
-      @_reporter_keys = [:total] + (opts[:keys] || [])
-      @_reporter_meta = opts[:reporter_meta]
-      @_reporter_meta_method = opts[:id_method] || :reporter_meta
+      raise "setup_reporter is deprecated, use (report_*) instead!"
     end
   end
 
@@ -43,23 +64,22 @@ module FFWD::Reporter
   end
 
   def reporter_data
-    @_reporter_keys ||= self.class.reporter_keys
-    @_reporter_data ||= Hash[@_reporter_keys.map{|k| [k, 0]}]
+    @_reporter_data ||= Hash[self.class.reporter_keys.map{|k| [k[:key], 0]}]
   end
 
   def increment n, c=1
     reporter_data[n] += c
-    reporter_data[:total] += c
   end
 
   def report!
-    @_reporter_meta ||= FFWD::Reporter.map_meta(
-      self.class.reporter_meta || send(self.class.reporter_meta_method))
-
-    reporter_data.each do |k, v|
-      yield(:key => k, :value => v,
-            :meta => @_reporter_meta)
+    self.class.reporter_keys.each do |key|
+      k = key[:key]
+      v = reporter_data[k]
       reporter_data[k] = 0
+
+      meta = ((@_reporter_meta ||= {})[k] ||= FFWD::Reporter.build_meta(self, key))
+
+      yield(:key => k, :value => v, :meta => meta)
     end
   end
 end

data/lib/ffwd/statistics/collector.rb

@@ -80,19 +80,18 @@ module FFWD::Statistics
     def generate! last, now
       if @system
         @system.collect @channel do |key, value|
-          key = "#{@prefix}.#{key}"
+          attributes = FFWD.merge_hashes @attributes, {:what => key, :component => :system}
           @emitter.metric.emit(
-            :key => key, :value => value,
-            :tags => @tags, :attributes => @attributes)
+            :key => @prefix, :value => value,
+            :tags => @tags, :attributes => attributes)
         end
       end
 
       @reporters.each do |id, reporter|
         reporter.report! do |d|
           attributes = FFWD.merge_hashes @attributes, d[:meta]
-          key = "#{@prefix}.#{d[:key]}"
           @emitter.metric.emit(
-            :key => key, :value => d[:value],
+            :key => @prefix, :value => d[:value],
             :tags => @tags, :attributes => attributes)
         end
       end

data/lib/ffwd/tunnel/tcp.rb

@@ -21,8 +21,13 @@ module FFWD::Tunnel
     include FFWD::Lifecycle
     include FFWD::Reporter
 
-    setup_reporter :keys => [
-      :received_events, :received_metrics, :failed_events, :failed_metrics]
+    report_meta :protocol => :tunnel_tcp
+
+    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
+    report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
+
+    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
+    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
 
     attr_reader :log
 

data/lib/ffwd/tunnel/udp.rb

@@ -21,8 +21,13 @@ module FFWD::Tunnel
     include FFWD::Lifecycle
     include FFWD::Reporter
 
-    setup_reporter :keys => [
-      :received_events, :received_metrics, :failed_events, :failed_metrics]
+    report_meta :protocol => :tunnel_udp
+
+    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
+    report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
+
+    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
+    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
 
     attr_reader :log
 

data/lib/ffwd/version.rb

@@ -14,5 +14,5 @@
 # the License.
 
 module FFWD
-  VERSION = "0.2.4"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ffwd
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - John-John Tedro
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-12 00:00:00.000000000 Z
+date: 2014-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -119,6 +119,8 @@ files:
 - lib/ffwd/core/reporter.rb
 - lib/ffwd/core/emitter.rb
 - lib/ffwd/handler.rb
+- lib/ffwd/flushing_output.rb
+- lib/ffwd/flushing_output_hook.rb
 - lib/ffwd/version.rb
 - lib/ffwd/core.rb
 - lib/ffwd/processor/rate.rb