ffwd 0.3.7 → 0.4.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NDAxMmI5NWEwMzNhOTJjNmUyZTJiNDc5MGNhM2I3NWY1NWIwMjI2MQ==
-  data.tar.gz: !binary |-
-    ZDFjMDgxMTcyNzdkYmVmYWUxMzZlNDdmZjY1ZjUxYWE4ZTA3YjQwYg==
+SHA1:
+  metadata.gz: b64d17883b2270f2881852ba03b835186eb7b5b8
+  data.tar.gz: 73fb67085c33020061b6673a38bd253a672de0be
 SHA512:
-  metadata.gz: !binary |-
-    YjJjMTkyNzQ5YjMwMmI5YzZmNWM0ZTgzNDkyYzAxOTgxMzA3NmFjYzkzYzNj
-    ZTNjMWI4ZjkyYzdlZmNlMWRiMDEyM2Q3YzMyMjJhNGJlNjYyMWI2NWQ0OTQ0
-    ZjIxYWVhOTczOGQ5ODBiNTk3ZTcyZjgwMGRjNmI0MWEzN2VhYjQ=
-  data.tar.gz: !binary |-
-    MzllOTIxOGM5YjJiYjY1NDM4NDA2Zjc0NjBhMjhjY2EyMTAyMTFkY2QxNjg3
-    YTc2MTdjOWZlYTU4MjZhZjFiNjNlN2M4NGI4NWY3M2M4ZDk5YzI3Y2E3Mjhi
-    YzgwYjQ2YTcyNzI1ZGRmN2QwNzBhYTNiMWJhYzRiNWZjZTNlMTQ=
+  metadata.gz: 06dc1d8011388e4d1a754e70bf5c40ec45b4037cd8c94a3327dec0a26217c65b1607d54b79df50f08f47d8e4b72fe4fb0c10f007f3ebbc0f14b95e7a2fe3aa77
+  data.tar.gz: eed054fc3a895e30a5351b3e96e2ecbd679036961b60ced3521c4a7dd4f3c4c81e44b716693e6024b74aa316e8d1a1aaafae1f68465666aa0c189f95974eafcc

data/lib/ffwd/core/processor.rb

@@ -59,9 +59,9 @@ module FFWD
       end
     end
 
-    def report!
+    def report!(diff)
       @reporters.each do |reporter|
-        reporter.report! do |d|
+        reporter.report!(diff) do |d|
           yield d
         end
       end

data/lib/ffwd/core/reporter.rb

@@ -21,9 +21,9 @@ module FFWD
       @reporters = reporters
     end
 
-    def report!
+    def report!(diff)
       @reporters.each do |reporter|
-        reporter.report! do |d|
+        reporter.report!(diff) do |d|
           yield d
         end
       end

data/lib/ffwd/flushing_output.rb

@@ -21,11 +21,11 @@ module FFWD
   class FlushingOutput
     include FFWD::Reporter
 
-    report_meta :direction => :out
+    report_meta :direction => :output
 
-    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
-    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
-    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+    report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
+    report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
+    report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
 
     attr_reader :log, :reporter_meta
 

data/lib/ffwd/plugin/log/writer.rb

@@ -13,6 +13,8 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
+require 'json'
+
 require_relative '../../logging'
 
 module FFWD::Plugin::Log
@@ -28,11 +30,11 @@ module FFWD::Plugin::Log
         log.info "Started (prefix: '#{@p}')"
 
         subs << core.output.event_subscribe do |e|
-          log.info "Event: #{@p}#{e.to_h}"
+          log.info "Event: #{@p}#{JSON.dump(e.to_h)}"
         end
 
         subs << core.output.metric_subscribe do |m|
-          log.info "Metric: #{@p}#{m.to_h}"
+          log.info "Metric: #{@p}#{JSON.dump(m.to_h)}"
         end
       end
 

data/lib/ffwd/plugin_channel.rb

@@ -26,9 +26,9 @@ module FFWD
     include FFWD::Reporter
     include FFWD::Logging
 
-    report_meta :component => :plugin_channel
-    report_key :metrics, :meta => {:what => :metrics, :unit => :metric}
-    report_key :events, :meta => {:what => :events, :unit => :event}
+    report_meta :component => :channel
+    report_key :metrics, :meta => {:what => "seen-metrics", :unit => :metric}
+    report_key :events, :meta => {:what => "seen-events", :unit => :event}
 
     attr_reader :id, :events, :metrics, :reporter_meta
 
@@ -42,7 +42,7 @@ module FFWD
       @id = id
       @events = events
       @metrics = metrics
-      @reporter_meta = {:plugin_channel => @id, :type => "plugin_channel"}
+      @reporter_meta = {:channel => @id}
     end
 
     def event_subscribe &block

data/lib/ffwd/producing_client.rb

@@ -32,34 +32,34 @@ module FFWD
     report_meta :component => :producing_client
 
     # number of events/metrics that we attempted to dispatch but failed.
-    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
-    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+    report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
+    report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
 
     # number of events/metrics that were dropped because the output buffers
     # are full.
-    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
-    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :event}
+    report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :event}
 
     # number of events/metrics successfully sent.
-    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
-    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :event}
+    report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
+    report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :event}
 
     # number of requests that take longer than the allowed period.
-    report_key :slow_requests, :meta => {:what => :slow_requests, :unit => :request}
+    report_key :slow_requests, :meta => {:what => "slow-requests", :unit => :request}
 
     def reporter_meta
       return {} if @producer_is_reporter
       @producer.class.reporter_meta.merge(@producer.reporter_meta)
     end
 
-    def report!
-      super do |m|
+    def report!(diff)
+      super(diff) do |m|
         yield m
       end
 
       return unless @producer_is_reporter
 
-      @producer.report! do |m|
+      @producer.report!(diff) do |m|
         yield m
       end
     end

data/lib/ffwd/protocol/tcp/bind.rb

@@ -30,10 +30,10 @@ module FFWD::TCP
       opts
     end
 
-    report_meta :protocol => :tcp, :direction => :in
+    report_meta :protocol => :tcp, :direction => :input
 
-    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
-    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
+    report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
+    report_key :received_events, :meta => {:what => "received-events", :unit => :event}
 
     report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
     report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}

data/lib/ffwd/protocol/tcp/flushing_connect.rb

@@ -29,18 +29,18 @@ module FFWD::TCP
     # maximum amount of metrics to buffer up.
     DEFAULT_METRIC_LIMIT = 10000
 
-    report_meta :protocol => :tcp, :direction => :out
+    report_meta :protocol => :tcp, :direction => :output
 
-    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
-    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
+    report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
 
-    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
-    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+    report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
+    report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
 
-    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
-    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+    report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
+    report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
 
-    report_key :forced_flush, :meta => {:what => :forced_flush, :unit => :flush}
+    report_key :forced_flush, :meta => {:what => "forced-flush", :unit => :flush}
 
     attr_reader :log
 

data/lib/ffwd/protocol/tcp/plain_connect.rb

@@ -25,16 +25,16 @@ module FFWD::TCP
       opts
     end
 
-    report_meta :protocol => :tcp, :direction => :out
+    report_meta :protocol => :tcp, :direction => :output
 
-    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
-    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
+    report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
 
-    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
-    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+    report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
+    report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
 
-    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
-    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+    report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
+    report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
 
     def reporter_meta
       @c.reporter_meta

data/lib/ffwd/protocol/udp/bind.rb

@@ -32,13 +32,13 @@ module FFWD::UDP
       opts
     end
 
-    report_meta :protocol => :udp, :direction => :out
+    report_meta :protocol => :udp, :direction => :output
 
-    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
-    report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
+    report_key :received_events, :meta => {:what => "received-events", :unit => :event}
+    report_key :received_metrics, :meta => {:what => "received-metrics", :unit => :metric}
 
-    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
-    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+    report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
+    report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
 
     attr_reader :reporter_meta, :log, :config
 

data/lib/ffwd/protocol/udp/connect.rb

@@ -29,13 +29,13 @@ module FFWD::UDP
 
     attr_reader :reporter_meta, :log, :config
 
-    report_meta :protocol => :udp, :direction => :out
+    report_meta :protocol => :udp, :direction => :output
 
-    report_key :sent_events, :meta => {:what => :sent_events, :unit => :event}
-    report_key :sent_metrics, :meta => {:what => :sent_metrics, :unit => :metric}
+    report_key :sent_events, :meta => {:what => "sent-events", :unit => :event}
+    report_key :sent_metrics, :meta => {:what => "sent-metrics", :unit => :metric}
 
-    report_key :dropped_events, :meta => {:what => :dropped_events, :unit => :event}
-    report_key :dropped_metrics, :meta => {:what => :dropped_metrics, :unit => :metric}
+    report_key :dropped_events, :meta => {:what => "dropped-events", :unit => :event}
+    report_key :dropped_metrics, :meta => {:what => "dropped-metrics", :unit => :metric}
 
     def initialize core, log, host, port, handler, config
       @log = log

data/lib/ffwd/reporter.rb

@@ -25,7 +25,9 @@ module FFWD::Reporter
       meta = meta.merge(instance.send(:reporter_meta))
     end
 
-    return meta.merge(k[:meta])
+    meta = meta.merge(k[:meta])
+    meta[:unit] = "#{meta[:unit] || 'n'}/s"
+    return meta
   end
 
   module ClassMethods
@@ -71,15 +73,16 @@ module FFWD::Reporter
     reporter_data[n] += c
   end
 
-  def report!
+  def report! diff
     self.class.reporter_keys.each do |key|
       k = key[:key]
       v = reporter_data[k]
       reporter_data[k] = 0
 
       meta = ((@_reporter_meta ||= {})[k] ||= FFWD::Reporter.build_meta(self, key))
+      rate = (v / diff).round(3)
 
-      yield(:key => k, :value => v, :meta => meta)
+      yield(:key => k, :value => rate, :meta => meta)
     end
   end
 end

data/lib/ffwd/statistics/collector.rb

@@ -78,9 +78,12 @@ module FFWD::Statistics
     end
 
     def generate! last, now
+      diff = now - last
+      return if diff <= 0
+
       if @system
-        @system.collect @channel do |key, value|
-          attributes = FFWD.merge_hashes @attributes, {:what => key, :component => :system}
+        @system.collect @channel do |key, unit, value|
+          attributes = FFWD.merge_hashes @attributes, {:what => key, :unit => unit, :component => :system}
           @emitter.metric.emit(
             :key => @prefix, :value => value,
             :tags => @tags, :attributes => attributes)
@@ -88,7 +91,7 @@ module FFWD::Statistics
       end
 
       @reporters.each do |id, reporter|
-        reporter.report! do |d|
+        reporter.report!(diff) do |d|
           attributes = FFWD.merge_hashes @attributes, d[:meta]
           @emitter.metric.emit(
             :key => @prefix, :value => d[:value],

data/lib/ffwd/statistics/system_statistics.rb

@@ -99,21 +99,18 @@ module FFWD::Statistics
     end
 
     def collect channel
-      memory_use = memory_usage
-      cpu_use = cpu_usage
+      cpu = cpu_use
+      memory = memory_use
 
-      cpu_use.each do |key, value|
-        yield "statistics.cpu-#{key}", value
+      cpu.each do |key, value|
+        yield "cpu-#{key}", "%", value
       end
 
-      memory_use.each do |key, value|
-        yield "statistics.memory-#{key}", value
+      memory.each do |key, value|
+        yield "memory-#{key}", "B", value
       end
 
-      channel << {
-        :cpu => cpu_use,
-        :memory => memory_use,
-      }
+      channel << {:memory => memory}
     end
 
     def check
@@ -140,7 +137,7 @@ module FFWD::Statistics
       return true
     end
 
-    def memory_usage
+    def memory_use
       result = {:resident => 0, :total => read_total_memory}
 
       read_smaps do |smap|
@@ -150,7 +147,7 @@ module FFWD::Statistics
       result
     end
 
-    def cpu_usage
+    def cpu_use
       stat = read_pid_stat
 
       current = {
@@ -167,11 +164,15 @@ module FFWD::Statistics
         @cpu_prev = current
       end
 
-      return {
-        :system => current[:system] - prev[:system],
-        :user => current[:user] - prev[:user],
-        :total => current[:total] - prev[:total],
-      }
+      total = current[:total] - prev[:total]
+      system = current[:system] - prev[:system]
+      user = current[:user] - prev[:user]
+
+      if total == 0
+        return {:user => 0, :system => 0}
+      end
+
+      {:user => (user / total).round(3), :system => (system / total).round(3)}
     end
 
     private

data/lib/ffwd/tunnel/tcp.rb

@@ -23,11 +23,11 @@ module FFWD::Tunnel
 
     report_meta :protocol => :tunnel_tcp
 
-    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
-    report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
+    report_key :received_events, :meta => {:what => "received-events", :unit => :event}
+    report_key :received_metrics, :meta => {:what => "received-metrics", :unit => :metric}
 
-    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
-    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+    report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
+    report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
 
     attr_reader :log
 

data/lib/ffwd/tunnel/udp.rb

@@ -23,11 +23,11 @@ module FFWD::Tunnel
 
     report_meta :protocol => :tunnel_udp
 
-    report_key :received_events, :meta => {:what => :received_events, :unit => :event}
-    report_key :received_metrics, :meta => {:what => :received_metrics, :unit => :metric}
+    report_key :received_events, :meta => {:what => "received-events", :unit => :event}
+    report_key :received_metrics, :meta => {:what => "received-metrics", :unit => :metric}
 
-    report_key :failed_events, :meta => {:what => :failed_events, :unit => :event}
-    report_key :failed_metrics, :meta => {:what => :failed_metrics, :unit => :metric}
+    report_key :failed_events, :meta => {:what => "failed-events", :unit => :event}
+    report_key :failed_metrics, :meta => {:what => "failed-metrics", :unit => :metric}
 
     attr_reader :log
 

data/lib/ffwd/version.rb

@@ -14,5 +14,5 @@
 # the License.
 
 module FFWD
-  VERSION = "0.3.7"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ffwd
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.4.0
 platform: ruby
 authors:
 - John-John Tedro
@@ -9,62 +9,62 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-17 00:00:00.000000000 Z
+date: 2015-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.4
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: autoversion
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -81,18 +81,17 @@ files:
 - bin/fwc
 - lib/em/all.rb
 - lib/em/protocols/frame_object_protocol.rb
-- lib/ffwd.rb
 - lib/ffwd/channel.rb
 - lib/ffwd/connection.rb
-- lib/ffwd/core.rb
 - lib/ffwd/core/emitter.rb
 - lib/ffwd/core/interface.rb
 - lib/ffwd/core/processor.rb
 - lib/ffwd/core/reporter.rb
-- lib/ffwd/debug.rb
+- lib/ffwd/core.rb
 - lib/ffwd/debug/connection.rb
 - lib/ffwd/debug/monitor_session.rb
 - lib/ffwd/debug/tcp.rb
+- lib/ffwd/debug.rb
 - lib/ffwd/event.rb
 - lib/ffwd/event_emitter.rb
 - lib/ffwd/flushing_output.rb
@@ -102,32 +101,29 @@ files:
 - lib/ffwd/logging.rb
 - lib/ffwd/metric.rb
 - lib/ffwd/metric_emitter.rb
-- lib/ffwd/plugin.rb
-- lib/ffwd/plugin/json.rb
 - lib/ffwd/plugin/json/connection.rb
-- lib/ffwd/plugin/log.rb
+- lib/ffwd/plugin/json.rb
 - lib/ffwd/plugin/log/writer.rb
+- lib/ffwd/plugin/log.rb
+- lib/ffwd/plugin.rb
 - lib/ffwd/plugin_channel.rb
 - lib/ffwd/plugin_loader.rb
 - lib/ffwd/processor.rb
-- lib/ffwd/processor/count.rb
-- lib/ffwd/processor/histogram.rb
-- lib/ffwd/processor/rate.rb
 - lib/ffwd/producing_client.rb
-- lib/ffwd/protocol.rb
-- lib/ffwd/protocol/tcp.rb
 - lib/ffwd/protocol/tcp/bind.rb
 - lib/ffwd/protocol/tcp/connection.rb
 - lib/ffwd/protocol/tcp/flushing_connect.rb
 - lib/ffwd/protocol/tcp/plain_connect.rb
-- lib/ffwd/protocol/udp.rb
+- lib/ffwd/protocol/tcp.rb
 - lib/ffwd/protocol/udp/bind.rb
 - lib/ffwd/protocol/udp/connect.rb
+- lib/ffwd/protocol/udp.rb
+- lib/ffwd/protocol.rb
 - lib/ffwd/reporter.rb
 - lib/ffwd/retrier.rb
-- lib/ffwd/schema.rb
 - lib/ffwd/schema/default.rb
 - lib/ffwd/schema/spotify100.rb
+- lib/ffwd/schema.rb
 - lib/ffwd/statistics/collector.rb
 - lib/ffwd/statistics/system_statistics.rb
 - lib/ffwd/test/protocol.rb
@@ -136,6 +132,7 @@ files:
 - lib/ffwd/tunnel/udp.rb
 - lib/ffwd/utils.rb
 - lib/ffwd/version.rb
+- lib/ffwd.rb
 - lib/fwc.rb
 homepage: https://github.com/spotify/ffwd
 licenses:
@@ -147,17 +144,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Core framework for the FastForward Daemon.

data/lib/ffwd/processor/count.rb

@@ -1,130 +0,0 @@
-# $LICENSE
-# Copyright 2013-2014 Spotify AB. All rights reserved.
-#
-# The contents of this file are licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with the
-# License. You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-require_relative '../processor'
-require_relative '../logging'
-require_relative '../event'
-require_relative '../utils'
-require_relative '../reporter'
-
-module FFWD::Processor
-  #
-  # Implements counting statistics (similar to statsd).
-  #
-  class CountProcessor
-    include FFWD::Logging
-    include FFWD::Processor
-    include FFWD::Reporter
-
-    register_processor "count"
-
-    report_meta :component => :processor, :processor => :count
-    report_key :dropped, :meta => {:what => :count_dropped, :unit => :value}
-    report_key :received, :meta => {:what => :count_received, :unit => :value}
-
-    def self.prepare config
-      config[:cache_limit] ||= 1000
-      config[:timeout] ||= 300
-      config[:window] ||= 30
-      config
-    end
-
-    def initialize emitter, config={}
-      @emitter = emitter
-
-      @cache_limit = config[:cache_limit]
-      @timeout = config[:timeout]
-      @window = config[:window]
-
-      @cache = {}
-      @timer = nil
-
-      starting do
-        log.info "Started"
-        log.info "  config: #{config.inspect}"
-      end
-
-      stopping do
-        log.info "Stopping count processor"
-        @timer.cancel if @timer
-        @timer = nil
-      end
-    end
-
-    def flush_caches! now
-      @cache.each do |cache_key, entry|
-        key = entry[:key]
-        count = entry[:count]
-        last = entry[:last]
-
-        # OK to modify the hash being iterated over.
-        if now - last > @timeout
-          @cache.delete cache_key
-          next
-        end
-
-        attributes = entry[:attributes]
-        tags = entry[:tags]
-
-        entry[:count] = 0
-
-        @emitter.metric.emit(
-          :key => key, :value => count, :source => key,
-          :attributes => attributes, :tags => tags)
-      end
-    end
-
-    def check_timer
-      return if @timer
-
-      log.debug "Starting timer"
-
-      @timer = EM::Timer.new(@window) do
-        @timer = nil
-        digest! Time.now
-      end
-    end
-
-    def digest! now
-      ms = FFWD.timing do
-        flush_caches! now
-      end
-
-      log.debug "Digest took #{ms}ms"
-    end
-
-    def process m
-      key = m[:key]
-      value = m[:value]
-
-      now = Time.now
-
-      cache_key = [key, (m[:attributes] || {}), (m[:tags] || [])].hash
-
-      unless (entry = @cache[cache_key])
-        return increment :dropped if @cache.size >= @cache_limit
-        entry = @cache[cache_key] = {
-          :key => key,
-          :count => 0, :last => now, :tags => m[:tags],
-          :attributes => m[:attributes]}
-      end
-
-      increment :received
-      entry[:count] += value
-      entry[:last] = now
-      check_timer
-    end
-  end
-end

data/lib/ffwd/processor/histogram.rb

@@ -1,212 +0,0 @@
-# $LICENSE
-# Copyright 2013-2014 Spotify AB. All rights reserved.
-#
-# The contents of this file are licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with the
-# License. You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-require 'ffwd/event'
-require 'ffwd/logging'
-require 'ffwd/processor'
-require 'ffwd/reporter'
-require 'ffwd/utils'
-
-module FFWD::Processor
-  #
-  # Implements histogram statistics over a tumbling time window.
-  #
-  # Histogram received metrics continiuosly and regularly flushes out the
-  # following statistics.
-  #
-  # <key>.min - Min value collected.
-  # <key>.max - Max value collected.
-  # <key>.mean - Mean value collected.
-  # <key>.p50 - The 50th percentile value collected.
-  # <key>.p75 - The 75th percentile value collected.
-  # <key>.p95 - The 95th percentile value collected.
-  # <key>.p99 - The 99th percentile value collected.
-  # <key>.p999 - The 99.9th percentile value collected.
-  #
-  class HistogramProcessor
-    include FFWD::Processor
-    include FFWD::Logging
-    include FFWD::Reporter
-
-    register_processor "histogram"
-
-    report_meta :component => :processor, :processor => :histogram
-    report_key :dropped, {:meta => {:what => :dropped, :unit => :value}}
-    report_key :bucket_dropped, {:meta => {:what => :bucket_dropped, :unit => :value}}
-    report_key :received, {:meta => {:what => :received, :unit => :value}}
-
-    DEFAULT_MISSING = 0
-
-    DEFAULT_PERCENTILES = {
-      :p50 => {:q => 0.50, :info => "50th"},
-      :p75 => {:q => 0.75, :info => "75th"},
-      :p95 => {:q => 0.95, :info => "95th"},
-      :p99 => {:q => 0.99, :info => "99th"},
-      :p999 => {:q => 0.999, :info => "99.9th"},
-    }
-
-    def self.prepare config
-      config[:window] ||= 10
-      config[:cache_limit] ||= 1000
-      config[:bucket_limit] ||= 10000
-      config[:precision] ||= 3
-      config[:missing] ||= DEFAULT_MISSING
-      config[:percentiles] ||= DEFAULT_PERCENTILES
-      config
-    end
-
-    #
-    # Options:
-    #
-    # :window - Define at what period the cache is flushed and generates
-    # metrics.
-    # :cache_limit - Limit the amount of cache entries (by key).
-    # :bucket_limit - Limit the amount of limits for each cache entry.
-    # :precision - Precision of emitted metrics.
-    # :percentiles - Configuration hash of percentile metrics.
-    # Structure:
-    #   {:p10 => {:info => "Some description", :q => 0.1}, ...}
-    def initialize emitter, config={}
-      @emitter = emitter
-
-      @window = config[:window]
-      @cache_limit = config[:cache_limit]
-      @bucket_limit = config[:bucket_limit]
-      @precision = config[:precision]
-      @missing = config[:missing]
-      @percentiles = config[:percentiles]
-
-      # Dropped values that would have gone into a bucket.
-      @cache = {}
-
-      starting do
-        log.info "Started"
-        log.info "  config: #{config.inspect}"
-      end
-
-      stopping do
-        log.info "Stopping histogram processor"
-        @timer.cancel if @timer
-        @timer = nil
-        digest!
-      end
-    end
-
-    def check_timer
-      return if @timer
-
-      log.debug "Starting timer"
-
-      @timer = EM::Timer.new(@window) do
-        @timer = nil
-        digest!
-      end
-    end
-
-    # Digest the cache.
-    def digest!
-      if @cache.empty?
-        return
-      end
-
-      ms = FFWD.timing do
-        @cache.each do |key, bucket|
-          calculate(bucket) do |p, info, value|
-            @emitter.metric.emit(
-              :key => "#{key}.#{p}", :source => key,
-              :value => value, :description => "#{info} of #{key}")
-          end
-        end
-
-        @cache = {}
-      end
-
-      log.debug "Digest took #{ms}ms"
-    end
-
-    def calculate bucket
-      total = bucket.size
-
-      map = {}
-
-      @percentiles.each do |k, v|
-        index = (total * v[:q]).ceil - 1
-
-        if (c = map[index]).nil?
-          info = "#{v[:info]} percentile"
-          c = map[index] = {:info => info, :values => []}
-        end
-
-        c[:values] << {:name => k, :value => nil}
-      end
-
-      max = nil
-      min = nil
-      sum = 0.0
-      mean = nil
-
-      bucket.sort.each_with_index do |t, index|
-        max = t if max.nil? or t > max
-        min = t if min.nil? or t < min
-        sum += t
-
-        unless (c = map[index]).nil?
-          c[:values].each{|d| d[:value] = t}
-        end
-      end
-
-      mean = sum / total
-
-      unless @precision.nil?
-        max = max.round(@precision)
-        min = min.round(@precision)
-        sum = sum.round(@precision)
-        mean = mean.round(@precision)
-
-        map.each do |index, c|
-          c[:values].each{|d| d[:value] = d[:value].round(@precision)}
-        end
-      end
-
-      yield "max", "Max", max
-      yield "min", "Min", min
-      yield "sum", "Sum", sum
-      yield "mean", "Mean", mean
-
-      map.each do |index, c|
-        c[:values].each do |d|
-          yield d[:name], c[:info], d[:value]
-        end
-      end
-    end
-
-    def process m
-      key = m[:key]
-      value = m[:value] || @missing
-
-      if (bucket = @cache[key]).nil?
-        return increment :dropped if @cache.size >= @cache_limit
-        @cache[key] = bucket = []
-      end
-
-      return increment :bucket_dropped if bucket.size >= @bucket_limit
-      return increment :dropped if stopped?
-      increment :received
-
-      bucket << value
-      check_timer
-    end
-  end
-end

data/lib/ffwd/processor/rate.rb

@@ -1,127 +0,0 @@
-# $LICENSE
-# Copyright 2013-2014 Spotify AB. All rights reserved.
-#
-# The contents of this file are licensed under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with the
-# License. You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-require 'ffwd/processor'
-require 'ffwd/logging'
-require 'ffwd/reporter'
-
-module FFWD::Processor
-  #
-  # Implements rate statistics (similar to derive in collectd).
-  #
-  class RateProcessor
-    include FFWD::Logging
-    include FFWD::Processor
-    include FFWD::Reporter
-
-    register_processor "rate"
-
-    report_meta :component => :processor, :processor => :rate
-    report_key :dropped, {:meta => {:what => :dropped, :unit => :value}}
-    report_key :expired, {:meta => {:what => :expired, :unit => :value}}
-    report_key :received, {:meta => {:what => :received, :unit => :value}}
-
-    # Options:
-    #
-    # :precision - The precision to round to for emitted values.
-    # :cache_limit - Maxiumum amount of items allowed in the cache.
-    # :min_age - Minimum age that an item has to have in the cache to calculate
-    # rates.
-    # This exists to prevent too frequent updates to the cache which would
-    # yield very static or jumpy rates.
-    # Can be set to null to disable.
-    # :ttl - Allowed age of items in cache in seconds.
-    # If this is nil, items will never expire, so old elements will not be
-    # expunged until data type is restarted.
-    def self.prepare config={}
-      config[:precision] ||= 3
-      config[:cache_limit] ||= 10000
-      config[:min_age] ||= 0.5
-      config[:ttl] ||= 600
-      config
-    end
-
-    def initialize emitter, config={}
-      @emitter = emitter
-
-      @precision = config[:precision]
-      @limit = config[:cache_limit]
-      @min_age = config[:min_age]
-      @ttl = config[:ttl]
-
-      # keep a reference to the expire cache to prevent having to allocate it
-      # all the time.
-      @expire = Hash.new
-      # Cache of active events.
-      @cache = Hash.new
-
-      starting do
-        @timer = EM.add_periodic_timer(@ttl){expire!} unless @ttl.nil?
-        log.info "Started"
-        log.info "  config: #{config.inspect}"
-      end
-
-      stopping do
-        log.info "Stopping rate processor"
-        @timer.cancel if @timer
-      end
-    end
-
-    # Expire cached events that are too old.
-    def expire!
-      now = Time.new
-
-      @cache.each do |key, value|
-        diff = (now - value[:time])
-        next if diff < @ttl
-        @expire[key] = value
-      end
-
-      unless @expire.empty?
-        increment :expired, @cache.size - @expire.size
-        @cache = @expire
-        @expire = Hash.new
-      end
-    end
-
-    def process msg
-      key = msg[:key]
-      time = msg[:time]
-      value = msg[:value] || 0
-
-      unless (prev = @cache[key]).nil?
-        prev_time = prev[:time]
-        prev_value = prev[:value]
-
-        diff = (time - prev_time)
-
-        valid = @ttl.nil? or diff < @ttl
-        aged = @min_age.nil? or diff > @min_age
-
-        if diff > 0 and valid and aged
-          rate = ((value - prev_value) / diff)
-          rate = rate.round(@precision) unless @precision.nil?
-          @emitter.metric.emit(
-            :key => "#{key}.rate", :source => key, :value => rate)
-        end
-      else
-        return increment :dropped if @cache.size >= @limit
-      end
-
-      increment :received
-      @cache[key] = {:key => key, :time => time, :value => value}
-    end
-  end
-end