ffwd 0.1.0

data/lib/ffwd/processor/histogram.rb

@@ -0,0 +1,200 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/event'
+require 'ffwd/logging'
+require 'ffwd/processor'
+require 'ffwd/reporter'
+require 'ffwd/utils'
+
+module FFWD::Processor
+  #
+  # Implements histogram statistics over a tumbling time window.
+  #
+  # Histogram received metrics continiuosly and regularly flushes out the
+  # following statistics.
+  #
+  # <key>.min - Min value collected.
+  # <key>.max - Max value collected.
+  # <key>.mean - Mean value collected.
+  # <key>.p50 - The 50th percentile value collected.
+  # <key>.p75 - The 75th percentile value collected.
+  # <key>.p95 - The 95th percentile value collected.
+  # <key>.p99 - The 99th percentile value collected.
+  # <key>.p999 - The 99.9th percentile value collected.
+  #
+  class HistogramProcessor
+    include FFWD::Processor
+    include FFWD::Logging
+    include FFWD::Reporter
+
+    register_processor "histogram"
+    setup_reporter(
+      :reporter_meta => {:processor => "histogram"},
+      :keys => [:dropped, :bucket_dropped, :received]
+    )
+
+    DEFAULT_MISSING = 0
+
+    DEFAULT_PERCENTILES = {
+      :p50 => {:percentage => 0.50, :info => "50th"},
+      :p75 => {:percentage => 0.75, :info => "75th"},
+      :p95 => {:percentage => 0.95, :info => "95th"},
+      :p99 => {:percentage => 0.99, :info => "99th"},
+      :p999 => {:percentage => 0.999, :info => "99.9th"},
+    }
+
+    #
+    # Options:
+    #
+    # :window - Define at what period the cache is flushed and generates
+    # metrics.
+    # :cache_limit - Limit the amount of cache entries (by key).
+    # :bucket_limit - Limit the amount of limits for each cache entry.
+    # :precision - Precision of emitted metrics.
+    # :percentiles - Configuration hash of percentile metrics.
+    # Structure:
+    #   {:p10 => {:info => "Some description", :percentage => 0.1}, ...}
+    def initialize emitter, opts={}
+      @emitter = emitter
+
+      @window = opts[:window] || 10
+      @cache_limit = opts[:cache_limit] || 1000
+      @bucket_limit = opts[:bucket_limit] || 10000
+      @precision = opts[:precision] || 3
+      @missing = opts[:missing] || DEFAULT_MISSING
+      @percentiles = opts[:percentiles] || DEFAULT_PERCENTILES
+
+      # Dropped values that would have gone into a bucket.
+      @cache = {}
+
+      starting do
+        log.info "Starting histogram processor on a window of #{@window}s"
+      end
+
+      stopping do
+        log.info "Stopping histogram processor"
+        @timer.cancel if @timer
+        @timer = nil
+        digest!
+      end
+    end
+
+    def check_timer
+      return if @timer
+
+      log.debug "Starting timer"
+
+      @timer = EM::Timer.new(@window) do
+        @timer = nil
+        digest!
+      end
+    end
+
+    # Digest the cache.
+    def digest!
+      if @cache.empty?
+        return
+      end
+
+      ms = FFWD.timing do
+        @cache.each do |key, bucket|
+          calculate(bucket) do |p, info, value|
+            @emitter.metric.emit(
+              :key => "#{key}.#{p}", :source => key,
+              :value => value, :description => "#{info} of #{key}")
+          end
+        end
+
+        @cache = {}
+      end
+
+      log.debug "Digest took #{ms}ms"
+    end
+
+    def calculate bucket
+      total = bucket.size
+
+      map = {}
+
+      @percentiles.each do |k, v|
+        index = (total * v[:percentage]).ceil - 1
+
+        if (c = map[index]).nil?
+          info = "#{v[:info]} percentile"
+          c = map[index] = {:info => info, :values => []}
+        end
+
+        c[:values] << {:name => k, :value => nil}
+      end
+
+      max = nil
+      min = nil
+      sum = 0.0
+      mean = nil
+
+      bucket.sort.each_with_index do |t, index|
+        max = t if max.nil? or t > max
+        min = t if min.nil? or t < min
+        sum += t
+
+        unless (c = map[index]).nil?
+          c[:values].each{|d| d[:value] = t}
+        end
+      end
+
+      mean = sum / total
+
+      unless @precision.nil?
+        max = max.round(@precision)
+        min = min.round(@precision)
+        sum = sum.round(@precision)
+        mean = mean.round(@precision)
+
+        map.each do |index, c|
+          c[:values].each{|d| d[:value] = d[:value].round(@precision)}
+        end
+      end
+
+      yield "max", "Max", max
+      yield "min", "Min", min
+      yield "sum", "Sum", sum
+      yield "mean", "Mean", mean
+
+      map.each do |index, c|
+        c[:values].each do |d|
+          yield d[:name], c[:info], d[:value]
+        end
+      end
+    end
+
+    def process m
+      key = m[:key]
+      value = m[:value] || @missing
+
+      if (bucket = @cache[key]).nil?
+        return increment :dropped if @cache.size >= @cache_limit
+        @cache[key] = bucket = []
+      end
+
+      return increment :bucket_dropped if bucket.size >= @bucket_limit
+      return increment :dropped if stopped?
+      increment :received
+
+      bucket << value
+      check_timer
+    end
+  end
+end

data/lib/ffwd/processor/rate.rb

@@ -0,0 +1,116 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/processor'
+require 'ffwd/logging'
+require 'ffwd/reporter'
+
+module FFWD::Processor
+  #
+  # Implements rate statistics (similar to derive in collectd).
+  #
+  class RateProcessor
+    include FFWD::Logging
+    include FFWD::Processor
+    include FFWD::Reporter
+
+    register_processor "rate"
+    setup_reporter(
+      :reporter_meta => {:processor => "rate"},
+      :keys => [:dropped, :expired, :received]
+    )
+
+    # Options:
+    #
+    # :precision - The precision to round to for emitted values.
+    # :cache_limit - Maxiumum amount of items allowed in the cache.
+    # :min_age - Minimum age that an item has to have in the cache to calculate
+    # rates.
+    # This exists to prevent too frequent updates to the cache which would
+    # yield very static or jumpy rates.
+    # Can be set to null to disable.
+    # :ttl - Allowed age of items in cache in seconds.
+    # If this is nil, items will never expire, so old elements will not be
+    # expunged until data type is restarted.
+    def initialize emitter, opts={}
+      @emitter = emitter
+
+      @precision = opts[:precision] || 3
+      @limit = opts[:cache_limit] || 10000
+      @min_age = opts[:min_age] || 0.5
+      @ttl = opts[:ttl] || 600
+      # keep a reference to the expire cache to prevent having to allocate it
+      # all the time.
+      @expire = Hash.new
+      # Cache of active events.
+      @cache = Hash.new
+
+      starting do
+        log.info "Starting rate processor (ttl: #{@ttl})"
+        @timer = EM.add_periodic_timer(@ttl){expire!} unless @ttl.nil?
+      end
+
+      stopping do
+        log.info "Stopping rate processor"
+        @timer.cancel if @timer
+      end
+    end
+
+    # Expire cached events that are too old.
+    def expire!
+      now = Time.new
+
+      @cache.each do |key, value|
+        diff = (now - value[:time])
+        next if diff < @ttl
+        @expire[key] = value
+      end
+
+      unless @expire.empty?
+        increment :expired, @cache.size - @expire.size
+        @cache = @expire
+        @expire = Hash.new
+      end
+    end
+
+    def process msg
+      key = msg[:key]
+      time = msg[:time]
+      value = msg[:value] || 0
+
+      unless (prev = @cache[key]).nil?
+        prev_time = prev[:time]
+        prev_value = prev[:value]
+
+        diff = (time - prev_time)
+
+        valid = @ttl.nil? or diff < @ttl
+        aged = @min_age.nil? or diff > @min_age
+
+        if diff > 0 and valid and aged
+          rate = ((value - prev_value) / diff)
+          rate = rate.round(@precision) unless @precision.nil?
+          @emitter.metric.emit(
+            :key => "#{key}.rate", :source => key, :value => rate)
+        end
+      else
+        return increment :dropped if @cache.size >= @limit
+      end
+
+      increment :received
+      @cache[key] = {:key => key, :time => time, :value => value}
+    end
+  end
+end

data/lib/ffwd/producing_client.rb

@@ -0,0 +1,181 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'lifecycle'
+require_relative 'reporter'
+require_relative 'logging'
+
+module FFWD
+  # A client implementation that delegates all work to other threads.
+  class ProducingClient
+    include FFWD::Reporter
+    include FFWD::Logging
+
+    class Producer
+      def setup; raise "not implemented: setup"; end
+      def teardown; raise "not implemented: teardown"; end
+      def produce events, metrics; raise "not implemented: produce"; end
+    end
+
+    setup_reporter :keys => [
+      # number of events/metrics that we attempted to dispatch but failed.
+      :failed_events, :failed_metrics,
+      # number of events/metrics that were dropped because the output buffers
+      # are full.
+      :dropped_events, :dropped_metrics,
+      # number of events/metrics successfully sent.
+      :sent_events, :sent_metrics,
+      # number of requests that take longer than the allowed period.
+      :slow_requests
+    ]
+
+    def reporter_meta
+      @reporter_meta ||= @producer.reporter_meta.merge(
+        :type => "producing_client_out")
+    end
+
+    def report!
+      super do |m|
+        yield m
+      end
+
+      return unless @producer_is_reporter
+
+      @producer.report! do |m|
+        yield m
+      end
+    end
+
+    def initialize channel, producer, flush_period, event_limit, metric_limit
+      @flush_period = flush_period
+      @event_limit = event_limit
+      @metric_limit = metric_limit
+
+      if @flush_period <= 0
+        raise "Invalid flush period: #{flush_period}"
+      end
+
+      @producer = producer
+      @producer_is_reporter = FFWD.is_reporter? producer
+
+      @events = []
+      @metrics = []
+
+      # Pending request.
+      @request = nil
+      @timer = nil
+
+      @subs = []
+
+      channel.starting do
+        @timer = EM::PeriodicTimer.new(@flush_period){safer_flush!}
+
+        @subs << channel.event_subscribe do |e|
+          if @events.size >= @event_limit
+            increment :dropped_events
+            next
+          end
+
+          @events << e
+        end
+
+        @subs << channel.metric_subscribe do |m|
+          if @metrics.size >= @metric_limit
+            increment :dropped_metrics
+            next
+          end
+
+          @metrics << m
+        end
+
+        @producer.setup
+      end
+
+      channel.stopping do
+        if @timer
+          @timer.cancel
+          @timer = nil
+        end
+
+        flush!
+
+        @subs.each(&:unsubscribe).clear
+
+        @metrics.clear
+        @events.clear
+
+        @producer.teardown
+      end
+    end
+
+    # Apply some heuristics to determine if we can 'ignore' the current flush
+    # to prevent loss of data.
+    #
+    # Checks that if a request is pending; we have not breached the limit of
+    # allowed events.
+    def safer_flush!
+      if @request
+        increment :slow_requests
+
+        ignore_flush = (
+          @events.size < @event_limit or
+          @metrics.size < @metric_limit)
+
+        return if ignore_flush
+      end
+
+      flush!
+    end
+
+    def flush!
+      if @request or not @request = @producer.produce(@events, @metrics)
+        increment :dropped_events, @events.size
+        increment :dropped_metrics, @metrics.size
+        return
+      end
+
+      @request.callback do
+        increment :sent_events, @events.size
+        increment :sent_metrics, @metrics.size
+        @request = nil
+      end
+
+      @request.errback do
+        increment :failed_events, @events.size
+        increment :failed_metrics, @metrics.size
+        @request = nil
+      end
+    rescue => e
+      increment :failed_events, @events.size
+      increment :failed_metrics, @metrics.size
+      log.error "Failed to produce", e
+    ensure
+      @events.clear
+      @metrics.clear
+    end
+  end
+
+  DEFAULT_FLUSH_PERIOD = 10
+  DEFAULT_EVENT_LIMIT = 10000
+  DEFAULT_METRIC_LIMIT = 10000
+  DEFAULT_FLUSH_SIZE = 1000
+
+  def self.producing_client channel, producer, opts
+    flush_period = opts[:flush_period] || DEFAULT_FLUSH_PERIOD
+    event_limit = opts[:event_limit] || DEFAULT_EVENT_LIMIT
+    metric_limit = opts[:metric_limit] || DEFAULT_METRIC_LIMIT
+    ProducingClient.new channel, producer, flush_period, event_limit, metric_limit
+  end
+end