fnordmetric 0.7.5 → 0.9.7

data/lib/fnordmetric/{inbound_stream.rb → acceptors/tcp_acceptor.rb}

@@ -1,9 +1,9 @@
-class FnordMetric::InboundStream < EventMachine::Connection
+class FnordMetric::TCPAcceptor < EventMachine::Connection
   @@opts = nil
 
   def self.start(opts)
     @@opts = opts
-    EM.start_server(*(opts[:inbound_stream] << self))
+    EM.start_server(*(opts[:listen] + [self]))
   end
 
   def self.options(opts)
@@ -30,17 +30,16 @@ class FnordMetric::InboundStream < EventMachine::Connection
   def push_next_event
     return true if @events.empty?
     @events_buffered -= 1
-    @api.event(@events.pop)
+    api.event(@events.pop)
     close_connection?
     EM.next_tick(&method(:push_next_event))
   end
 
   def close_connection?
-    @api.disconnect unless @streaming || (@events_buffered!=0)
+    #@backend.hangup unless @streaming || (@events_buffered!=0)
   end
 
   def post_init
-    @api = FnordMetric::API.new(@@opts)
     @events_buffered = 0
     @streaming = true
     @buffer = ""
@@ -51,4 +50,8 @@ class FnordMetric::InboundStream < EventMachine::Connection
     @streaming = false
     close_connection?
   end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
 end

data/lib/fnordmetric/{inbound_datagram.rb → acceptors/udp_acceptor.rb}

@@ -1,4 +1,4 @@
-class FnordMetric::InboundDatagram < EventMachine::Connection
+class FnordMetric::UDPAcceptor < EventMachine::Connection
 
   class << self
     attr_accessor :opts
@@ -6,7 +6,8 @@ class FnordMetric::InboundDatagram < EventMachine::Connection
 
   def self.start(opts)
     self.opts = opts
-    EM.open_datagram_socket(*(opts[:inbound_stream] << self << opts))
+
+    EM.open_datagram_socket(*(opts[:listen] << self << opts))
   end
 
   def receive_data(event)
@@ -16,20 +17,20 @@ class FnordMetric::InboundDatagram < EventMachine::Connection
 
   def push_next_event
     return true if events.empty?
-    api.event(@events.pop)
+    ev = @events.pop
+    api.event(ev)
     EM.next_tick(&method(:push_next_event))
   end
 
   def unbind
-    api.disconnect
-  end
-
-  def api
-    @api ||= FnordMetric::API.new(self.class.opts)
+    #backend.hangup
   end
 
   def events
     @events ||= []
   end
 
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
 end

data/lib/fnordmetric/api.rb

@@ -1,7 +1,8 @@
 class FnordMetric::API
+
   @@opts = nil
 
-  def initialize opts
+  def initialize(opts={})
     @@opts = FnordMetric.default_options(opts)
     connect
   end
@@ -32,7 +33,6 @@ class FnordMetric::API
 
   def push_event(event_id, event_data)
     prefix = @@opts[:redis_prefix]
-    @redis.hincrby "#{prefix}-testdata",          "events_received", 1
     @redis.hincrby "#{prefix}-stats",             "events_received", 1
     @redis.set     "#{prefix}-event-#{event_id}", event_data
     @redis.lpush   "#{prefix}-queue",             event_id

data/lib/fnordmetric/context.rb

@@ -2,6 +2,18 @@ class FnordMetric::Context
 
   include FnordMetric::GaugeModifiers
 
+  class Proxy
+
+    def initialize(_ref)
+      @ref = _ref
+    end
+
+    def method_missing(method, *args, &block)
+      @ref.dispatch(method, *args, &block)
+    end
+
+  end
+
   def initialize(opts, block)
     @block = block
     @opts = opts    
@@ -10,10 +22,31 @@ class FnordMetric::Context
   def call(event, redis)
     @redis = redis
     @event = event    
-    self.instance_eval(&@block)
+    proxy.instance_eval(&@block)
   rescue Exception => e
-    raise e  if ENV['FNORDMETRIC_ENV'] == 'test'
+    raise e if ENV['FNORDMETRIC_ENV'] == 'test'
     puts "error: #{e.message}"
+    puts e.backtrace.push("").join("\n") if ENV['FNORDMETRIC_ENV'] == 'dev'
+  end
+
+  def proxy
+    @proxy ||= Proxy.new(self)
+  end
+
+  def dispatch(method, *args, &block)
+    if args.size > 0 && @opts[:gauges][args[0]].try(:renderable?)
+      @opts[:gauges][args.delete_at(0)].execute(method, *args.unshift(self), &block)
+    else
+      send(method, *args, &block)
+    end
+  rescue Exception => e
+    raise e if ENV['FNORDMETRIC_ENV'] == 'test'
+    puts "error: #{e.message}"
+    puts e.backtrace.push("\n").join("\n") if ENV['FNORDMETRIC_ENV'] == 'dev'
+  end
+
+  def redis_exec(*args)
+    @redis.send(*args)
   end
 
 private
@@ -35,7 +68,7 @@ private
   end
 
 protected
-
+  
   def fetch_gauge(_gauge)
     _gauge.is_a?(FnordMetric::Gauge) ? _gauge : @opts[:gauges].fetch(_gauge)
   rescue
@@ -43,23 +76,9 @@ protected
   end
 
   def error!(msg)
-    FnordMetric.error!(msg)
+    FnordMetric.error(msg)
   end
 
-  def assure_two_dimensional!(gauge)
-    return true if gauge.two_dimensional?
-    error! "error: #{caller[0].split(" ")[-1]} can only be used with 2-dimensional gauges" 
-  end
-
-  def assure_three_dimensional!(gauge)
-    return true unless gauge.two_dimensional?
-    error! "error: #{caller[0].split(" ")[-1]} can only be used with 3-dimensional gauges" 
-  end
-
-  def assure_non_progressive!(gauge)
-    return true unless gauge.progressive?
-    error! "error: #{caller[0].split(" ")[-1]} can only be used with non-progressive gauges" 
-  end
 
 end
     

data/lib/fnordmetric/defaults.rb

@@ -0,0 +1,9 @@
+FnordMetric::COLORS = ["#4572a7", "#aa4643", "#89a54e", "#80699b", "#3d96ae", "#db843d"].reverse
+
+FnordMetric::TICKS = lambda{ |tick, span| [tick, 60, 300, 1200, 3600, 86400]
+  .select{ |t| (t >= tick) && ((span/t) > 5) }
+  .uniq }
+
+
+FnordMetric::DEFAULT_PROC = lambda{ |arg| }
+

data/lib/fnordmetric/ext.rb

@@ -1,3 +1,75 @@
+module Enumerable
+
+  def each_with_log(total = nil, &block)
+    log_every = ((total ||= self.count) / 150)
+    self.each_with_index do |item, index|
+      if index % log_every == 0
+        STDOUT.puts "#{index}/#{total} (#{((index/total.to_f)*100).to_i}%)" 
+      end
+      block.call(item, index)
+    end
+  end
+
+end
+
+module Haml::Filters::Gaugejs
+  include Haml::Filters::Base
+
+  def render(text)
+    "<FNORDMETRIC-GAUGEJS>#{text}</FNORDMETRIC-GAUGEJS>"
+  end
+end
+
 class Symbol
   alias :intern :to_sym
+end
+
+class Range
+
+  def size
+    self.last - self.first
+  end
+
+end
+
+class Array
+
+  def mean
+    return 0 if empty?
+    inject(&:+).to_f / size
+  end
+
+  alias :average :mean
+
+  def median
+    return 0 if empty?
+    (_sorted = self.dup.sort)[_sorted.size/2]
+  end
+
+  def range
+    return 0 if empty?
+    max - min
+  end
+
+  def mode
+    return 0 if empty?
+    inject({}){ |h,v| h[v] = h[v].to_i+1; h }.to_a
+    .sort{ |a,b| b.last <=> a.last }[0][0]
+  end
+  
+  def emtpy
+    self.size == 0
+  end
+
+end
+
+class Thin::Connection
+
+  alias :pre_process_orig :pre_process
+
+  def pre_process
+    @request.env['async.connection'] = self
+    pre_process_orig
+  end
+
 end

data/lib/fnordmetric/gauge.rb

@@ -1,6 +1,9 @@
 class FnordMetric::Gauge
   
   include FnordMetric::GaugeCalculations
+  include FnordMetric::GaugeModifiers
+  include FnordMetric::GaugeValidations
+  include FnordMetric::GaugeRendering
 
   def initialize(opts)
     opts.fetch(:key) && opts.fetch(:key_prefix)
@@ -8,11 +11,15 @@ class FnordMetric::Gauge
   end
 
   def tick
-    (@opts[:tick] || 3600).to_i
+    (@opts[:tick] || @opts[:resolution] || 3600).to_i
   end
 
-  def tick_at(time)    
-    (time/tick.to_f).floor*tick
+  def retention
+    tick * 10 # FIXPAUL!
+  end
+
+  def tick_at(time, _tick=tick)    
+    (time/_tick.to_f).floor*_tick
   end
 
   def name
@@ -22,6 +29,14 @@ class FnordMetric::Gauge
   def title
     @opts[:title] || name
   end
+
+  def group
+    @opts[:group] || "Gauges"
+  end
+
+  def key_nouns
+    @opts[:key_nouns] || ["Key", "Keys"]
+  end
   
   def key(_append=nil)
     [@opts[:key_prefix], "gauge", name, tick, _append].flatten.compact.join("-")
@@ -31,6 +46,14 @@ class FnordMetric::Gauge
     key([(progressive? ? :progressive : tick_at(_time).to_s), _append])
   end
 
+  def tick_keys(_range, _append=nil)
+    ticks_in(_range).map{ |_t| tick_key(_t, _append) }
+  end
+
+  def retention_key(_time, _append=nil)
+    key([tick_at(_time, retention).to_s, _append])
+  end
+
   def two_dimensional?
     !@opts[:three_dimensional]
   end
@@ -51,16 +74,20 @@ class FnordMetric::Gauge
     !!@opts[:average]
   end
 
-  def add_redis(_redis)
-    @opts[:redis] = _redis
+  def has_series?
+    false
   end
 
-  def ticks_in(r)
-    (((r.last-r.first)/tick.to_f).ceil+1).times.map{ |n| tick_at(r.first + tick*(n-1)) }
+  def redis
+    @redis ||= EM::Hiredis.connect(FnordMetric.options[:redis_url]) # FIXPAUL
   end
 
-  def values_in(range)
-    values_at(ticks_in(range))
+  def sync_redis
+    @sync_redis ||= FnordMetric.mk_redis # FIXPAUL
   end
 
-end
+  def error!(msg)
+    FnordMetric.error(msg)
+  end
+  
+end

data/lib/fnordmetric/gauge_calculations.rb

@@ -1,20 +1,35 @@
 module FnordMetric::GaugeCalculations
 
   @@avg_per_session_proc = proc{ |_v, _t|
-    (_v.to_f / (redis.get(tick_key(_t, :"sessions-count"))||1).to_i)
+    (_v.to_f / (sync_redis.get(tick_key(_t, :"sessions-count"))||1).to_i)
   }
 
   @@count_per_session_proc = proc{ |_v, _t|
-    (redis.get(tick_key(_t, :"sessions-count"))||0).to_i
+    (sync_redis.get(tick_key(_t, :"sessions-count"))||0).to_i
   }
 
   @@avg_per_count_proc = proc{ |_v, _t|
-    (_v.to_f / (redis.get(tick_key(_t, :"value-count"))||1).to_i)
+    (_v.to_f / (sync_redis.get(tick_key(_t, :"value-count"))||1).to_i)
   }
 
+  def ticks_in(r, _tick=tick, overflow=0)
+    (((r.last-r.first)/_tick.to_f).ceil+1+overflow).times.map{ |n| tick_at(r.first + _tick*(n-1), _tick) }
+  end
+
+  def values_in(range)
+    ticks = ticks_in(range)
+    ticks << tick_at(range.last) if ticks.size == 0
+    values_at(ticks)
+  end
+  
   def value_at(time, opts={}, &block)
     _t = tick_at(time)
-    _v = redis.hget(key, _t)
+
+    _v = if respond_to?(:_value_at)
+      _value_at(key, _t)
+    else
+      sync_redis.hget(key, _t)
+    end
 
     calculate_value(_v, _t, opts, block)
   end
@@ -22,17 +37,17 @@ module FnordMetric::GaugeCalculations
   def values_at(times, opts={}, &block)
     times = times.map{ |_t| tick_at(_t) }
     Hash.new.tap do |ret|
-      redis.hmget(key, *times).each_with_index do |_v, _n|
+      if respond_to?(:_values_at)
+        _values_at(times, opts={}, &block)
+      else
+        sync_redis.hmget(key, *times)
+      end.each_with_index do |_v, _n|
         _t = times[_n]
         ret[_t] = calculate_value(_v, _t, opts, block)
       end
     end
   end
 
-  def values_in(range, opts={}, &block)
-    values_at((tick_at(range.first)..range.last).step(tick))
-  end
-
   def calculate_value(_v, _t, opts, block)
     block = @@avg_per_count_proc if average?
     #block = @@count_per_session_proc if unique?
@@ -47,9 +62,9 @@ module FnordMetric::GaugeCalculations
 
   def field_values_at(time, opts={}, &block)
     opts[:max_fields] ||= 50
-    field_values = redis.zrevrange(
-      tick_key(time), 
-      0, opts[:max_fields]-1, 
+    field_values = sync_redis.zrevrange(
+      tick_key(time),
+      0, opts[:max_fields]-1,
       :withscores => true
     )
 
@@ -65,11 +80,18 @@ module FnordMetric::GaugeCalculations
   end
 
   def field_values_total(time)
-    (redis.get(tick_key(time, :count))||0).to_i
+    (sync_redis.get(tick_key(time, :count))||0).to_i
   end
 
-  def redis
-    @opts[:redis]
+  def fraction_values_in(range, _append=nil)
+    Hash.new{ |h,k| h[k] = [0,0] }.tap do |vals|
+      ticks_in(range, retention).each do |_tick|
+        sync_redis.hgetall(retention_key(_tick, _append)).each do |k, v|
+          kx = k.split("-")
+          vals[kx.first.to_i][kx.last == "denominator" ? 1 : 0] += v.to_f
+        end
+      end
+    end
   end
 
-end
+end

data/lib/fnordmetric/gauge_modifiers.rb

@@ -72,5 +72,72 @@ module FnordMetric::GaugeModifiers
     @redis.zadd(gauge.tick_key(time), value, field_name)
   end
 
+  def incr_numerator(ctx, series_name=:default, value=1, prog=false)
+    incr_fraction(ctx, series_name, :numerator, value, prog)
+  end
+
+  def incr_denominator(ctx, series_name=:default, value=1, prog=false)
+    incr_fraction(ctx, series_name, :denominator, value, prog)
+  end
+
+  def incr_fraction(ctx, series_name, part, value, prog)
+    return unless series_name = assure_series_exists!(series_name)
+    assure_has_series!
+
+    at = ctx.send(:time)
+    value = parse_numeric(value)
+
+    if prog
+      raise "FIXPAUL: not yet implemented: progressive fraction gauges"
+    end
+
+    ctx.redis_exec(:hincrby, retention_key(at, series_name), "#{tick_at(at)}-#{part}", value).callback do 
+      ctx.redis_exec :expire,  retention_key(at, series_name)
+    end
+  end
+  
+  def assure_has_series!
+    return true if has_series?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with series gauges" 
+  end
+
+  def assure_two_dimensional!(gauge)
+    return true if gauge.two_dimensional?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with 2-dimensional gauges" 
+  end
+
+  def assure_three_dimensional!(gauge)
+    return true unless gauge.two_dimensional?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with 3-dimensional gauges" 
+  end
+
+  def assure_non_progressive!(gauge)
+    return true unless gauge.progressive?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with non-progressive gauges" 
+  end
+
+  def assure_series_exists!(series_name)
+    if series_name == :default && @opts[:series].size > 1
+      error! "gauge '#{name}': don't know which series to increment"
+    elsif series_name == :default
+      return @opts[:series].first
+    elsif !series_name.respond_to?(:to_sym) || !@opts[:series].include?(series_name.to_sym)
+      error! "gauge '#{name}': unknown series: #{series_name}"
+    else 
+      return series_name
+    end
+  end
+
+  def parse_numeric(val)
+    if val.is_a?(Numeric)
+      return val
+    elsif val.is_a?(String) && val.match(/[0-9]+/)
+      val.to_i
+    elsif val.is_a?(String) && val.match(/[0-9]+(\.|,)[0-9]+/)
+      val.to_f
+    else
+      error! "gauge '#{name}': incr called with non-numerical value: #{val}"
+    end
+  end
 
 end

data/lib/fnordmetric/gauge_rendering.rb

@@ -0,0 +1,40 @@
+module FnordMetric::GaugeRendering
+
+  def renderable?
+    false
+  end
+
+  def render_to_event(*args)
+    { :title => name }.merge(render(*args))
+  end
+
+private
+
+  def render_page(in_file)
+    exec_js = []
+    content = render_haml(in_file)
+    content.scan(/<FNORDMETRIC-GAUGEJS>(.*)<\/FNORDMETRIC-GAUGEJS>/m){ |x| exec_js << x }
+    content.gsub!(/<FNORDMETRIC-GAUGEJS>(.*)<\/FNORDMETRIC-GAUGEJS>/m, "")
+    { :html => content, :exec => exec_js.flatten * "" }
+  end
+
+  def render_haml(in_file)
+    haml_engine = Haml::Engine.new(File.read(
+      File.expand_path("../../../web/haml/#{in_file}.haml", __FILE__)
+    )).render(binding) 
+  end
+
+  def parse_interval(interval_str)
+    match = interval_str.match(/([0-9]+)-([0-9]+)/)
+    raise "invalid interval: #{interval_str}" unless match
+    (match[1].to_i..match[2].to_i)
+  end
+
+  # FIXPAUL: move to apphelper or something
+  def fancy_timerange(range)
+    [range.first, range.last].map do |time|
+      Time.at(time).strftime("%d.%m.%y %H:%M")
+    end
+  end
+
+end

data/lib/fnordmetric/gauge_validations.rb

@@ -0,0 +1,15 @@
+module FnordMetric::GaugeValidations
+
+  def validate_series!
+    if !@opts[:series].is_a?(Array) || @opts[:series].size == 0
+      raise "#{@opts[:key]}: missing option series"
+    end
+
+    if @opts[:series].size != @opts[:series].uniq.size
+      raise "#{@opts[:key]}: series are not unique"
+    end
+
+    @opts[:series] = @opts[:series].map(&:to_sym)
+  end
+
+end

data/lib/fnordmetric/gauges/distribution_gauge.rb

@@ -0,0 +1,85 @@
+class FnordMetric::DistributionGauge < FnordMetric::Gauge
+
+  def render(namespace, event)
+    @interval = parse_interval(event["interval"])
+    colors = ["#2F635E", "#606B36", "#727070", "#936953", "#CD645A", "#FACE4F", "#42436B"]
+
+    @opts[:value_scale] ||= 1
+
+    #@num_min =
+    #@num_max =
+
+    @histogram = FnordMetric::Histogram.new
+    @values = []
+
+    @samples = 0
+
+    @mmm_timeseries = Hash.new do |h,k| 
+      h[k] = { :min => nil, :max => 0, :avg => [] }
+    end
+
+    ticks_in(@interval, tick, 1).each do |_tick|
+      tkey = tick_key(_tick, :histogram)
+
+      sync_redis.hgetall(tkey).each do |_val, _count|        
+        _count = _count.to_f
+        _val = _val.to_f * @opts[:value_scale]
+
+        @samples += _count
+
+        @histogram[_val] += _count
+        @values += [_val] * _count
+
+        if !@mmm_timeseries[_tick][:min] || (_val < @mmm_timeseries[_tick][:min])
+          @mmm_timeseries[_tick][:min] = _val
+        end
+
+        if _val > @mmm_timeseries[_tick][:max]
+          @mmm_timeseries[_tick][:max] = _val
+        end
+
+        @mmm_timeseries[_tick][:avg] += [_val] * _count
+      end
+    end
+
+    if @opts[:value_ranges]
+      @histogram_mode = @opts[:value_ranges]
+    else
+      @histogram_mode = [23, @histogram.max || 1].min
+    end
+
+    @mmm_timeseries_arr = @mmm_timeseries.to_a
+      .map{ |k,v| [k, Hash[v.map{ |vk, vv| [vk, (vv.is_a?(Numeric) || vv.is_a?(Array)) ? vv : 0 ] }]] }
+      .sort{ |a,b| a.first.to_i <=> b.first.to_i}
+
+    render_page(:distribution_gauge)
+  end
+
+  def execute(cmd, context, *args)
+    return observe(context, args.first) if cmd == :observe
+    FnordMetric.error("gauge '#{name}': unknown command: #{cmd}")
+  end
+
+  def renderable?
+    true
+  end
+
+private
+
+  def observe(ctx, value)
+    at = ctx.send(:time)
+
+    if value.is_a?(String) && value.match(/[0-9]+/)
+      value = value.to_i
+    elsif value.is_a?(String) && value.match(/[0-9]+(\.|,)[0-9]+/)
+      value = value.to_f
+    end
+
+    unless value.is_a?(Float) || value.is_a?(Fixnum)
+      return FnordMetric.error("gauge '#{name}': observe called with non-numerical value: #{value}")
+    end
+
+    ctx.redis_exec :hincrby, tick_key(at, :histogram), value.round(2), 1
+  end
+
+end