johnf-fnordmetric 1.2.7

data/lib/fnordmetric/gauge.rb

@@ -0,0 +1,98 @@
+class FnordMetric::Gauge
+  
+  include FnordMetric::GaugeCalculations
+  include FnordMetric::GaugeModifiers
+  include FnordMetric::GaugeValidations
+  include FnordMetric::GaugeRendering
+
+  def initialize(opts)
+    opts.fetch(:key) && opts.fetch(:key_prefix)
+    @opts = opts
+  end
+
+  def tick
+    (@opts[:tick] || @opts[:resolution] || @opts[:flush_interval] ||
+      FnordMetric.options[:default_flush_interval]).to_i
+  end
+
+  def retention
+    tick * 10 # FIXPAUL!
+  end
+
+  def tick_at(time, _tick=tick)    
+    (time/_tick.to_f).floor*_tick
+  end
+
+  def name
+    @opts[:key]
+  end
+
+  def title
+    @opts[:title] || name
+  end
+
+  def group
+    @opts[:group] || "Gauges"
+  end
+
+  def unit
+    @opts[:unit]
+  end
+
+  def key_nouns
+    @opts[:key_nouns] || ["Key", "Keys"]
+  end
+
+  def key(_append=nil)
+    [@opts[:key_prefix], "gauge", name, tick, _append].flatten.compact.join("-")
+  end
+
+  def tick_key(_time, _append=nil)
+    key([(progressive? ? :progressive : tick_at(_time).to_s), _append])
+  end
+
+  def tick_keys(_range, _append=nil)
+    ticks_in(_range).map{ |_t| tick_key(_t, _append) }
+  end
+
+  def retention_key(_time, _append=nil)
+    key([tick_at(_time, retention).to_s, _append])
+  end
+
+  def two_dimensional?
+    !@opts[:three_dimensional]
+  end
+
+  def three_dimensional?
+    !!@opts[:three_dimensional]
+  end
+
+  def progressive?
+    !!@opts[:progressive]
+  end
+
+  def unique?
+    !!@opts[:unique]
+  end
+
+  def average?
+    !!@opts[:average]
+  end
+
+  def has_series?
+    false
+  end
+
+  def redis
+    @redis ||= EM::Hiredis.connect(FnordMetric.options[:redis_url]) # FIXPAUL
+  end
+
+  def sync_redis
+    @sync_redis ||= FnordMetric.mk_redis # FIXPAUL
+  end
+
+  def error!(msg)
+    FnordMetric.error(msg)
+  end
+
+end

data/lib/fnordmetric/gauge_calculations.rb

@@ -0,0 +1,106 @@
+module FnordMetric::GaugeCalculations
+
+  @@avg_per_session_proc = proc{ |_v, _t|
+    (_v.to_f / (sync_redis.get(tick_key(_t, :"sessions-count"))||1).to_i)
+  }
+
+  @@count_per_session_proc = proc{ |_v, _t|
+    (sync_redis.get(tick_key(_t, :"sessions-count"))||0).to_i
+  }
+
+  @@avg_per_count_proc = proc{ |_v, _t|
+    (_v.to_f / (sync_redis.get(tick_key(_t, :"value-count"))||1).to_i)
+  }
+
+  def ticks_in(r, _tick=tick, overflow=0)
+    (((r.last-r.first)/_tick.to_f).ceil+1+overflow).times.map{ |n| tick_at(r.first + _tick*(n-1), _tick) }
+  end
+
+  def values_in(range)
+    ticks = ticks_in(range)
+    ticks << tick_at(range.last) if ticks.size == 0
+    values_at(ticks)
+  end
+
+  def value_at(time, opts={}, &block)
+    _t = tick_at(time)
+
+    _v = if respond_to?(:_value_at)
+      _value_at(key, _t)
+    else
+      _c = sync_redis.hget(key(:"mean-counts"), _t)
+      sync_redis.hget(key, _t)
+    end
+
+    calculate_value(_v, _t, opts, block, _c)
+  end
+
+  def values_at(times, opts={}, &block)
+    times = times.map{ |_t| tick_at(_t) }
+    Hash.new.tap do |ret|
+      if respond_to?(:_values_at)
+        _values_at(times, opts={}, &block)
+      else
+        ret_counts = sync_redis.hmget(key(:"mean-counts"), *times)
+        sync_redis.hmget(key, *times)
+      end.each_with_index do |_v, _n|
+        _t = times[_n]
+        _c = ret_counts ? ret_counts[_n] : nil
+        ret[_t] = calculate_value(_v, _t, opts, block, _c)
+      end
+    end
+  end
+
+  def calculate_value(_v, _t, opts, block, _c = nil)
+    block = @@avg_per_session_proc if unique? && average?
+
+    calc = if average? && _c
+      (_v.to_f / (_c||1).to_i)
+    elsif block
+      instance_exec(_v, _t, &block)
+    else
+      _v
+    end
+
+    if calc && @opts[:scale_by]
+      calc = calc.to_f * @opts[:scale_by].to_f
+    end
+
+    calc
+  end
+
+  def field_values_at(time, opts={}, &block)
+    opts[:max_fields] ||= 50
+    field_values = sync_redis.zrevrange(
+      tick_key(time, opts[:append]),
+      0, opts[:max_fields]-1,
+      :withscores => true
+    )
+
+    unless field_values.first.is_a?(Array)
+      field_values = field_values.in_groups_of(2).map do |key, val|
+        [key, Float(val)]
+      end
+    end
+
+    field_values.map do |key, val|
+      [key, calculate_value("%.f" % val, time, opts, block)]
+    end
+  end
+
+  def field_values_total(time)
+    (sync_redis.get(tick_key(time, :count))||0).to_i
+  end
+
+  def fraction_values_in(range, _append=nil)
+    Hash.new{ |h,k| h[k] = [0,0] }.tap do |vals|
+      ticks_in(range, retention).each do |_tick|
+        sync_redis.hgetall(retention_key(_tick, _append)).each do |k, v|
+          kx = k.split("-")
+          vals[kx.first.to_i][kx.last == "denominator" ? 1 : 0] += v.to_f
+        end
+      end
+    end
+  end
+
+end

data/lib/fnordmetric/gauge_modifiers.rb

@@ -0,0 +1,144 @@
+module FnordMetric::GaugeModifiers
+
+  def incr(gauge_name, value=1)
+    value = value.to_i
+    gauge = fetch_gauge(gauge_name)
+    assure_two_dimensional!(gauge)
+    if gauge.unique? 
+      incr_uniq(gauge, value)
+    elsif gauge.average? 
+      incr_avg(gauge, value)
+    else
+      incr_tick(gauge, value)
+    end
+  end
+
+  def incr_tick(gauge, value)
+    if gauge.progressive?
+      @redis.incrby(gauge.key(:head), value).callback do |head|
+        @redis.hsetnx(gauge.key, gauge.tick_at(time), head).callback do |_new|
+          @redis.hincrby(gauge.key, gauge.tick_at(time), value) unless _new
+        end
+      end
+    else
+      @redis.hsetnx(gauge.key, gauge.tick_at(time), 0).callback do
+        @redis.hincrby(gauge.key, gauge.tick_at(time), value)
+      end
+    end
+  end
+
+  def incr_uniq(gauge, value, field_name=nil)
+    return false if session_key.blank?
+    @redis.sadd(gauge.tick_key(time, :sessions), session_key).callback do |_new|
+      @redis.expire(gauge.tick_key(time, :sessions), gauge.tick)
+      if (_new == 1) || (_new == true) #redis vs. em-redis
+        @redis.incr(gauge.tick_key(time, :"sessions-count")).callback do |sc|
+          field_name ? incr_field_by(gauge, field_name, value) : incr_tick(gauge, value)
+        end
+      end
+    end
+  end
+
+  def incr_avg(gauge, value)
+    @redis.hincrby(gauge.key(:"mean-counts"), gauge.tick_at(time), 1).callback do
+      incr_tick(gauge, value)
+    end
+  end
+
+  def incr_field(gauge_name, field_name, value=1)
+    gauge = fetch_gauge(gauge_name)
+    assure_three_dimensional!(gauge)
+    if gauge.unique? 
+      incr_uniq(gauge, value, field_name)
+    else
+      incr_field_by(gauge, field_name, value)
+    end
+  end
+
+  def incr_field_by(gauge, field_name, value)
+    @redis.zincrby(gauge.tick_key(time), value, field_name).callback do
+      @redis.incrby(gauge.tick_key(time, :count), 1)
+    end
+  end
+
+  def set_value(gauge_name, value)
+    gauge = fetch_gauge(gauge_name)
+    assure_two_dimensional!(gauge)
+    @redis.hset(gauge.key, gauge.tick_at(time), value)
+  end
+
+  def set_field(gauge_name, field_name, value)
+    gauge = fetch_gauge(gauge_name)
+    assure_three_dimensional!(gauge)
+    @redis.zadd(gauge.tick_key(time), value, field_name)
+  end
+
+  def incr_numerator(ctx, series_name=:default, value=1, prog=false)
+    incr_fraction(ctx, series_name, :numerator, value, prog)
+  end
+
+  def incr_denominator(ctx, series_name=:default, value=1, prog=false)
+    incr_fraction(ctx, series_name, :denominator, value, prog)
+  end
+
+  def incr_fraction(ctx, series_name, part, value, prog)
+    return unless series_name = assure_series_exists!(series_name)
+    assure_has_series!
+
+    at = ctx.send(:time)
+    value = parse_numeric(value)
+
+    if prog
+      raise "FIXPAUL: not yet implemented: progressive fraction gauges"
+    end
+
+    ctx.redis_exec(:hincrby, retention_key(at, series_name), "#{tick_at(at)}-#{part}", value).callback do 
+      ctx.redis_exec :expire,  retention_key(at, series_name), retention
+    end
+  end
+  
+  def assure_has_series!
+    return true if has_series?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with series gauges" 
+  end
+
+  def assure_two_dimensional!(gauge)
+    return true if gauge.two_dimensional?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with 2-dimensional gauges" 
+  end
+
+  def assure_three_dimensional!(gauge)
+    return true unless gauge.two_dimensional?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with 3-dimensional gauges" 
+  end
+
+  def assure_non_progressive!(gauge)
+    return true unless gauge.progressive?
+    error! "error: #{caller[0].split(" ")[-1]} can only be used with non-progressive gauges" 
+  end
+
+  def assure_series_exists!(series_name)
+    if series_name == :default && @opts[:series].size > 1
+      error! "gauge '#{name}': don't know which series to increment"
+    elsif series_name == :default
+      return @opts[:series].first
+    elsif !series_name.respond_to?(:to_sym) || !@opts[:series].include?(series_name.to_sym)
+      error! "gauge '#{name}': unknown series: #{series_name}"
+    else 
+      return series_name
+    end
+  end
+
+  def parse_numeric(val)
+    if val.is_a?(Numeric)
+      return val
+    elsif val.is_a?(String) && val.match(/[0-9]+/)
+      val.to_i
+    elsif val.is_a?(String) && val.match(/[0-9]+(\.|,)[0-9]+/)
+      val.to_f
+    else
+      error! "gauge '#{name}': incr called with non-numerical value: #{val}"
+    end
+  end
+
+end

data/lib/fnordmetric/gauge_rendering.rb

@@ -0,0 +1,40 @@
+module FnordMetric::GaugeRendering
+
+  def renderable?
+    false
+  end
+
+  def render_to_event(*args)
+    { :title => name }.merge(render(*args))
+  end
+
+private
+
+  def render_page(in_file)
+    exec_js = []
+    content = render_haml(in_file)
+    content.scan(/<FNORDMETRIC-GAUGEJS>(.*)<\/FNORDMETRIC-GAUGEJS>/m){ |x| exec_js << x }
+    content.gsub!(/<FNORDMETRIC-GAUGEJS>(.*)<\/FNORDMETRIC-GAUGEJS>/m, "")
+    { :html => content, :exec => exec_js.flatten * "" }
+  end
+
+  def render_haml(in_file)
+    haml_engine = Haml::Engine.new(File.read(
+      File.expand_path("../../../web/haml/#{in_file}.haml", __FILE__)
+    )).render(binding) 
+  end
+
+  def parse_interval(interval_str)
+    match = interval_str.match(/([0-9]+)-([0-9]+)/)
+    raise "invalid interval: #{interval_str}" unless match
+    (match[1].to_i..match[2].to_i)
+  end
+
+  # FIXPAUL: move to apphelper or something
+  def fancy_timerange(range)
+    [range.first, range.last].map do |time|
+      Time.at(time).strftime("%d.%m.%y %H:%M")
+    end
+  end
+
+end

data/lib/fnordmetric/gauge_validations.rb

@@ -0,0 +1,15 @@
+module FnordMetric::GaugeValidations
+
+  def validate_series!
+    if !@opts[:series].is_a?(Array) || @opts[:series].size == 0
+      raise "#{@opts[:key]}: missing option series"
+    end
+
+    if @opts[:series].size != @opts[:series].uniq.size
+      raise "#{@opts[:key]}: series are not unique"
+    end
+
+    @opts[:series] = @opts[:series].map(&:to_sym)
+  end
+
+end

data/lib/fnordmetric/gauges/distribution_gauge.rb

@@ -0,0 +1,87 @@
+class FnordMetric::DistributionGauge < FnordMetric::Gauge
+
+  def render(namespace, event)
+    @interval = parse_interval(event["interval"])
+    colors = ["#2F635E", "#606B36", "#727070", "#936953", "#CD645A", "#FACE4F", "#42436B"]
+
+    @opts[:value_scale] ||= 1
+    @opts[:precision]   ||= 1
+
+    #@num_min =
+    #@num_max =
+
+    @histogram = FnordMetric::Histogram.new
+    @values = []
+    @histogram.set_opts(@opts)
+
+    @samples = 0
+
+    @mmm_timeseries = Hash.new do |h,k|
+      h[k] = { :min => nil, :max => 0, :avg => [] }
+    end
+
+    ticks_in(@interval, tick, 1).each do |_tick|
+      tkey = tick_key(_tick, :histogram)
+
+      sync_redis.hgetall(tkey).each do |_val, _count|
+        _count = _count.to_f
+        _val = _val.to_f * @opts[:value_scale]
+
+        @samples += _count
+
+        @histogram[_val] += _count
+        @values += [_val] * _count
+
+        if !@mmm_timeseries[_tick][:min] || (_val < @mmm_timeseries[_tick][:min])
+          @mmm_timeseries[_tick][:min] = _val
+        end
+
+        if _val > @mmm_timeseries[_tick][:max]
+          @mmm_timeseries[_tick][:max] = _val
+        end
+
+        @mmm_timeseries[_tick][:avg] += [_val] * _count
+      end
+    end
+
+    if @opts[:value_ranges]
+      @histogram_mode = @opts[:value_ranges]
+    else
+      @histogram_mode = [23, @histogram.max || 1].min
+    end
+
+    @mmm_timeseries_arr = @mmm_timeseries.to_a
+      .map{ |k,v| [k, Hash[v.map{ |vk, vv| [vk, (vv.is_a?(Numeric) || vv.is_a?(Array)) ? vv : 0 ] }]] }
+      .sort{ |a,b| a.first.to_i <=> b.first.to_i}
+
+    render_page(:distribution_gauge)
+  end
+
+  def execute(cmd, context, *args)
+    return observe(context, args.first) if cmd == :observe
+    FnordMetric.error("gauge '#{name}': unknown command: #{cmd}")
+  end
+
+  def renderable?
+    true
+  end
+
+private
+
+  def observe(ctx, value)
+    at = ctx.send(:time)
+
+    if value.is_a?(String) && value.match(/[0-9]+/)
+      value = value.to_i
+    elsif value.is_a?(String) && value.match(/[0-9]+(\.|,)[0-9]+/)
+      value = value.to_f
+    end
+
+    unless value.is_a?(Float) || value.is_a?(Fixnum)
+      return FnordMetric.error("gauge '#{name}': observe called with non-numerical value: #{value}")
+    end
+
+    ctx.redis_exec :hincrby, tick_key(at, :histogram), value.round(2), 1
+  end
+
+end

data/lib/fnordmetric/gauges/server_health_gauge.rb

@@ -0,0 +1,13 @@
+# ServerHealthGauge
+# here be dragons....
+#
+# This gauge listens for JSON events that look like this:
+#
+# {
+#   "hostname": "my-hostname",
+#   "uptime": 1231523,
+#   "load_avg": [0.32, 0.16, 0.13],
+#   "mem": { "total": 4069, "free": 1096 },
+#   "swap": { "total": 0, "free": 0 },
+#   "disk_usage": "43%"
+# }

data/lib/fnordmetric/gauges/timeseries_gauge.rb

@@ -0,0 +1,138 @@
+class FnordMetric::TimeseriesGauge < FnordMetric::Gauge
+
+  def initialize(opts)
+    super(opts)
+    
+    @opts[:series] = @opts[:series].map(&:to_sym)
+
+    if @opts[:calculate]
+      unless [:sum, :average, :progressive_sum].include?(@opts[:calculate].to_sym)
+        raise "unknown calculate option: #{@opts[:calculate]}"
+      end
+      @calculate = @opts[:calculate].to_sym
+    end 
+
+    @calculate ||= :sum
+    @calculate_proc = lambda{ |c,d| d > 0 ? (c/d.to_f).round(2) : c }
+  end
+
+  def render(namespace, event)
+    @interval = parse_interval(event["interval"])
+    colors = FnordMetric::COLORS.dup
+
+    @series = Hash.new
+    @zooms  = FnordMetric::TICKS[tick, @interval.size]
+
+    @total = 0
+
+    @opts[:series].each do |series|
+      ts = FnordMetric::Timeseries.new
+
+      fraction_values_in(@interval, series).each do |time, frac|
+        @total += frac.first # FIXPAUL
+        ts.incr_fraction(time, *frac)
+      end
+
+      @series[series] = { 
+        :color => colors.unshift(colors.pop).first,
+        :data => Hash[@zooms.map{ |int| [int, ts.timeseries(@interval, int) ] }],
+        :timeseries => ts
+      }
+    end
+
+    render_page(:timeseries_gauge)
+  end
+
+  def execute(cmd, context, *args)
+    return incr(context, *args) if cmd == :incr
+    return incr_numerator(context, *args) if cmd == :incr_numerator
+    return incr_denominator(context, *args) if cmd == :incr_denominator
+
+    FnordMetric.error("gauge '#{name}': unknown command: #{cmd}")
+  end
+
+  def renderable?
+    true
+  end
+
+  def has_series?
+    true
+  end
+
+private
+
+  def incr(ctx, series_name = :default, value = 1)
+    if @calculate == :average
+      incr_numerator(ctx, series_name, value)
+      incr_denominator(ctx, series_name, 1)
+    elsif @calculate == :sum
+      incr_numerator(ctx, series_name, value)
+    elsif @calculate == :progressive_sum
+      incr_numerator(ctx, series_name, value, true)
+    end
+  end
+
+end
+
+# class FnordMetric::NumericGauge < FnordMetric::MultiGauge
+
+#   def initialize(opts)   
+#     super(opts)
+
+#     validate_series!
+#     validate_ticks!
+
+#     timeline_widget(
+#       :tab => "Overview",
+#       :title => "Total #{key_nouns.last}",
+#       :ticks => @opts[:ticks],
+#       :series => @opts[:series],
+#       :series_titles => Hash[@opts[:series].map{|s| [s, s]}],
+#       :autoupdate => 10,
+#       :height => 350
+#     ).on(:values_at) do |_series, _ticks, _tick|
+#       series_count_metrics[_series][_tick].values_at(_ticks)
+#     end
+
+#     numbers_widget(
+#       :tab => "Overview",
+#       :title => "Total #{key_nouns.last}",
+#       :series => @opts[:series],
+#       :series_titles => Hash[@opts[:series].map{|s| [s, s]}],
+#       :autoupdate => 2
+#     ).on(:values_for) do |_series|
+#       render_series_numbers(_series.to_sym)
+#     end
+
+#   end
+
+#   def react(event)
+#     if event["_class"] == "incrby" || event["_class"] == "incr"
+#       series = event["series"]
+#       series ||= @opts[:series][0] if @opts[:series].size == 1
+#       incr_series(series.to_sym, event["_time"], event["value"])
+#     end
+#   end
+
+
+
+
+      
+#   def render_series_numbers(series)
+#     _t = Time.now.to_i
+
+#     {}.tap do |out|
+#       @opts[:ticks].each do |tick|
+#         out["#{tick}-now"]  = { 
+#           :value => series_count_metrics[series][tick].value_at(_t),
+#           :desc  => "$formatTimeRangePre(#{tick}, 0)"
+#         }
+#         out["#{tick}-last"] = { 
+#           :value => series_count_metrics[series][tick].value_at(_t-tick),
+#           :desc  => "$formatTimeRangePre(#{tick}, -1)"
+#         }
+#       end
+#     end
+#   end
+
+# end

data/lib/fnordmetric/gauges/toplist_gauge.rb

@@ -0,0 +1,44 @@
+class FnordMetric::ToplistGauge < FnordMetric::Gauge
+
+  def render(namespace, event)
+    @interval = parse_interval(event["interval"])
+
+    @toplist = FnordMetric::Toplist.new
+    @all_ticks = ticks_in(@interval, tick, 1)
+
+    @all_ticks.each do |_tick|
+      field_values_at(_tick, :limit => top_k, :append => :toplist).each do |*args|
+        item, count = args.flatten[0..1] # what the fnord... ~paul
+        @toplist.incr_item(_tick, item, count)
+      end
+    end
+
+    @toplist.total = @all_ticks.inject(0){ |s,t| s + sync_redis.get(tick_key(t, :total)).to_i }
+
+    render_page(:toplist_gauge)
+  end
+
+  def execute(cmd, context, *args)
+    return observe(context, args.first) if cmd == :observe
+    FnordMetric.error("gauge '#{name}': unknown command: #{cmd}")
+  end
+
+  def renderable?
+    true
+  end
+
+private
+
+  def observe(ctx, item)
+    at = ctx.send(:time)
+    ctx.redis_exec :zincrby, tick_key(at, :toplist), 1, item
+    ctx.redis_exec :incrby, tick_key(at, :total), 1
+    ctx.redis_exec :zremrangebyrank, tick_key(at, :toplist), 0, -top_k
+    ctx.redis_exec :expire, tick_key(at, :toplist), retention
+  end
+
+  def top_k
+    (@opts[:top_k] || 1000).to_i
+  end
+
+end