fnordmetric 0.7.5 → 0.9.7

data/lib/fnordmetric/gauges/timeseries_gauge.rb

@@ -0,0 +1,143 @@
+class FnordMetric::TimeseriesGauge < FnordMetric::Gauge
+
+  def initialize(opts)
+    super(opts)
+    
+    @opts[:series] = @opts[:series].map(&:to_sym)
+
+    if @opts[:calculate]
+      unless [:sum, :average, :progressive_sum].include?(@opts[:calculate].to_sym)
+        raise "unknown calculate option: #{@opts[:calculate]}"
+      end
+      @calculate = @opts[:calculate].to_sym
+    end 
+
+    @calculate ||= :sum
+
+    if @calculate == :average
+      @calculate_proc = lambda{ |c,d| d > 0 ? (c/d.to_f).round(2) : 0 }
+    else
+      @calculate_proc = lambda{ |c,d| c }
+    end
+  end
+
+  def render(namespace, event)
+    @interval = parse_interval(event["interval"])
+    colors = FnordMetric::COLORS.dup
+
+    @series = Hash.new
+    @zooms  = FnordMetric::TICKS[tick, @interval.size]
+
+    @total = 0
+
+    @opts[:series].each do |series|
+      ts = FnordMetric::Timeseries.new
+
+      fraction_values_in(@interval, series).each do |time, frac|
+        @total += frac.first # FIXPAUL
+        ts.incr_fraction(time, *frac)
+      end
+
+      @series[series] = { 
+        :color => colors.unshift(colors.pop).first,
+        :data => Hash[@zooms.map{ |int| [int, ts.timeseries(@interval, int) ] }],
+        :timeseries => ts
+      }
+    end
+
+    render_page(:timeseries_gauge)
+  end
+
+  def execute(cmd, context, *args)
+    return incr(context, *args) if cmd == :incr
+    return incr_numerator(context, *args) if cmd == :incr_numerator
+    return incr_denominator(context, *args) if cmd == :incr_denominator
+
+    FnordMetric.error("gauge '#{name}': unknown command: #{cmd}")
+  end
+
+  def renderable?
+    true
+  end
+
+  def has_series?
+    true
+  end
+
+private
+
+  def incr(ctx, series_name = :default, value = 1)
+    if @calculate == :average
+      incr_numerator(ctx, series_name, value)
+      incr_denominator(ctx, series_name, 1)
+    elsif @calculate == :sum
+      incr_numerator(ctx, series_name, value)
+    elsif @calculate == :progressive_sum
+      incr_numerator(ctx, series_name, value, true)
+    end
+  end
+
+end
+
+# class FnordMetric::NumericGauge < FnordMetric::MultiGauge
+
+#   def initialize(opts)   
+#     super(opts)
+
+#     validate_series!
+#     validate_ticks!
+
+#     timeline_widget(
+#       :tab => "Overview",
+#       :title => "Total #{key_nouns.last}",
+#       :ticks => @opts[:ticks],
+#       :series => @opts[:series],
+#       :series_titles => Hash[@opts[:series].map{|s| [s, s]}],
+#       :autoupdate => 10,
+#       :height => 350
+#     ).on(:values_at) do |_series, _ticks, _tick|
+#       series_count_metrics[_series][_tick].values_at(_ticks)
+#     end
+
+#     numbers_widget(
+#       :tab => "Overview",
+#       :title => "Total #{key_nouns.last}",
+#       :series => @opts[:series],
+#       :series_titles => Hash[@opts[:series].map{|s| [s, s]}],
+#       :autoupdate => 2
+#     ).on(:values_for) do |_series|
+#       render_series_numbers(_series.to_sym)
+#     end
+
+#   end
+
+#   def react(event)
+#     if event["_class"] == "incrby" || event["_class"] == "incr"
+#       series = event["series"]
+#       series ||= @opts[:series][0] if @opts[:series].size == 1
+#       incr_series(series.to_sym, event["_time"], event["value"])
+#     end
+#   end
+
+
+
+
+      
+#   def render_series_numbers(series)
+#     _t = Time.now.to_i
+
+#     {}.tap do |out|
+#       @opts[:ticks].each do |tick|
+#         out["#{tick}-now"]  = { 
+#           :value => series_count_metrics[series][tick].value_at(_t),
+#           :desc  => "$formatTimeRangePre(#{tick}, 0)"
+#         }
+#         out["#{tick}-last"] = { 
+#           :value => series_count_metrics[series][tick].value_at(_t-tick),
+#           :desc  => "$formatTimeRangePre(#{tick}, -1)"
+#         }
+#       end
+#     end
+#   end
+
+# end

data/lib/fnordmetric/gauges/toplist_gauge.rb

@@ -0,0 +1,44 @@
+class FnordMetric::ToplistGauge < FnordMetric::Gauge
+
+  def render(namespace, event)
+    @interval = parse_interval(event["interval"])
+
+    @toplist = FnordMetric::Toplist.new
+    @all_ticks = ticks_in(@interval, tick, 1)
+
+    @all_ticks.each do |_tick|
+      field_values_at(_tick, :limit => top_k, :append => :toplist).each do |*args|
+        item, count = args.flatten[0..1] # what the fnord... ~paul
+        @toplist.incr_item(_tick, item, count)
+      end
+    end
+
+    @toplist.total = @all_ticks.inject(0){ |s,t| s + sync_redis.get(tick_key(t, :total)).to_i }
+
+    render_page(:toplist_gauge)
+  end
+
+  def execute(cmd, context, *args)
+    return observe(context, args.first) if cmd == :observe
+    FnordMetric.error("gauge '#{name}': unknown command: #{cmd}")
+  end
+
+  def renderable?
+    true
+  end
+
+private
+
+  def observe(ctx, item)
+    at = ctx.send(:time)
+    ctx.redis_exec :zincrby, tick_key(at, :toplist), 1, item
+    ctx.redis_exec :incrby, tick_key(at, :total), 1
+    ctx.redis_exec :zremrangebyrank, tick_key(at, :toplist), 0, -top_k
+    ctx.redis_exec :expire, tick_key(at, :toplist), retention
+  end
+
+  def top_k
+    (@opts[:top_k] || 1000).to_i
+  end
+
+end

data/lib/fnordmetric/histogram.rb

@@ -0,0 +1,57 @@
+class FnordMetric::Histogram < Hash
+
+  def initialize
+    super{ |h,k| h[k]=0 }
+  end
+
+  def [](key)
+  	super(key.to_f)
+  end
+
+  def []=(key, val)
+  	super(key.to_f, val)
+  end
+
+  def min
+  	keys.sort.first.to_i
+  end
+
+  def max
+    keys.sort.last.to_i
+  end
+
+  def histogram(windows)
+    windows = histogram_windows(windows) unless windows.is_a?(Array)
+    Hash[windows.map{ |w| [w,0] }].tap do |histo|
+      self.each do |k,v|
+        histo.detect do |win, wval|          
+          histo[win] += v if win.include?(k)
+        end
+      end
+    end
+  end
+
+  def json_histogram(windows)
+    histogram(windows).to_a.sort do |a, b|
+      a[0].first <=> b[0].first
+    end.map do |r, v|
+      [r.size == 1.0 ? r.last.to_s :
+        "#{r.first.round(1).to_s}-#{r.last.round(1).to_s}", v.to_i]
+    end.to_json
+  end
+
+private
+
+  def histogram_windows(windows)
+    _min = min 
+    _max = max
+
+    return [(0..1)] if (_max-_min == 0)
+
+    windows.times
+      .inject((_min.._max)
+      .step(((_max-_min)/windows.to_f)).to_a << _max){ |a,n| 
+        a[n]=(a[n]..a[n+1]); a }.take(windows)
+  end
+
+end

data/lib/fnordmetric/logger.rb

@@ -1,57 +1,63 @@
 class FnordMetric::Logger
 
-  def self.start(logfile_path)    
-    require 'json'
-    event_ids = Queue.new
+  def self.import(logfile_path)
+    expire = FnordMetric.options[:event_queue_ttl]
+    redis  = Redis.new
+ 
+    @opts[:channels] ||= []
+    @opts[:channels] = @opts[:channels].map(&:to_s) 
+
+    dump_file = File.open(logfile_path, 'r')
+    num_lines = %x{wc -l #{logfile_path}}.to_i
+    puts "importing #{num_lines} events..."
+
+    dump_file.each_with_log(num_lines) do |line, ind|
+      (8**64).to_s(36).tap do |uuid|
+        redis.set    "fnordmetric-event-#{uuid}", line
+        redis.lpush  "fnordmetric-queue"        , uuid
+        redis.expire "fnordmetric-event-#{uuid}", expire
+      end
+    end
+  end
+
+  def initialize(opts)
+    @opts = opts
+    opts.fetch(:file)
+
+    FnordMetric.register(self)
+  end
+
+  def initialized 
+    logfile_path = @opts[:file]
+
+    events = Queue.new
     dump_file = File.open(logfile_path, 'a+')
 
     fetcher = Thread.new do
-      redis = Redis.new
       loop do
-        event_id = event_ids.pop
-        event_data = redis.get("fnordmetric-event-#{event_id}")        
-        event_hash = JSON.parse(event_data) rescue next
+        event = events.pop
 
-        event_hash.merge!(:_time => Time.now.to_i)
-
-        dump_file.write(event_hash.to_json+"\n")
+        dump_file.write(event.to_json+"\n")
         dump_file.flush
-
-        print "\033[1;34m" 
-        print event_hash.inspect
-        print "\033[0m\n"             
       end
     end
 
     listener = Thread.new do  
-      redis = Redis.new
-      redis.subscribe("fnordmetric-announce") do |on|
-        on.message do |channel, event_id|      
-          event_ids << event_id
-        end
+      backend = FnordMetric.backend
+      backend.subscribe do |event|
+        events << event if log_channel?(event["_channel"])
       end
     end
 
-    fetcher.join
+    FnordMetric.log "logging to #{logfile_path}"
   end
 
-  def self.import(logfile_path)
-    redis = Redis.new
-    dump_file = File.open(logfile_path, 'r')
 
-    puts "reading #{logfile_path}..."
-    dump_lines = dump_file.read.split("\n")
-
-    puts "importing #{dump_lines.length} events..."
-    pre_uuid = rand(999999999999999999999)
-    log_every = (dump_lines.length / 150)
-    dump_lines.each_with_index do |line,n|
-      puts "#{n}/#{dump_lines.length} (#{((n/dump_lines.length.to_f)*100).to_i}%)" if n%log_every==0
-      my_uuid = "#{pre_uuid}-#{n}"
-      redis.set("fnordmetric-event-#{my_uuid}", line)
-      redis.lpush("fnordmetric-queue", my_uuid) 
-      redis.expire("fnordmetric-event-#{my_uuid}", 3600*12)
-    end
+private
+
+  def log_channel?(channel)
+    return !!@opts[:channels] if !channel
+    @opts[:channels].include?(channel.to_s)
   end
 
 end

data/lib/fnordmetric/namespace.rb

@@ -1,38 +1,43 @@
 class FnordMetric::Namespace
   
-  attr_reader :handlers, :gauges, :opts, :key, :dashboards
+  attr_reader :handlers, :gauges, :opts, :key, :dashboards, :flags
 
-  @@opts = [:event, :gauge, :widget, :set_title, :active_users_available]
+  @@opts = [:event, :gauge, :widget, :set_title, :hide_active_users, :hide_overview]
+  @@multi_gauges = [:timeseries_gauge, :toplist_gauge, :distribution_gauge]
 
-  def initialize(key, opts)    
+  def initialize(key, opts)
     @gauges = Hash.new
     @dashboards = Hash.new
     @handlers = Hash.new
+    @flags = Hash.new
     @title = key
     @active_users_available = true
     @opts = opts
-    @key = key      
+    @key = key
   end
 
   def ready!(redis)
     @redis = redis
-    @gauges.map{ |k,g| g.add_redis(@redis) }
     self
   end
 
-  def announce(event)                  
+  def announce(event)
     announce_to_timeline(event)
     announce_to_typelist(event)
-    
+
     if event[:_session]
       event[:_session_key] = announce_to_session(event).session_key 
     end
 
-    [
+    res = [
       @handlers[event[:_type].to_s],
       @handlers["*"]
-    ].flatten.compact.each do |context| 
-      context.call(event, @redis) 
+    ].flatten.compact.each do |context|
+      context.call(event, @redis)
+    end.size
+
+    if res == 0
+      FnordMetric.error("no handler for event-type: #{event[:_type]}")
     end
 
     self
@@ -40,11 +45,11 @@ class FnordMetric::Namespace
 
   def announce_to_session(event)
     FnordMetric::Session.create(@opts.clone.merge(
-      :namespace_key => @key, 
+      :namespace_key => @key,
       :namespace_prefix => key_prefix,
       :redis => @redis,
       :event => event
-    )) 
+    ))
   end
 
   def announce_to_timeline(event)
@@ -65,15 +70,15 @@ class FnordMetric::Namespace
   def token
     @key
   end
-  
+
   def title
     @title
   end
-  
+
   def active_users_available
     @active_users_available
   end
-  
+
   def dashboards(name=nil)
     return @dashboards unless name
     dash = FnordMetric::Dashboard.new(:title => name)
@@ -91,29 +96,41 @@ class FnordMetric::Namespace
   end
 
   def method_missing(m, *args, &block)
+    return send(:opt_multigauge, *args.unshift(m), &block) if @@multi_gauges.include?(m)
     raise "unknown option: #{m}" unless @@opts.include?(m)
     send(:"opt_#{m}", *args, &block)
   end
 
-  def hide_active_users
-    @active_users_available = false
+  def opt_hide_active_users
+    @flags[:hide_active_users] = true
   end
-  
-  def set_title(key)
-    @title = key
+
+  def opt_hide_overview
+    @flags[:hide_overview] = true
   end
-  
+
+  def opt_set_title(title)
+    @title = title
+  end
+
   def opt_event(event_type, opts={}, &block)    
     opts.merge!(:redis => @redis, :gauges => @gauges)   
     FnordMetric::Context.new(opts, block).tap do |context|
       @handlers[event_type.to_s] ||= []
       @handlers[event_type.to_s] << context
-    end      
+    end
   end
 
   def opt_gauge(gauge_key, opts={})
     opts.merge!(:key => gauge_key, :key_prefix => key_prefix)
-    @gauges[gauge_key] ||= FnordMetric::Gauge.new(opts)   
+    klass = "FnordMetric::#{(opts[:type] || "").to_s.camelize}Gauge".constantize
+    @gauges[gauge_key] ||= klass.new(opts)
+  end
+
+  def opt_multigauge(gauge_type, gauge_key, opts={})
+    opts.merge!(:key => gauge_key, :key_prefix => key_prefix)
+    klass = "FnordMetric::#{gauge_type.to_s.camelize}"
+    @gauges[gauge_key] ||= klass.constantize.new(opts)   
   end
 
   def opt_widget(dashboard, widget)
@@ -135,4 +152,11 @@ class FnordMetric::Namespace
     )
   end
 
+  def to_json
+    flags.merge(
+      :token => token,
+      :title => title
+    ).to_json
+  end
+
 end

data/lib/fnordmetric/session.rb

@@ -1,21 +1,21 @@
 class FnordMetric::Session
- 
+
   attr_accessor :updated_at, :name, :picture
 
   @@meta_attributes = %w(name picture)
 
-  def self.create(opts)        
+  def self.create(opts)
     redis = opts.fetch(:redis)
-    event = opts[:event]   
+    event = opts[:event]
 
     hash = Digest::MD5.hexdigest(event[:_session])
     set_key = "#{opts[:namespace_prefix]}-session"
 
     self.new(hash).tap do |session|
-      session.add_redis(redis, set_key)      
-      session.add_event(event)    
+      session.add_redis(redis, set_key)
+      session.add_event(event)
       session.expire(opts[:session_data_ttl])
-    end    
+    end
   end
 
   def self.find(session_key, opts)

data/lib/fnordmetric/standalone.rb

@@ -1,40 +1,20 @@
-require 'rake'
-require 'redis'
+_opts = FnordMetric.options
 
-task :run do  
-  FnordMetric.run
+if _opts[:web_interface]
+  FnordMetric::Web.new(
+    :host => _opts[:web_interface][0],
+    :port => _opts[:web_interface][1]
+  )
 end
 
-task :worker do
-  FnordMetric.server_configuration = {
-    :web_interface  => false,
-    :inbound_stream => false,
-    :start_worker   => true
-  }
-  FnordMetric.run
+if _opts[:inbound_stream]
+  FnordMetric::Acceptor.new(
+    :protocol => _opts[:inbound_protocol],
+    :host => _opts[:inbound_stream][0],
+    :port => _opts[:inbound_stream][1]
+  )
 end
 
-task :log do
-  FnordMetric::Logger.start(dump_file_path)
-end
-
-task :import do
-  FnordMetric::Logger.import(dump_file_path)
-end
-
-task :help do
-  puts "usage: #{$0} {run|worker|log|import} [DUMP_FILE=fm_dump.json]"
-end
-
-task :default => :help
-
-def dump_file_path
-  if ENV["DUMP_FILE"].blank?
-    Rake::Task[:help].execute; exit!
-  else
-    ::File.expand_path(ENV["DUMP_FILE"], ::File.dirname($0))
-  end
-end
-
-Rake.application.init('fnordmetric')
-Rake.application.top_level
+if _opts[:start_worker]
+  FnordMetric::Worker.new()
+end

data/lib/fnordmetric/timeseries.rb

@@ -0,0 +1,79 @@
+class FnordMetric::Timeseries
+
+  def initialize(timeline = {})
+    @timeline = Hash.new{ |h,k| h[k] = [0,nil] }
+    @timeline.merge!(timeline)
+  end
+
+  def incr_fraction(time, numerator, denominator)
+  	incr_numerator(time, numerator) if numerator
+  	incr_denominator(time, denominator) if denominator
+  end
+
+  def incr_numerator(time, value)
+  	@timeline[time.to_i][0] += value
+  end
+
+  def incr_denominator(time, value)
+    @timeline[time.to_i][-1] ||= 0
+    @timeline[time.to_i][-1] += value
+  end
+
+  def timeseries(range, window, &block)
+    res = Hash.new{ |h,k| h[k] = [0,0] }
+
+    (((range.size)/window.to_f).ceil+1).times.map do |n| 
+      res[((range.first+window*(n-1))/window.to_f).floor*window] = [0,0]
+    end
+
+    @timeline.each do |time, vals|
+      next unless range.include?(time)
+      wtime = (time/window.to_f).floor * window
+      if block
+        res[wtime] = block.call(*vals)
+      else
+        res[wtime][0] += vals[0]
+        res[wtime][1] += vals[1]
+      end
+    end
+    
+    FnordMetric::Timeseries.new(res)
+  end
+
+  def sum(range = (ticks.first..ticks.last))
+    @timeline
+      .inject(0){ |s,(t,v)| s + (range.include?(t) ? value_at(t) : 0) }
+  end
+
+  def trend(range = (ticks.first..ticks.last))
+    range ||= (ticks.first..ticks.last)    
+
+    rvals = @timeline.to_a
+      .select{ |t,v| range.include?(t) }
+      .sort{ |a,b| a.first <=> b.first }
+      .map{ |t,v| value_at(t) }
+
+    return 0 if rvals.size == 0
+    (rvals.last - rvals.first).to_f / rvals.first
+  end
+
+  def ticks
+    @timeline.keys.sort
+  end
+
+  def value_at(time)
+    if @timeline[time][1].to_i > 0
+      @timeline[time][0] / @timeline[time][1].to_f
+    else
+      @timeline[time][0]
+    end    
+  end
+
+  def to_json(&block)
+    @timeline.to_a
+      .sort{ |a,b| a[0] <=> b[0] }
+      .map { |t,v| { :x => t, :y => block.call(*v), :v0 => v[0], :v1 => v[1] } }
+      .to_json
+  end
+
+end