johnf-fnordmetric 1.2.7

data/lib/fnordmetric/histogram.rb

@@ -0,0 +1,64 @@
+class FnordMetric::Histogram < Hash
+
+  def initialize
+    super{ |h,k| h[k]=0 }
+  end
+
+  def set_opts(opts = {})
+    @opts = opts
+  end
+
+  def [](key)
+  	super(key.to_f)
+  end
+
+  def []=(key, val)
+  	super(key.to_f, val)
+  end
+
+  def min
+  	keys.sort.first.to_i
+  end
+
+  def max
+    keys.sort.last.to_i
+  end
+
+  def histogram(windows)
+    windows = histogram_windows(windows) unless windows.is_a?(Array)
+    Hash[windows.map{ |w| [w,0] }].tap do |histo|
+      self.each do |k,v|
+        histo.detect do |win, wval|
+          histo[win] += v if win.include?(k)
+        end
+      end
+    end
+  end
+
+  def json_histogram(windows)
+    histogram(windows).to_a.sort do |a, b|
+      a[0].first <=> b[0].first
+    end.map do |r, v|
+      [r.size == 1.0 ? r.last.to_s : json_value(r), v.to_i]
+    end.to_json
+  end
+
+  def json_value(r)
+    "#{r.first.round(@opts[:precision]).to_s}-#{r.last.round(@opts[:precision]).to_s}"
+  end
+
+private
+
+  def histogram_windows(windows)
+    _min = min
+    _max = max
+
+    return [(0..1)] if (_max-_min == 0)
+
+    windows.times
+      .inject((_min.._max)
+      .step(((_max-_min)/windows.to_f)).to_a << _max){ |a,n|
+        a[n]=(a[n]..a[n+1]); a }.take(windows)
+  end
+
+end

data/lib/fnordmetric/logger.rb

@@ -0,0 +1,63 @@
+class FnordMetric::Logger
+
+  def self.import(logfile_path)
+    expire = FnordMetric.options[:event_queue_ttl]
+    redis  = Redis.new
+ 
+    @opts[:channels] ||= []
+    @opts[:channels] = @opts[:channels].map(&:to_s) 
+
+    dump_file = File.open(logfile_path, 'r')
+    num_lines = %x{wc -l #{logfile_path}}.to_i
+    puts "importing #{num_lines} events..."
+
+    dump_file.each_with_log(num_lines) do |line, ind|
+      (8**64).to_s(36).tap do |uuid|
+        redis.set    "fnordmetric-event-#{uuid}", line
+        redis.lpush  "fnordmetric-queue"        , uuid
+        redis.expire "fnordmetric-event-#{uuid}", expire
+      end
+    end
+  end
+
+  def initialize(opts)
+    @opts = opts
+    opts.fetch(:file)
+
+    FnordMetric.register(self)
+  end
+
+  def initialized 
+    logfile_path = @opts[:file]
+
+    events = Queue.new
+    dump_file = File.open(logfile_path, 'a+')
+
+    fetcher = Thread.new do
+      loop do
+        event = events.pop
+
+        dump_file.write(event.to_json+"\n")
+        dump_file.flush
+      end
+    end
+
+    listener = Thread.new do  
+      backend = FnordMetric.backend
+      backend.subscribe do |event|
+        events << event if log_channel?(event["_channel"])
+      end
+    end
+
+    FnordMetric.log "logging to #{logfile_path}"
+  end
+
+
+private
+
+  def log_channel?(channel)
+    return !!@opts[:channels] if !channel
+    @opts[:channels].include?(channel.to_s)
+  end
+
+end

data/lib/fnordmetric/namespace.rb

@@ -0,0 +1,208 @@
+class FnordMetric::Namespace
+
+  attr_reader :handlers, :gauges, :opts, :key, :dashboards, :flags
+
+  @@opts = [:event, :gauge, :widget, :set_title, :hide_active_users, :hide_overview,
+            :hide_gauge_explorer, :dashboard]
+
+  @@multi_gauges = [:timeseries_gauge, :toplist_gauge, :distribution_gauge]
+
+  def initialize(key, opts)
+    @gauges = Hash.new
+    @dashboards = Hash.new
+    @handlers = Hash.new.with_indifferent_access
+    @title = key
+    @opts = opts
+    @key = key
+
+    @flags = {
+      :hide_active_users => (FnordMetric.options[:enable_active_users] == false),
+      :hide_gauge_explorer => (FnordMetric.options[:enable_gauge_explorer] == false)
+    }
+  end
+
+  def ready!(redis, sync_redis = nil)
+    @redis = redis
+    @sync_redis = sync_redis
+    load_gauges
+    self
+  end
+
+  def announce(event)
+    if !@flags[:hide_active_users]
+      announce_to_timeline(event)
+      announce_to_typelist(event)
+    end
+
+    if event[:_session]
+      event[:_session_key] = announce_to_session(event).session_key 
+    end
+
+    if event[:_type].to_sym == :_enterprise
+      ctx = FnordMetric::Context.new(opts, FnordMetric::Enterprise::CompatibilityHandler)
+      ctx.call(event, @redis, self)
+      return self
+    end
+
+    if FnordMetric::ZeroConfigGauge::TYPES.include?(event[:_type].to_sym)
+      ctx = FnordMetric::Context.new(opts, FnordMetric::ZeroConfigGauge::Handler)
+      ctx.call(event, @redis, self)
+      return self
+    end
+
+    res = [
+      @handlers[event[:_type].to_s],
+      @handlers["*"]
+    ].flatten.compact.each do |context|
+      context.call(event, @redis, self)
+    end.size
+
+    if res == 0
+      FnordMetric.error("no handler for event-type: #{event[:_type]}")
+    end
+
+    self
+  end
+
+  def announce_to_session(event)
+    FnordMetric::Session.create(@opts.clone.merge(
+      :namespace_key => @key,
+      :namespace_prefix => key_prefix,
+      :redis => @redis,
+      :event => event
+    ))
+  end
+
+  def announce_to_timeline(event)
+    timeline_key = key_prefix(:timeline)
+    @redis.zadd(timeline_key, event[:_time], event[:_eid])
+  end
+
+  def announce_to_typelist(event)
+    typelist_key = key_prefix("type-#{event[:_type]}")
+    @redis.lpush(typelist_key, event[:_eid])
+  end
+
+
+  def key_prefix(append=nil)
+    [@opts[:redis_prefix], @key, append].compact.join("-")
+  end
+
+  def token
+    @key
+  end
+
+  def title
+    @title
+  end
+
+  def dashboards(name=nil, opts = {})
+    return @dashboards unless name
+    dash = FnordMetric::Dashboard.new(opts.merge(:title => name))
+    @dashboards[dash.token.to_s] ||= dash
+  end
+
+  def sessions(_ids, opts={})
+    return FnordMetric::Session.all(extend_opts(opts)) if _ids == :all
+  end
+
+  def events(_ids, opts={})
+    return FnordMetric::Event.all(extend_opts(opts)) if _ids == :all
+    return FnordMetric::Event.by_type(opts.delete(:type), extend_opts(opts)) if _ids == :by_type
+    return FnordMetric::Event.by_session_key(opts.delete(:session_key), extend_opts(opts)) if _ids == :by_session_key
+  end
+
+  def method_missing(m, *args, &block)
+    return send(:opt_multigauge, *args.unshift(m), &block) if @@multi_gauges.include?(m)
+    raise "unknown option: #{m}" unless @@opts.include?(m)
+    send(:"opt_#{m}", *args, &block)
+  end
+
+  def opt_hide_active_users
+    @flags[:hide_active_users] = true
+  end
+
+  def opt_hide_gauge_explorer
+    @flags[:hide_gauge_explorer] = true
+  end
+
+  def opt_hide_overview
+    @flags[:hide_overview] = true
+  end
+
+  def opt_set_title(title)
+    @title = title
+  end
+
+  def opt_event(event_type, opts={}, &block)
+    FnordMetric::Context.new(opts, block).tap do |context|
+      @handlers[event_type.to_s] ||= []
+      @handlers[event_type.to_s] << context
+    end
+  end
+
+  def opt_gauge(gauge_key, opts={})
+    opts.merge!(:key => gauge_key)
+    store_gauge(gauge_key, opts) if opts[:zero_config]
+    opts.merge!(:key_prefix => key_prefix)
+    klass = "FnordMetric::#{(opts[:type] || "").to_s.camelize}Gauge".constantize
+    @gauges[gauge_key] ||= klass.new(opts)
+  end
+
+  def opt_multigauge(gauge_type, gauge_key, opts={})
+    opts.merge!(:key => gauge_key, :key_prefix => key_prefix)
+    klass = "FnordMetric::#{gauge_type.to_s.camelize}"
+    @gauges[gauge_key] ||= klass.constantize.new(opts)   
+  end
+
+  def opt_widget(dashboard, widget)
+    widget = build_widget(widget) if widget.is_a?(Hash)
+    dashboards(dashboard).add_widget(widget)
+  end
+
+  def opt_dashboard(dashboard, opts)
+    dashboards(dashboard, opts)
+  end
+
+  def build_widget(opts)
+    _gauges = [opts[:gauges]].flatten.map do |g|
+      @gauges[g] || FnordMetric::ZeroConfigGauge.new(g, self)
+    end
+    widget_klass = "FnordMetric::#{opts.fetch(:type).to_s.capitalize}Widget"
+    widget_klass.constantize.new(opts.merge(:gauges => _gauges))
+  end
+
+  def store_gauge(gauge_key, opts)
+    gaugelist_key = key_prefix("zero-config-gauges")
+    sync_redis.hset(gaugelist_key, gauge_key, opts.to_json)
+  end
+
+  def load_gauges
+    gaugelist_key = key_prefix("zero-config-gauges")
+    sync_redis.hgetall(gaugelist_key).each do |gauge_key, gauge_opts|
+      gopts = JSON.parse(gauge_opts).symbolize_keys
+      gopts.delete(:zero_config)
+      opt_gauge(gauge_key.to_sym, gopts)
+    end
+  end
+
+  def sync_redis
+    @sync_redis || @redis
+  end
+
+  def extend_opts(opts)
+    opts.merge(
+      :namespace_prefix => key_prefix,
+      :redis_prefix => @opts[:redis_prefix],
+      :redis => @redis
+    )
+  end
+
+  def to_json
+    flags.merge(
+      :token => token,
+      :title => title
+    ).to_json
+  end
+
+end

data/lib/fnordmetric/session.rb

@@ -0,0 +1,139 @@
+class FnordMetric::Session
+
+  attr_accessor :updated_at, :name, :picture
+
+  @@meta_attributes = %w(name picture)
+
+  def self.create(opts)
+    redis = opts.fetch(:redis)
+    event = opts[:event]
+
+    hash = Digest::MD5.hexdigest(event[:_session])
+    set_key = "#{opts[:namespace_prefix]}-session"
+
+    self.new(hash).tap do |session|
+      session.add_redis(redis, set_key)
+      session.add_event(event)
+      session.expire(opts[:session_data_ttl])
+    end
+  end
+
+  def self.find(session_key, opts)
+    set_key = "#{opts[:namespace_prefix]}-session"
+    self.new(session_key, [opts[:redis], set_key])
+  end
+
+  def self.all(opts)    
+    set_key = "#{opts[:namespace_prefix]}-session"
+    limit = (opts[:limit].try(:to_i)||0)-1
+    session_ids = opts[:redis].zrevrange(set_key, 0, limit, :withscores => true)
+
+    unless session_ids.first.is_a?(Array)
+      session_ids = session_ids.in_groups_of(2).map do |session_id, ts|
+        [session_id, Float(ts)]
+      end
+    end
+
+    session_ids.map do |session_key, ts|
+      find(session_key, opts).tap{ |s| s.updated_at = "%.f" % ts }
+    end
+  end
+
+  def initialize(session_key, redis_opts=nil)
+    @session_key = session_key
+    add_redis(*redis_opts) if redis_opts
+  end
+
+  def session_key
+    @session_key
+  end
+
+  def picture
+    @picture
+  end
+
+  def name
+    @name
+  end
+
+  def data(key=nil)
+    key ? @data[key] : @data 
+  end
+
+  def event_ids
+    @event_ids || []
+  end
+
+  def events
+    [] 
+  end
+
+  def to_json
+    { :session_key => session_key }.tap do |hash| 
+      hash.merge!(:_picture => @picture) if @picture
+      hash.merge!(:_name => @name) if @name
+      hash.merge!(:_updated_at => @updated_at) if @updated_at
+    end
+  end
+
+  def redis_key(append=nil)
+    [@redis_prefix, @session_key, append].compact.join("-")
+  end
+
+  def add_redis(redis, prefix)
+    @redis_prefix = prefix
+    @redis = redis
+  end
+
+  def touch(time=Time.now.to_i)      
+    @redis.zadd(@redis_prefix, time, @session_key)
+  end
+
+  def expire(time)
+    @redis.expire(redis_key(:events), time)
+    @redis.expire(redis_key(:data), time)
+  end
+
+  def add_event(event)    
+    @redis.zadd(redis_key(:events), event[:_time], event[:_eid])    
+
+    add_data(:_picture, event[:url]) if event[:_type] == "_set_picture"    
+    add_data(:_name, event[:name]) if event[:_type] == "_set_name"    
+    add_event_data(event) if event[:_type] == "_set_data" 
+    touch(event[:_time])
+  end
+
+  def add_event_data(event)
+    event.each do |key, value|
+      add_data(key, value) unless key.to_s.starts_with?("_")
+    end
+  end
+
+  def add_data(key, value)    
+    @redis.hset(redis_key(:data), key, value)
+  end
+  
+  def fetch_data!
+    @data = Hash.new
+    @redis.hgetall(redis_key(:data)).each do |key, value|    
+      if key.to_s.starts_with?("_")
+        fetch_meta_key(key, value)
+      else      
+        @data[key.intern] = value 
+      end
+    end
+  end
+
+  def fetch_meta_key(key, value)
+    meta_key = key[1..-1]
+    if @@meta_attributes.include?(meta_key)
+      instance_variable_set(:"@#{meta_key}", value)
+    end
+  end
+
+  def fetch_event_ids!(since=-1)
+    # FIXME: use WITHSCORE to get the timestamps and return event objects
+    @event_ids = @redis.zrevrange(redis_key(:events), 0, since)
+  end
+
+end

data/lib/fnordmetric/standalone.rb

@@ -0,0 +1,20 @@
+_opts = FnordMetric.options
+
+if _opts[:web_interface]
+  FnordMetric::Web.new(
+    :host => _opts[:web_interface][0],
+    :port => _opts[:web_interface][1]
+  )
+end
+
+if _opts[:inbound_stream]
+  FnordMetric::Acceptor.new(
+    :protocol => _opts[:inbound_protocol],
+    :host => _opts[:inbound_stream][0],
+    :port => _opts[:inbound_stream][1]
+  )
+end
+
+if _opts[:start_worker]
+  FnordMetric::Worker.new()
+end

data/lib/fnordmetric/timeseries.rb

@@ -0,0 +1,79 @@
+class FnordMetric::Timeseries
+
+  def initialize(timeline = {})
+    @timeline = Hash.new{ |h,k| h[k] = [0,nil] }
+    @timeline.merge!(timeline)
+  end
+
+  def incr_fraction(time, numerator, denominator)
+  	incr_numerator(time, numerator) if numerator
+  	incr_denominator(time, denominator) if denominator
+  end
+
+  def incr_numerator(time, value)
+  	@timeline[time.to_i][0] += value
+  end
+
+  def incr_denominator(time, value)
+    @timeline[time.to_i][-1] ||= 0
+    @timeline[time.to_i][-1] += value
+  end
+
+  def timeseries(range, window, &block)
+    res = Hash.new{ |h,k| h[k] = [0,0] }
+
+    (((range.size)/window.to_f).ceil+1).times.map do |n| 
+      res[((range.first+window*(n-1))/window.to_f).floor*window] = [0,0]
+    end
+
+    @timeline.each do |time, vals|
+      next unless range.include?(time)
+      wtime = (time/window.to_f).floor * window
+      if block
+        res[wtime] = block.call(*vals)
+      else
+        res[wtime][0] += vals[0]
+        res[wtime][1] += vals[1]
+      end
+    end
+    
+    FnordMetric::Timeseries.new(res)
+  end
+
+  def sum(range = (ticks.first..ticks.last))
+    @timeline
+      .inject(0){ |s,(t,v)| s + (range.include?(t) ? value_at(t) : 0) }
+  end
+
+  def trend(range = (ticks.first..ticks.last))
+    range ||= (ticks.first..ticks.last)    
+
+    rvals = @timeline.to_a
+      .select{ |t,v| range.include?(t) }
+      .sort{ |a,b| a.first <=> b.first }
+      .map{ |t,v| value_at(t) }
+
+    return 0 if rvals.size == 0
+    (rvals.last - rvals.first).to_f / rvals.first
+  end
+
+  def ticks
+    @timeline.keys.sort
+  end
+
+  def value_at(time)
+    if @timeline[time][1].to_i > 0
+      @timeline[time][0] / @timeline[time][1].to_f
+    else
+      @timeline[time][0]
+    end    
+  end
+
+  def to_json(&block)
+    @timeline.to_a
+      .sort{ |a,b| a[0] <=> b[0] }
+      .map { |t,v| { :x => t, :y => block.call(*v), :v0 => v[0], :v1 => v[1] } }
+      .to_json
+  end
+
+end

data/lib/fnordmetric/toplist.rb

@@ -0,0 +1,61 @@
+  class FnordMetric::Toplist
+
+  attr_accessor :timelines, :total
+
+  def initialize(timeline = {})
+    @total   = 0
+    @toplist = Hash.new{ |h,k| h[k] = 0 }
+
+    @timelines = Hash.new do |h,k|
+      h[k] = Hash.new{ |h,k| h[k] = 0 }
+    end
+  end
+
+  def incr_item(time, item, value)
+    @toplist[item] += value.to_f
+    @timelines[item][time] += value.to_f
+    @total += value.to_f
+  end
+
+  def prepare!
+    @toplist_arr = @toplist.to_a.sort do |a,b|
+      b.last <=> a.last
+    end
+  end
+
+  def toplist(take = 100)
+    prepare! unless @toplist_arr
+    @toplist_arr[0..(take-1)]
+  end
+
+  def percentage(item)
+    (@toplist[item].to_f / total.to_f) * 100.0
+  end
+
+  def value(item)
+    @toplist[item].to_f
+  end
+
+  def trend(item)
+    times = @timelines[item].keys.sort
+
+    (@timelines[item][times.last] - 
+    @timelines[item][times.first]) / 
+    @timelines[item][times.first]
+  end
+
+  def rank(item)
+    prepare! unless @toplist_arr
+
+    @toplist_arr.index([item, value(item)]) + 1
+  end
+
+  def trending(take = 100)
+    @toplist.to_a.map{ |k,v| 
+      [k, trend(k)]
+    }.sort{ |a,b|
+      b.last <=> a.last
+    }[0..(take-1)]
+  end
+
+end

data/lib/fnordmetric/udp_client.rb

@@ -0,0 +1,22 @@
+class FnordMetric::UDPClient
+
+  def initialize(host, port)
+    @sock = UDPSocket.new
+    @sock.connect(host, port)
+  end
+
+  def event(event_data)
+    begin
+      if event_data.is_a?(Hash)
+        event_data = event_data.to_json
+      else
+        JSON.parse(event_data) # void ;)
+      end
+    rescue JSON::ParserError
+      FnordMetric.log("event_lost: can't parse json")
+    else
+      @sock.send(event_data, 0)
+    end
+  end
+
+end

data/lib/fnordmetric/util.rb

@@ -0,0 +1,25 @@
+class FnordMetric::Util
+
+  def self.parse_time(str)
+    str = str.downcase
+
+    if (str == "now")
+      Time.now.to_i
+    elsif str =~ /^([0-9]+(?:\.[0-9]+)?)$/
+      $1.to_i
+    elsif str =~ /^-([0-9]+(?:\.[0-9]+)?)$/
+      Time.now.to_i - $1.to_i
+    elsif str =~ /^-([0-9]+(?:\.[0-9]+)?)s(ec(ond)?(s?))?$/
+      Time.now.to_i - $1.to_f
+    elsif str =~ /^-([0-9]+(?:\.[0-9]+)?)m(in(ute)?(s?))?$/
+      Time.now.to_i - ($1.to_f * 60)
+    elsif str =~ /^-([0-9]+(?:\.[0-9]+)?)h(our(s?))?$/
+      Time.now.to_i - ($1.to_f * 3600)
+    elsif str =~ /^-([0-9]+(?:\.[0-9]+)?)d(ay(s?))?$/
+      Time.now.to_i - ($1.to_i * 86400)
+    else 
+      raise "invalid time specifiation: #{str}"
+    end
+  end
+
+end

data/lib/fnordmetric/version.rb

@@ -0,0 +1,3 @@
+module FnordMetric
+  VERSION = "1.2.7"
+end