fnordmetric 0.7.5 → 0.9.7

data/lib/fnordmetric/toplist.rb

@@ -0,0 +1,61 @@
+  class FnordMetric::Toplist
+
+  attr_accessor :timelines, :total
+
+  def initialize(timeline = {})
+    @total   = 0
+    @toplist = Hash.new{ |h,k| h[k] = 0 }
+
+    @timelines = Hash.new do |h,k|
+      h[k] = Hash.new{ |h,k| h[k] = 0 }
+    end    
+  end
+
+  def incr_item(time, item, value)
+    @toplist[item] += value.to_f
+    @timelines[item][time] += value.to_f
+    @total += value.to_f
+  end
+
+  def prepare!
+    @toplist_arr = @toplist.to_a.sort do |a,b|
+      b.last <=> a.last
+    end
+  end
+
+  def toplist(take = 100)
+    prepare! unless @toplist_arr
+    @toplist_arr[0..(take-1)]
+  end
+
+  def percentage(item)
+    (@toplist[item].to_f / total.to_f) * 100.0
+  end
+
+  def value(item)
+    @toplist[item].to_f
+  end
+
+  def trend(item)
+    times = @timelines[item].keys.sort
+
+    (@timelines[item][times.last] - 
+    @timelines[item][times.first]) / 
+    @timelines[item][times.first]
+  end
+
+  def rank(item)
+    prepare! unless @toplist_arr
+
+    @toplist_arr.index([item, value(item)]) + 1
+  end
+
+  def trending(take = 100)
+    @toplist.to_a.map{ |k,v| 
+      [k, trend(k)]
+    }.sort{ |a,b|
+      b.last <=> a.last
+    }[0..(take-1)]
+  end
+
+end

data/lib/fnordmetric/version.rb

@@ -1,3 +1,3 @@
 module FnordMetric
-  VERSION = "0.7.5"
+  VERSION = "0.9.7"
 end

data/lib/fnordmetric/web/app.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+
+class FnordMetric::App < Sinatra::Base
+
+  include FnordMetric::AppHelpers
+
+  if RUBY_VERSION =~ /1.9.\d/
+    Encoding.default_external = Encoding::UTF_8
+  end
+
+  if ENV['RACK_ENV'] == "test"
+    set :raise_errors, true
+  end
+
+  enable :session
+  set :haml, :format => :html5
+  set :views, ::File.expand_path('../../../../web/haml', __FILE__)
+  set :public_folder, ::File.expand_path('../../../../web', __FILE__)
+
+  helpers do
+    include Rack::Utils    
+    include FnordMetric::AppHelpers
+  end
+
+  def initialize(opts)
+    @opts = opts
+
+    @namespaces = FnordMetric.namespaces
+    @redis = Redis.connect(:url => opts[:redis_url])
+
+    super(nil)
+  end
+
+  get '/' do
+    redirect "#{path_prefix}/#{@namespaces.keys.first}"
+  end
+
+  get '/:namespace' do
+    pass unless current_namespace
+    haml :app
+  end
+
+  post '/events' do
+    halt 400, 'please specify the event_type (_type)' unless params["_type"]
+    track_event((8**32).to_s(36), parse_params(params))
+  end
+
+  # FIXPAUL move to websockets
+  get '/:namespace/dashboard/:dashboard' do
+    dashboard = current_namespace.dashboards.fetch(params[:dashboard])
+
+    dashboard.to_json
+  end
+
+
+  # get '/favicon.ico' do
+  #   ""
+  # end
+
+  # get '/:namespace/gauge/:name' do
+  #   gauge = if params[:name].include?("++")
+  #     parts = params[:name].split("++")
+  #     current_namespace.gauges[parts.first.to_sym].fetch_gauge(parts.last, params[:tick].to_i)
+  #   else
+  #     current_namespace.gauges[params[:name].to_sym]
+  #   end
+
+  #   if !gauge
+  #     status 404
+  #     return ""
+  #   end
+
+  #   data = if gauge.three_dimensional?
+  #     _t = (params[:at] || Time.now).to_i
+  #     { :count => gauge.field_values_total(_t), :values => gauge.field_values_at(_t) }
+  #   elsif params[:at] && params[:at] =~ /^[0-9]+$/
+  #     { (_t = gauge.tick_at(params[:at].to_i)) => gauge.value_at(_t) }
+  #   elsif params[:at] && params[:at] =~ /^([0-9]+)-([0-9]+)$/
+  #     _range = params[:at].split("-").map(&:to_i)
+  #     _values = gauge.values_in(_range.first.._range.last)
+  #     params[:sum] ? { :sum => _values.values.compact.map(&:to_i).sum } : _values
+  #   else
+  #     { (_t = gauge.tick_at(Time.now.to_i-gauge.tick)) => gauge.value_at(_t) }
+  #   end
+
+  #   data.to_json
+  # end
+
+  # get '/:namespace/sessions' do
+  #   sessions = current_namespace.sessions(:all, :limit => 100).map do |session|
+  #     session.fetch_data!
+  #     session.to_json
+  #   end
+
+  #   { :sessions => sessions }.to_json
+  # end
+
+  # get '/:namespace/events' do
+  #   events = if params[:type]
+  #     current_namespace.events(:by_type, :type => params[:type])
+  #   elsif params[:session_key]
+  #     current_namespace.events(:by_session_key, :session_key => params[:session_key])
+  #   else 
+  #     find_opts = { :limit => 100 }
+  #     find_opts.merge!(:since => params[:since].to_i+1) if params[:since]
+  #     current_namespace.events(:all, find_opts)
+  #   end
+
+  #   { :events => events.map(&:to_json) }.to_json
+  # end
+
+  # get '/:namespace/event_types' do
+  #   types_key = current_namespace.key_prefix("type-")
+  #   keys = @redis.keys("#{types_key}*").map{ |k| k.gsub(types_key,'') }
+
+  #   { :types => keys }.to_json
+  # end
+
+
+
+end
+

data/lib/fnordmetric/web/app_helpers.rb

@@ -0,0 +1,42 @@
+module FnordMetric::AppHelpers
+
+  def h(*args)
+    escape_html(*args)
+  end
+  
+  def path_prefix
+    request.env["SCRIPT_NAME"]
+  end
+
+  def namespaces
+    @namespaces
+  end
+
+  def current_namespace
+    @namespaces[@namespaces.keys.detect{ |k|
+      k.to_s == params[:namespace]
+    }.try(:intern)]
+  end
+
+  def parse_params(hash)
+    hash.tap do |h|
+      h.keys.each{ |k| h[k] = parse_param(h[k]) }
+    end
+  end
+
+  def parse_param(object)
+    return object unless object.is_a?(String)
+    return object.to_f if object.match(/^[0-9]+[,\.][0-9]+$/)
+    return object.to_i if object.match(/^[0-9]+$/)
+    object
+  end
+
+  def track_event(event_id, event_data)
+    # FIXPAUL: use api
+    @redis.hincrby "#{@opts[:redis_prefix]}-stats",             "events_received", 1
+    @redis.set     "#{@opts[:redis_prefix]}-event-#{event_id}", event_data.to_json
+    @redis.lpush   "#{@opts[:redis_prefix]}-queue",             event_id
+    @redis.expire  "#{@opts[:redis_prefix]}-event-#{event_id}", @opts[:event_queue_ttl]
+  end
+
+end

data/lib/fnordmetric/{dashboard.rb → web/dashboard.rb}

@@ -16,6 +16,10 @@ class FnordMetric::Dashboard
     @options[:title]
   end
 
+  def group
+    @options[:group] || "Dashboards"
+  end
+
   def token
     token = title.to_s.gsub(/[\W]/, '')
     token = Digest::SHA1.hexdigest(title.to_s) if token.empty?

data/lib/fnordmetric/{event.rb → web/event.rb}

@@ -6,15 +6,20 @@ module FnordMetric
     #def self.track!(event_type, event_data)
     #end
 
-    def self.all(opts)    
+    def self.all(opts)
+      opts[:limit] ||= 100
+
       range_opts = { :withscores => true }
       range_opts.merge!(:limit => [0,opts[:limit]]) if opts[:limit]
+
       events = opts[:redis].zrevrangebyscore(
         "#{opts[:namespace_prefix]}-timeline", 
         '+inf', opts[:since]||'0',
         range_opts
       )
 
+      events = events[0..opts[:limit] - 1]
+
       unless events.first.is_a?(Array)
         events = events.in_groups_of(2).map do |event_id, ts|
           [event_id, Float(ts)]
@@ -75,7 +80,7 @@ module FnordMetric
     def id
       @event_id
     end
-    
+
     def data(key=nil)
       key ? @data[key.to_s] : @data
     end

data/lib/fnordmetric/web/reactor.rb

@@ -0,0 +1,87 @@
+class FnordMetric::Reactor
+
+  def initialize
+    @redis = FnordMetric.mk_redis
+    @namespaces = FnordMetric.namespaces
+  end
+
+  def execute(*args)
+    execute_unsafe(*args)
+  rescue Exception => e
+    FnordMetric.error("reactor crashed: " + e.to_s) 
+    puts (e.backtrace * "\n") if ENV["FNORDMETRIC_ENV"] == "dev"
+    []
+  end
+
+private
+
+  def execute_unsafe(socket, event, messages = [])
+    return false unless event["namespace"]
+    return false unless ns = @namespaces[event["namespace"].to_sym]
+    messages << discover(ns) if event["type"] == "discover_request"
+    messages << widget(ns, event) if event["type"] == "widget_request"
+    messages << gauge(ns, event) if event["type"] == "render_request"
+    messages << active_users(ns, event) if event["type"] == "active_users_request"
+    messages.flatten.compact
+  end
+
+  def widget(namespace, event)
+    "FnordMetric::#{event["klass"]}".constantize.execute(namespace, event) # FIXPAUL
+  end
+
+  def gauge(namespace, event)
+    return false unless gauge = namespace.gauges[event["gauge"].to_sym]
+
+    {
+      :type    => "render_response",
+      :gauge   => gauge.name,
+      :payload => gauge.render_to_event(namespace, event)
+    }
+  end
+
+  def discover(namespace)
+    [namespace.dashboards.map do |dash_key, dash|
+      { "type" => "discover_response", "gauge_key" => dash_key, "view" => "dashboard",
+        "group" => dash.group }
+    end,
+    namespace.gauges.map do |gauge_key, gauge|
+      next unless gauge.renderable?
+      { "type" => "discover_response", "gauge_key" => gauge_key, "view" => "gauge",
+        "title" => gauge.title, "group" => gauge.group, "tick" => gauge.tick }
+    end.compact]
+  end
+
+  def active_users(namespace, event)
+    namespace.ready!(@redis)
+
+    events = if event["filter_by_type"]
+      namespace.events(:by_type, :type => event["type"])
+    elsif event["filter_by_session_key"]
+      namespace.events(:by_session_key, :session_key => params["session_key"])
+    else
+      find_opts = { :limit => 100 }
+      find_opts.merge!(:since => event["since"].to_i+1) if event["since"]
+      namespace.events(:all, find_opts)
+    end
+
+    sessions = namespace.sessions(:all, :limit => 100).map do |session|
+      session.fetch_data!
+      session.to_json
+    end
+
+    types_key = namespace.key_prefix("type-")
+    types = if event["first_poll"]
+      @redis.keys("#{types_key}*").map{ |k| k.gsub(types_key,'') }
+    else
+      []
+    end
+
+    {
+      :type => "active_users_response",
+      :sessions => sessions,
+      :events => events.map(&:to_json),
+      :types => types
+    }
+  end
+
+end

data/lib/fnordmetric/web/web.rb

@@ -0,0 +1,53 @@
+class FnordMetric::Web
+
+  def initialize(opts)
+    @opts = opts
+
+    @opts[:server] ||= "thin"
+    @opts[:host]   ||= "0.0.0.0"
+    @opts[:port]   ||= "4242"
+
+    FnordMetric.register(self)
+  end
+
+  def initialized
+    server = @opts[:server].downcase
+
+    middleware_stack = @opts[:use] || []
+
+    websocket = FnordMetric::WebSocket.new
+    webapp    = FnordMetric::App.new(@opts)
+
+    dispatch  = Rack::Builder.app do
+      use Rack::CommonLogger
+      use Rack::ShowExceptions
+
+      map "/stream" do
+        run websocket
+      end
+
+      map "/" do
+        middleware_stack.each do |middleware| 
+          use(*middleware[0..1], &middleware[2])
+        end
+
+        run webapp
+      end
+    end
+
+    unless ["thin", "hatetepe"].include? server
+      raise "Need an EventMachine webserver, but #{server} isn't"
+    end
+
+    host = @opts[:host] 
+    port = @opts[:port]
+
+    Rack::Server.start(
+      :app => dispatch,
+      :server => server,
+      :Host => host, 
+      :Port => port
+    ) && FnordMetric.log("listening on http://#{host}:#{port}")
+  end
+
+end

data/lib/fnordmetric/web/websocket.rb

@@ -0,0 +1,38 @@
+require "eventmachine"
+require 'rack/websocket'
+require "em-hiredis"
+require "json"
+
+class FnordMetric::WebSocket < Rack::WebSocket::Application
+
+  def initialize
+    super
+
+    @reactor = FnordMetric::Reactor.new
+    @uuid = "websocket-#{get_uuid}"
+  end
+
+  def on_open(env)
+    # socket openened :)
+  end
+
+  def on_message(env, message)
+    begin
+      message = JSON.parse(message)
+    rescue
+      puts "websocket: invalid json"
+    else
+      message["_eid"] ||= get_uuid
+      message["_sender"] = @uuid
+
+      @reactor.execute(self, message).each do |m|
+        send_data m.to_json
+      end
+    end
+  end
+
+  def get_uuid
+    rand(8**64).to_s(36)
+  end
+
+end

data/lib/fnordmetric/widgets/bars_widget.rb

@@ -0,0 +1,44 @@
+class FnordMetric::BarsWidget  < FnordMetric::Widget
+
+  def self.execute(namespace, event)
+    resp = if event["cmd"] == "values_for"
+      {
+        :cmd => :values_for,
+        :values => execute_values_for(namespace.gauges[event["gauge"].to_sym], event["until"])
+      }
+    end
+
+    return false unless resp
+
+    resp.merge(
+      :class => "widget_response",
+      :widget_key => event["widget_key"]
+    )
+  end
+
+  def self.execute_values_for(gauge, time)
+    gauge.field_values_at(time).sort do |a,b|
+      a.first.to_i <=> b.first.to_i
+    end.map do |a|
+      [a.first, a.second.to_i]
+    end
+  end
+
+  def data
+    super.merge(
+      :gauge => gauges.first.name,
+      :title => gauges.first.title,
+      :autoupdate => (@opts[:autoupdate] || 60),
+      :order_by => (@opts[:order_by] || 'value'),
+      :plot_style => (@opts[:plot_style] || 'vertical'),
+      :async_chart => true,
+      :color => FnordMetric::COLORS.last,
+      :tick => tick
+    )
+  end
+
+  def has_tick?
+    false
+  end
+
+end

data/lib/fnordmetric/widgets/numbers_widget.rb

@@ -0,0 +1,56 @@
+class FnordMetric::NumbersWidget < FnordMetric::Widget
+
+  def self.execute(namespace, event)
+    resp = if event["cmd"] == "values_for"
+      execute_values_for(namespace.gauges[event["gauge"].to_sym], event) 
+    end
+
+    return false unless resp
+
+    resp.merge(
+      :class => "widget_response",
+      :widget_key => event["widget_key"]
+    )
+  end
+
+  def self.execute_values_for(gauge, event)
+    _t = Time.now.to_i
+
+    values = {}.tap do |out|
+      event["offsets"].each do |off|
+        if off.to_s.starts_with?("s")
+          offset = 0
+          span = (gauge.tick * off.to_s[1..-1].to_i)
+          values = gauge.values_in((_t-span).._t+gauge.tick)
+          value = values.values.compact.map(&:to_i).sum
+        else
+          offset = off.to_i * gauge.tick
+          span = gauge.tick
+          value = gauge.value_at(_t-offset)
+        end
+
+        out["#{gauge.name}-#{offset}-#{span}"]  = { 
+          :value => value,
+          :desc  => "$formatOffset(#{offset}, #{span})"
+        }
+      end
+    end
+
+    { "cmd" => "values_for", 
+      "series" => gauge.name, 
+      "values" => values }
+  end
+
+  def data
+    super.merge(
+      :series => gauges.map(&:name),
+      :offsets => (@opts[:offsets] || [0, 1, "s30"]),
+      :autoupdate => (@opts[:autoupdate] || 60)
+    )
+  end
+
+  def has_tick?
+    false
+  end
+  
+end

data/lib/fnordmetric/widgets/timeseries_widget.rb

@@ -0,0 +1,55 @@
+class FnordMetric::TimeseriesWidget < FnordMetric::Widget
+
+  def self.execute(namespace, event)
+    resp = if event["cmd"] == "values_at"
+      {
+        :cmd => :values_at,
+        :gauges => event["gauges"].map{ |gkey|
+          vals = namespace.gauges[gkey.to_sym].values_in(event["since"]..event["until"])
+          { :key => gkey, :vals => vals }
+        }
+      }
+    end
+
+    return false unless resp
+
+    resp.merge(
+      :class => "widget_response",
+      :widget_key => event["widget_key"]
+    )
+  end
+
+  def data    
+    super.merge(
+      :series => series,
+      :gauges => gauges.map(&:name),
+      :start_timestamp => ticks.first,
+      :end_timestamp => ticks.last,
+      :autoupdate => (@opts[:autoupdate] || 60),
+      :include_current => !!@opts[:include_current],
+      :default_style => (@opts[:plot_style] || 'line'),
+      :async_chart => true,
+      :tick => tick
+    )
+  end
+
+  def series
+    colors = FnordMetric::COLORS.dup
+    
+    gauges.map do |gauge|
+      { 
+        :name => gauge.name, 
+        :data => [{:x => ticks.first, :y => 0}], 
+        :color => colors.unshift(colors.pop).first 
+      }
+    end
+  end
+
+  def has_tick?
+    true
+  end
+
+end
+
+class FnordMetric::TimelineWidget < FnordMetric::TimeseriesWidget
+end

data/lib/fnordmetric/widgets/toplist_widget.rb

@@ -0,0 +1,64 @@
+class FnordMetric::ToplistWidget < FnordMetric::Widget
+
+  def self.execute(namespace, event)
+    t = Time.now.to_i
+
+    return false unless event["gauge"]
+
+    resp = if event["cmd"] == "values_for"
+      {
+        :cmd => :values_for,
+        :gauge => event["gauge"],
+        :values => execute_values_for(namespace.gauges[event["gauge"].to_sym], t),
+        :count  => namespace.gauges[event["gauge"].to_sym].field_values_total(t)
+      }
+    end
+
+    return false unless resp
+
+    resp.merge(
+      :class => "widget_response",
+      :widget_key => event["widget_key"]
+    )
+  end
+
+  def self.execute_values_for(gauge, time)
+    gauge.field_values_at(time).sort do |a,b|
+      a.first.to_i <=> b.first.to_i
+    end.map do |a|
+      [a.first, a.second.to_i]
+    end
+  end
+
+  def data
+    super.merge(
+      :gauges => data_gauges,
+      :autoupdate => (@opts[:autoupdate] || 0),
+      :render_target => @opts[:render_target],
+      :ticks => @opts[:ticks],
+      :click_callback => @opts[:click_callback],
+      :async_chart => true,
+      :tick => tick
+    ).tap do |dat|
+      dat.merge!(
+        :gauges => @opts[:_gauges]
+      ) if dat[:ticks]
+    end
+  end
+
+  def data_gauges
+    Hash.new.tap do |hash|
+      gauges.each do |g|
+        hash[g.name] = {
+          :tick => g.tick,
+          :title => g.title
+        }
+      end
+    end
+  end
+
+  def has_tick?
+    false
+  end
+
+end