johnf-fnordmetric 1.2.7

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem "rake"
+gem "yajl-ruby", :git => "git://github.com/brianmario/yajl-ruby.git"

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rspec'
+require 'rspec/core/rake_task'
+desc "Run all examples"
+task RSpec::Core::RakeTask.new('fnordmetric-core/spec')
+
+task :default => "spec"
+
+require "bundler/gem_tasks"
+

data/fnordmetric.gemspec

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "date"
+require "fnordmetric/version"
+
+Gem::Specification.new do |s|
+  s.name        = "johnf-fnordmetric"
+  s.version     = FnordMetric::VERSION
+  s.date        = Date.today.to_s
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Paul Asmuth"]
+  s.email       = ["paul@paulasmuth.com"]
+  s.homepage    = "http://github.com/paulasmuth/fnordmetric"
+  s.summary     = %q{FnordMetric is a Ruby Event-Tracking gem on steroids}
+  s.description = %q{FnordMetric is a Ruby Event-Tracking gem on steroids}
+  s.licenses    = ["MIT"]
+
+  s.add_dependency "sinatra", ">= 1.2.6"
+  s.add_dependency "redis", ">= 2.2.2"
+  s.add_dependency "eventmachine"
+  s.add_dependency "websocket-rack", "0.4.0"
+  s.add_dependency "em-hiredis", ">= 0.1.1"
+  s.add_dependency "json"
+  s.add_dependency "i18n"
+  s.add_dependency "haml"
+  s.add_dependency "rack"
+  s.add_dependency "rack-test"
+  s.add_dependency "yajl-ruby"
+  s.add_dependency "thin", ">= 1.3.0"
+  s.add_dependency "activesupport"
+
+  s.add_development_dependency "delorean"
+  s.add_development_dependency "rspec", "~> 2.8.0"
+  s.add_development_dependency "shoulda"
+
+  s.files         = `git ls-files`.split("\n") - [".gitignore", ".rspec", ".travis.yml"]
+  s.files         += ["web/fnordmetric-ui.js", "web/fnordmetric-ui.css", "web/fnordmetric-core.js", "web/fnordmetric-core.css"]
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/fnordmetric/acceptors/acceptor.rb

@@ -0,0 +1,42 @@
+class FnordMetric::Acceptor
+
+  def initialize(opts)
+    @opts = opts
+
+    FnordMetric.register(self)
+  end
+
+  def initialized
+    inbound_class = if @opts[:protocol] == :udp
+      FnordMetric::UDPAcceptor
+    elsif @opts[:protocol] == :tcp
+      FnordMetric::TCPAcceptor
+    elsif @opts[:protocol] == :fyrehose
+      FnordMetric::FyrehoseAcceptor
+    elsif @opts[:protocol] == :amqp
+      FnordMetric::AMQPAcceptor
+    elsif @opts[:protocol] == :stomp
+      FnordMetric::STOMPAcceptor
+    else
+      raise "unknown protocol: #{@opts[:protocol]}"
+    end
+
+    @opts[:listen] = [
+      @opts[:host] || "0.0.0.0",
+      @opts[:port] || 2323
+    ]
+
+    begin
+      inbound_stream = inbound_class.start(@opts)
+      if inbound_class.respond_to?(:outbound?) && inbound_class.outbound?
+        FnordMetric.log "connected to #{@opts[:protocol]}://#{@opts[:listen][0..1].join(":")}"
+      else
+        FnordMetric.log "listening on #{@opts[:protocol]}://#{@opts[:listen][0..1].join(":")}"
+      end
+    rescue Exception => e
+      raise e if ENV["FNORDMETRIC_ENV"] == "dev"
+      FnordMetric.log "cant start #{inbound_class.name} on #{@opts[:protocol]}://#{@opts[:listen][0..1].join(":")}. port in use?"
+    end
+  end
+
+end

data/lib/fnordmetric/acceptors/amqp_acceptor.rb

@@ -0,0 +1,56 @@
+class FnordMetric::AMQPAcceptor
+
+  def self.start(opts)
+    begin
+      require "amqp"
+    rescue LoadError
+      FnordMetric.error("require 'amqp' failed, you need the amqp gem")
+      exit 1
+    end
+
+    new(opts)
+  end
+
+  def initialize(opts)
+    amqp = AMQP.connect(:host => 'firehose')
+    amqp_channel = AMQP::Channel.new(amqp)
+
+    msg_handler = lambda do |channel, data|
+      event = begin
+        JSON.parse(data)
+      rescue
+        FnordMetric.log("[AMQP] received invalid JSON: #{data[0..60]}")
+      end
+
+      if event
+        event["_type"] ||= channel
+        events << event
+        push_next_event
+      end
+    end
+
+    opts[:channels].each do |channel|
+      queue = amqp_channel.queue(channel, :auto_delete => true)
+      queue.subscribe{ |data| msg_handler[channel, data] }
+    end
+  end
+
+  def push_next_event
+    return true if events.empty?
+    api.event(@events.pop)
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def events
+    @events ||= []
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+  def self.outbound?
+    true
+  end
+
+end

data/lib/fnordmetric/acceptors/fyrehose_acceptor.rb

@@ -0,0 +1,43 @@
+class FnordMetric::FyrehoseAcceptor
+
+  def self.start(opts)
+    require "fyrehose"
+    require "fyrehose/reactor"
+
+    new(opts)
+  end
+
+  def initialize(opts)
+    reactor = EM.connect(opts[:host], opts[:port], Fyrehose::Reactor)
+
+    reactor.on_message do |channel, data|
+      event = JSON.parse(data)
+      event["_type"] ||= channel
+      events << event
+      push_next_event
+    end
+
+    opts[:channels].each do |channel|
+      reactor.subscribe(channel)
+    end
+  end
+
+  def push_next_event
+    return true if events.empty?
+    api.event(@events.pop)
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def events
+    @events ||= []
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+  def self.outboud?
+    true
+  end
+
+end

data/lib/fnordmetric/acceptors/stomp_acceptor.rb

@@ -0,0 +1,71 @@
+class FnordMetric::STOMPAcceptor
+
+  def self.start(opts)
+    begin
+      require "stomp"
+    rescue LoadError
+      FnordMetric.error("require 'stomp' failed, you need the stomp gem")
+      exit 1
+    end
+
+    new(opts)
+  end
+
+  def initialize(opts)
+    @mutex = Mutex.new
+
+    client = Stomp::Client.new(:hosts => [{
+      :host => opts[:host],
+      :port => opts[:port],
+      :passcode => opts[:password],
+      :login => opts[:username]}])
+
+    msg_handler = lambda do |topic, msg|
+      data = msg.body
+
+      event = begin
+        JSON.parse(data)
+      rescue
+        FnordMetric.log("[STOMP] received invalid JSON: #{data[0..60]}")
+      end
+
+      if event
+        event["_type"] ||= topic.gsub(/^\/topic\//, '')
+        @mutex.synchronize{ events << event }
+      end
+    end
+
+    opts[:topics].each do |topic|
+      client.subscribe(topic){ |data| msg_handler[topic, data] }
+    end
+
+    Thread.new do
+      client.join
+    end
+
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def push_next_event
+    nxt = @mutex.synchronize{ events.pop }
+    unless nxt
+      EM::Timer.new(0.01, &method(:push_next_event))
+      return true
+    end
+    api.event(nxt)
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def events
+    @events ||= []
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+  def self.outbound?
+    true
+  end
+
+end

data/lib/fnordmetric/acceptors/tcp_acceptor.rb

@@ -0,0 +1,58 @@
+class FnordMetric::TCPAcceptor < EventMachine::Connection
+  @@opts = nil
+
+  def self.start(opts)
+    @@opts = opts
+    EM.start_server(*(opts[:listen] + [self]))
+  end
+
+  def self.options(opts)
+    @@opts = opts
+  end
+
+  def receive_data(chunk)
+    @buffer << chunk
+    next_event
+  end
+
+  def next_event
+    read_next_event
+    push_next_event
+  end
+
+  def read_next_event
+    while (event = @buffer.slice!(/^(.*)\n/))
+      @events_buffered += 1
+      @events << event
+    end
+  end
+
+  def push_next_event
+    return true if @events.empty?
+    @events_buffered -= 1
+    api.event(@events.pop)
+    close_connection?
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def close_connection?
+    #@backend.hangup unless @streaming || (@events_buffered!=0)
+  end
+
+  def post_init
+    @events_buffered = 0
+    @streaming = true
+    @buffer = ""
+    @events = []
+  end
+
+  def unbind
+    @streaming = false
+    close_connection?
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+end

data/lib/fnordmetric/acceptors/udp_acceptor.rb

@@ -0,0 +1,37 @@
+class FnordMetric::UDPAcceptor < EventMachine::Connection
+
+  class << self
+    attr_accessor :opts
+  end
+
+  def self.start(opts)
+    self.opts = opts
+
+    EM.open_datagram_socket(*(opts[:listen] << self << opts))
+  end
+
+  def receive_data(event)
+    events << event
+    push_next_event
+  end
+
+  def push_next_event
+    return true if events.empty?
+    ev = @events.pop
+    api.event(ev)
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def unbind
+    #backend.hangup
+  end
+
+  def events
+    @events ||= []
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+end

data/lib/fnordmetric/api.rb

@@ -0,0 +1,46 @@
+class FnordMetric::API
+
+  @@opts = nil
+
+  def initialize(opts={})
+    @@opts = FnordMetric.default_options(opts)
+    connect
+  end
+
+  def connect
+    @redis = Redis.connect(:url => @@opts[:redis_url])
+  end
+
+  def event(event_data)
+    begin
+      if event_data.is_a?(Hash)
+        event_data = event_data.to_json
+      else
+        JSON.parse(event_data) # void ;)
+      end
+    rescue JSON::ParserError
+      FnordMetric.log("event_lost: can't parse json")
+    else
+      push_event(get_next_uuid, event_data)
+    end
+  end
+
+  def disconnect
+    @redis.quit
+  end
+
+  private
+
+  def push_event(event_id, event_data)
+    prefix = @@opts[:redis_prefix]
+    @redis.hincrby "#{prefix}-stats",             "events_received", 1
+    @redis.set     "#{prefix}-event-#{event_id}", event_data
+    @redis.lpush   "#{prefix}-queue",             event_id
+    @redis.expire  "#{prefix}-event-#{event_id}", @@opts[:event_queue_ttl]
+    event_id
+  end
+
+  def get_next_uuid
+    rand(8**32).to_s(36)
+  end
+end

data/lib/fnordmetric/cache.rb

@@ -0,0 +1,20 @@
+class FnordMetric::Cache
+#  include Mongoid::Document
+
+#  self.collection_name = 'fnordmetric_cache' 
+
+#  field :cache_key, :type => String
+#  field :data, :type => Hash
+
+  def self.store!(cache_key, data)
+    data = { :value => data } unless data.is_a?(Hash)
+    self.create(:cache_key => cache_key, :data => data)
+  end
+
+  def self.get(cache_key)
+    item = self.where(:cache_key => cache_key).last
+    return nil unless item
+    item.data.keys == ["value"] ? item.data["value"] : item.data
+  end
+
+end

data/lib/fnordmetric/context.rb

@@ -0,0 +1,96 @@
+class FnordMetric::Context
+
+  include FnordMetric::GaugeModifiers
+
+  class Proxy
+
+    def initialize(_ref)
+      @ref = _ref
+    end
+
+    def method_missing(method, *args, &block)
+      @ref.dispatch(method, *args, &block)
+    end
+
+  end
+
+  def initialize(opts, block)
+    @block = block
+    @opts = opts
+  end
+
+  def call(event, redis, namespace)
+    @redis = redis
+    @event = event
+    @namespace = namespace
+    proxy.instance_eval(&@block)
+  rescue Exception => e
+    raise e if ENV['FNORDMETRIC_ENV'] == 'test'
+    puts "error: #{e.message}"
+    puts e.backtrace.push("").join("\n") if ENV['FNORDMETRIC_ENV'] == 'dev'
+  end
+
+  def proxy
+    @proxy ||= Proxy.new(self)
+  end
+
+  def namespace
+    @namespace
+  end
+
+  def dispatch(method, *args, &block)
+    if args.size > 0 && gauges[args[0]].try(:renderable?)
+      gauges[args.delete_at(0)].execute(method, *args.unshift(self), &block)
+    else
+      send(method, *args, &block)
+    end
+  rescue Exception => e
+    raise e if ENV['FNORDMETRIC_ENV'] == 'test'
+    puts "error: #{e.message}"
+    puts e.backtrace.push("\n").join("\n") if ENV['FNORDMETRIC_ENV'] == 'dev'
+  end
+
+  def redis_exec(*args)
+    @redis.send(*args)
+  end
+
+private
+
+  def session_key
+    @event[:_session_key]
+  end
+
+  def data
+    @event
+  end
+
+  def key(gauge)
+    fetch_gauge(gauge).key
+  end
+
+  def time
+    @event[:_time].to_i
+  end
+
+  def type
+    @event[:_type].to_sym
+  end
+
+  def gauges
+    @namespace.gauges
+  end
+
+protected
+
+  def fetch_gauge(_gauge)
+    _gauge.is_a?(FnordMetric::Gauge) ? _gauge : gauges.fetch(_gauge)
+  rescue
+    error! "error: gauge '#{_gauge}' is undefined"
+  end
+
+  def error!(msg)
+    FnordMetric.error(msg)
+  end
+
+
+end

data/lib/fnordmetric/defaults.rb

@@ -0,0 +1,22 @@
+FnordMetric::COLORS = ["#4572a7", "#aa4643", "#89a54e", "#80699b", "#3d96ae", "#db843d"].reverse
+
+FnordMetric::DEFAULT_PROC = lambda{ |arg| }
+
+FnordMetric::TICKS = lambda{ |tick, span| [tick, 60, 300, 1200, 3600, 86400]
+  .select{ |t| (t >= tick) && ((span/t) > 5) }
+  .uniq }
+
+FnordMetric::DEFAULT_OPTIONS = {
+  :redis_url => "redis://localhost:6379",
+  :redis_prefix => "fnordmetric",
+  :inbound_stream => nil,
+  :inbound_protocol => nil,
+  :web_interface => ["0.0.0.0", "4242"],
+  :web_interface_server => "thin",
+  :start_worker => true,
+  :print_stats => 3,
+  :event_queue_ttl => 120,
+  :event_data_ttl => 3600*24*30,
+  :session_data_ttl => 3600*24*30,
+  :default_flush_interval => 10
+}

data/lib/fnordmetric/enterprise/compatibility_handler.rb

@@ -0,0 +1,42 @@
+module FnordMetric::Enterprise
+
+  CompatibilityHandler = proc do
+    if data[:_cmd]
+     cmd = data[:_cmd].to_s
+    else
+      FnordMetric.error("missing key: _cmd")
+      next
+    end
+
+    if m = cmd.match(/^SAMPLE (.*)(delta|mean|sum)-([0-9]+) ([0-9]+)$/)
+      gauge_type = m[2].to_sym
+      gauge_tick = m[3].to_i
+      gauge_key = :"#{m[1]}#{gauge_type}-#{gauge_tick}"
+      op_value = m[4].to_i
+    else
+      FnordMetric.error("invalid _cmd")
+      next
+    end
+
+    gauge = if namespace.gauges.has_key?(gauge_key)
+      namespace.gauges[gauge_key]
+    else
+      namespace.opt_gauge(gauge_key,
+        :flush_interval => gauge_tick,
+        :average => (gauge_type == :mean),
+        :zero_config => true)
+    end
+
+    case gauge_type
+
+      when :delta, :sum
+        incr gauge, op_value
+
+      when :mean
+        incr_avg gauge, op_value
+
+    end
+
+  end
+
+end

data/lib/fnordmetric/ext.rb

@@ -0,0 +1,75 @@
+module Enumerable
+
+  def each_with_log(total = nil, &block)
+    log_every = ((total ||= self.count) / 150)
+    self.each_with_index do |item, index|
+      if index % log_every == 0
+        STDOUT.puts "#{index}/#{total} (#{((index/total.to_f)*100).to_i}%)" 
+      end
+      block.call(item, index)
+    end
+  end
+
+end
+
+module Haml::Filters::Gaugejs
+  include Haml::Filters::Base
+
+  def render(text)
+    "<FNORDMETRIC-GAUGEJS>#{text}</FNORDMETRIC-GAUGEJS>"
+  end
+end
+
+class Symbol
+  alias :intern :to_sym
+end
+
+class Range
+
+  def size
+    self.last - self.first
+  end
+
+end
+
+class Array
+
+  def mean
+    return 0 if empty?
+    inject(&:+).to_f / size
+  end
+
+  alias :average :mean
+
+  def median
+    return 0 if empty?
+    (_sorted = self.dup.sort)[_sorted.size/2]
+  end
+
+  def range
+    return 0 if empty?
+    max - min
+  end
+
+  def mode
+    return 0 if empty?
+    inject({}){ |h,v| h[v] = h[v].to_i+1; h }.to_a
+    .sort{ |a,b| b.last <=> a.last }[0][0]
+  end
+  
+  def emtpy
+    self.size == 0
+  end
+
+end
+
+class Thin::Connection
+
+  alias :pre_process_orig :pre_process
+
+  def pre_process
+    @request.env['async.connection'] = self
+    pre_process_orig
+  end
+
+end