fnordmetric 1.0.1 → 1.2.0

data/Rakefile

@@ -1,6 +1,6 @@
 require 'rspec'
 require 'rspec/core/rake_task'
 desc "Run all examples"
-task RSpec::Core::RakeTask.new('spec')
+task RSpec::Core::RakeTask.new('fnordmetric-core/spec')
 
 task :default => "spec"

data/fnordmetric.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "shoulda"
 
   s.files         = `git ls-files`.split("\n") - [".gitignore", ".rspec", ".travis.yml"]
+  s.files         += ["web/fnordmetric-ui.js", "web/fnordmetric-ui.css", "web/fnordmetric-core.js", "web/fnordmetric-core.css"]
   s.test_files    = `git ls-files -- spec/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]

data/lib/fnordmetric.rb

@@ -51,19 +51,7 @@ module FnordMetric
   end
 
   def self.default_options(opts = {})
-    {
-      :redis_url => "redis://localhost:6379",
-      :redis_prefix => "fnordmetric",
-      :inbound_stream => ["0.0.0.0", "1337"],
-      :inbound_protocol => :tcp,
-      :web_interface => ["0.0.0.0", "4242"],
-      :web_interface_server => "thin",
-      :start_worker => true,
-      :print_stats => 3,
-      :event_queue_ttl => 120,
-      :event_data_ttl => 3600*24*30,
-      :session_data_ttl => 3600*24*30
-    }.merge(opts)
+    FnordMetric::DEFAULT_OPTIONS.merge(opts)
   end
 
   def self.log(msg)
@@ -123,6 +111,7 @@ require "fnordmetric/gauge_modifiers"
 require "fnordmetric/gauge_validations"
 require "fnordmetric/gauge_rendering"
 require "fnordmetric/gauge"
+require "fnordmetric/zero_config_gauge"
 require "fnordmetric/gauges/timeseries_gauge"
 require "fnordmetric/gauges/toplist_gauge"
 require "fnordmetric/gauges/distribution_gauge"
@@ -136,6 +125,7 @@ require "fnordmetric/api"
 require "fnordmetric/worker"
 require "fnordmetric/logger"
 require "fnordmetric/defaults"
+require "fnordmetric/util"
 require "fnordmetric/web/web"
 require "fnordmetric/web/app_helpers"
 require "fnordmetric/web/app"
@@ -146,6 +136,9 @@ require "fnordmetric/web/dashboard"
 require "fnordmetric/acceptors/acceptor"
 require "fnordmetric/acceptors/tcp_acceptor"
 require "fnordmetric/acceptors/udp_acceptor"
+require "fnordmetric/acceptors/fyrehose_acceptor"
+require "fnordmetric/acceptors/amqp_acceptor"
+require "fnordmetric/acceptors/stomp_acceptor"
 require "fnordmetric/widget"
 require "fnordmetric/widgets/timeseries_widget"
 require "fnordmetric/widgets/numbers_widget"

data/lib/fnordmetric/acceptors/acceptor.rb

@@ -6,11 +6,19 @@ class FnordMetric::Acceptor
     FnordMetric.register(self)
   end
 
-  def initialized   
-    inbound_class = if @opts[:protocol] == :udp 
+  def initialized
+    inbound_class = if @opts[:protocol] == :udp
       FnordMetric::UDPAcceptor
-    else
+    elsif @opts[:protocol] == :tcp
       FnordMetric::TCPAcceptor
+    elsif @opts[:protocol] == :fyrehose
+      FnordMetric::FyrehoseAcceptor
+    elsif @opts[:protocol] == :amqp
+      FnordMetric::AMQPAcceptor
+    elsif @opts[:protocol] == :stomp
+      FnordMetric::STOMPAcceptor
+    else
+      raise "unknown protocol: #{@opts[:protocol]}"
     end
 
     @opts[:listen] = [
@@ -20,10 +28,15 @@ class FnordMetric::Acceptor
 
     begin
       inbound_stream = inbound_class.start(@opts)
-      FnordMetric.log "listening on #{@opts[:protocol]}://#{@opts[:listen][0..1].join(":")}"
-    #rescue
-    #  FnordMetric.log "cant start #{inbound_class.name}. port in use?"
+      if inbound_class.respond_to?(:outbound?) && inbound_class.outbound?
+        FnordMetric.log "connected to #{@opts[:protocol]}://#{@opts[:listen][0..1].join(":")}"
+      else
+        FnordMetric.log "listening on #{@opts[:protocol]}://#{@opts[:listen][0..1].join(":")}"
+      end
+    rescue Exception => e
+      raise e if ENV["FNORDMETRIC_ENV"] == "dev"
+      FnordMetric.log "cant start #{inbound_class.name} on #{@opts[:protocol]}://#{@opts[:listen][0..1].join(":")}. port in use?"
     end
   end
 
-end
+end

data/lib/fnordmetric/acceptors/amqp_acceptor.rb

@@ -0,0 +1,56 @@
+class FnordMetric::AMQPAcceptor
+
+  def self.start(opts)
+    begin
+      require "amqp"
+    rescue LoadError
+      FnordMetric.error("require 'amqp' failed, you need the amqp gem")
+      exit 1
+    end
+
+    new(opts)
+  end
+
+  def initialize(opts)
+    amqp = AMQP.connect(:host => 'firehose')
+    amqp_channel = AMQP::Channel.new(amqp)
+
+    msg_handler = lambda do |channel, data|
+      event = begin
+        JSON.parse(data)
+      rescue
+        FnordMetric.log("[AMQP] received invalid JSON: #{data[0..60]}")
+      end
+
+      if event
+        event["_type"] ||= channel
+        events << event
+        push_next_event
+      end
+    end
+
+    opts[:channels].each do |channel|
+      queue = amqp_channel.queue(channel, :auto_delete => true)
+      queue.subscribe{ |data| msg_handler[channel, data] }
+    end
+  end
+
+  def push_next_event
+    return true if events.empty?
+    api.event(@events.pop)
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def events
+    @events ||= []
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+  def self.outbound?
+    true
+  end
+
+end

data/lib/fnordmetric/acceptors/fyrehose_acceptor.rb

@@ -0,0 +1,43 @@
+class FnordMetric::FyrehoseAcceptor
+
+  def self.start(opts)
+    require "fyrehose"
+    require "fyrehose/reactor"
+
+    new(opts)
+  end
+
+  def initialize(opts)
+    reactor = EM.connect(opts[:host], opts[:port], Fyrehose::Reactor)
+
+    reactor.on_message do |channel, data|
+      event = JSON.parse(data)
+      event["_type"] ||= channel
+      events << event
+      push_next_event
+    end
+
+    opts[:channels].each do |channel|
+      reactor.subscribe(channel)
+    end
+  end
+
+  def push_next_event
+    return true if events.empty?
+    api.event(@events.pop)
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def events
+    @events ||= []
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+  def self.outboud?
+    true
+  end
+
+end

data/lib/fnordmetric/acceptors/stomp_acceptor.rb

@@ -0,0 +1,71 @@
+class FnordMetric::STOMPAcceptor
+
+  def self.start(opts)
+    begin
+      require "stomp"
+    rescue LoadError
+      FnordMetric.error("require 'stomp' failed, you need the stomp gem")
+      exit 1
+    end
+
+    new(opts)
+  end
+
+  def initialize(opts)
+    @mutex = Mutex.new
+
+    client = Stomp::Client.new(:hosts => [{
+      :host => opts[:host],
+      :port => opts[:port],
+      :passcode => opts[:password],
+      :login => opts[:username]}])
+
+    msg_handler = lambda do |topic, msg|
+      data = msg.body
+
+      event = begin
+        JSON.parse(data)
+      rescue
+        FnordMetric.log("[STOMP] received invalid JSON: #{data[0..60]}")
+      end
+
+      if event
+        event["_type"] ||= topic.gsub(/^\/topic\//, '')
+        @mutex.synchronize{ events << event }
+      end
+    end
+
+    opts[:topics].each do |topic|
+      client.subscribe(topic){ |data| msg_handler[topic, data] }
+    end
+
+    Thread.new do
+      client.join
+    end
+
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def push_next_event
+    nxt = @mutex.synchronize{ events.pop }
+    unless nxt
+      EM::Timer.new(0.01, &method(:push_next_event))
+      return true
+    end
+    api.event(nxt)
+    EM.next_tick(&method(:push_next_event))
+  end
+
+  def events
+    @events ||= []
+  end
+
+  def api
+    @api ||= FnordMetric::API.new(FnordMetric.options)
+  end
+
+  def self.outbound?
+    true
+  end
+
+end

data/lib/fnordmetric/acceptors/tcp_acceptor.rb

@@ -54,4 +54,5 @@ class FnordMetric::TCPAcceptor < EventMachine::Connection
   def api
     @api ||= FnordMetric::API.new(FnordMetric.options)
   end
+
 end

data/lib/fnordmetric/acceptors/udp_acceptor.rb

@@ -33,4 +33,5 @@ class FnordMetric::UDPAcceptor < EventMachine::Connection
   def api
     @api ||= FnordMetric::API.new(FnordMetric.options)
   end
-end
+
+end

data/lib/fnordmetric/context.rb

@@ -16,12 +16,13 @@ class FnordMetric::Context
 
   def initialize(opts, block)
     @block = block
-    @opts = opts    
+    @opts = opts
   end
 
-  def call(event, redis)
+  def call(event, redis, namespace)
     @redis = redis
-    @event = event    
+    @event = event
+    @namespace = namespace
     proxy.instance_eval(&@block)
   rescue Exception => e
     raise e if ENV['FNORDMETRIC_ENV'] == 'test'
@@ -33,9 +34,13 @@ class FnordMetric::Context
     @proxy ||= Proxy.new(self)
   end
 
+  def namespace
+    @namespace
+  end
+
   def dispatch(method, *args, &block)
-    if args.size > 0 && @opts[:gauges][args[0]].try(:renderable?)
-      @opts[:gauges][args.delete_at(0)].execute(method, *args.unshift(self), &block)
+    if args.size > 0 && gauges[args[0]].try(:renderable?)
+      gauges[args.delete_at(0)].execute(method, *args.unshift(self), &block)
     else
       send(method, *args, &block)
     end
@@ -56,7 +61,7 @@ private
   end
 
   def data
-  	@event
+    @event
   end
 
   def key(gauge)
@@ -67,10 +72,18 @@ private
     @event[:_time].to_i
   end
 
+  def type
+    @event[:_type].to_sym
+  end
+
+  def gauges
+    @namespace.gauges
+  end
+
 protected
-  
+
   def fetch_gauge(_gauge)
-    _gauge.is_a?(FnordMetric::Gauge) ? _gauge : @opts[:gauges].fetch(_gauge)
+    _gauge.is_a?(FnordMetric::Gauge) ? _gauge : gauges.fetch(_gauge)
   rescue
     error! "error: gauge '#{_gauge}' is undefined"
   end
@@ -81,4 +94,3 @@ protected
 
 
 end
-    

data/lib/fnordmetric/defaults.rb

@@ -1,9 +1,22 @@
 FnordMetric::COLORS = ["#4572a7", "#aa4643", "#89a54e", "#80699b", "#3d96ae", "#db843d"].reverse
 
+FnordMetric::DEFAULT_PROC = lambda{ |arg| }
+
 FnordMetric::TICKS = lambda{ |tick, span| [tick, 60, 300, 1200, 3600, 86400]
   .select{ |t| (t >= tick) && ((span/t) > 5) }
   .uniq }
 
-
-FnordMetric::DEFAULT_PROC = lambda{ |arg| }
-
+FnordMetric::DEFAULT_OPTIONS = {
+  :redis_url => "redis://localhost:6379",
+  :redis_prefix => "fnordmetric",
+  :inbound_stream => nil,
+  :inbound_protocol => nil,
+  :web_interface => ["0.0.0.0", "4242"],
+  :web_interface_server => "thin",
+  :start_worker => true,
+  :print_stats => 3,
+  :event_queue_ttl => 120,
+  :event_data_ttl => 3600*24*30,
+  :session_data_ttl => 3600*24*30,
+  :default_flush_interval => 10
+}

data/lib/fnordmetric/gauge.rb

@@ -11,7 +11,8 @@ class FnordMetric::Gauge
   end
 
   def tick
-    (@opts[:tick] || @opts[:resolution] || 3600).to_i
+    (@opts[:tick] || @opts[:resolution] || @opts[:flush_interval] ||
+      FnordMetric.options[:default_flush_interval]).to_i
   end
 
   def retention
@@ -34,10 +35,14 @@ class FnordMetric::Gauge
     @opts[:group] || "Gauges"
   end
 
+  def unit
+    @opts[:unit]
+  end
+
   def key_nouns
     @opts[:key_nouns] || ["Key", "Keys"]
   end
-  
+
   def key(_append=nil)
     [@opts[:key_prefix], "gauge", name, tick, _append].flatten.compact.join("-")
   end
@@ -89,5 +94,5 @@ class FnordMetric::Gauge
   def error!(msg)
     FnordMetric.error(msg)
   end
-  
+
 end

data/lib/fnordmetric/gauge_calculations.rb

@@ -52,12 +52,18 @@ module FnordMetric::GaugeCalculations
     block = @@avg_per_count_proc if average?
     #block = @@count_per_session_proc if unique?
     block = @@avg_per_session_proc if unique? && average?
-    
-    if block
+
+    calc = if block
       instance_exec(_v, _t, &block)
     else
       _v
     end
+
+    if calc && @opts[:scale_by]
+      calc = calc.to_f * @opts[:scale_by].to_f
+    end
+
+    calc
   end
 
   def field_values_at(time, opts={}, &block)

data/lib/fnordmetric/gauge_modifiers.rb

@@ -1,6 +1,7 @@
 module FnordMetric::GaugeModifiers
 
   def incr(gauge_name, value=1)
+    value = value.to_i
     gauge = fetch_gauge(gauge_name)
     assure_two_dimensional!(gauge)
     if gauge.unique? 
@@ -13,7 +14,7 @@ module FnordMetric::GaugeModifiers
   end
 
   def incr_tick(gauge, value)
-    if gauge.progressive?      
+    if gauge.progressive?
       @redis.incrby(gauge.key(:head), value).callback do |head|
         @redis.hsetnx(gauge.key, gauge.tick_at(time), head).callback do |_new|
           @redis.hincrby(gauge.key, gauge.tick_at(time), value) unless _new
@@ -24,7 +25,7 @@ module FnordMetric::GaugeModifiers
         @redis.hincrby(gauge.key, gauge.tick_at(time), value)
       end
     end
-  end  
+  end
 
   def incr_uniq(gauge, value, field_name=nil)
     return false if session_key.blank?
@@ -58,7 +59,7 @@ module FnordMetric::GaugeModifiers
     @redis.zincrby(gauge.tick_key(time), value, field_name).callback do
       @redis.incrby(gauge.tick_key(time, :count), 1)
     end
-  end  
+  end
 
   def set_value(gauge_name, value)
     gauge = fetch_gauge(gauge_name)
@@ -140,4 +141,4 @@ module FnordMetric::GaugeModifiers
     end
   end
 
-end
+end