fnordmetric 0.7.5 → 0.9.7

data/readme.md

@@ -0,0 +1,362 @@
+FnordMetric
+===========
+
+FnordMetric is a highly configurable (and pretty fast) realtime app/event 
+tracking thing based on ruby eventmachine and redis. You define your own 
+plotting and counting functions as ruby blocks!
+
+[ ![Build status - Travis-ci](https://secure.travis-ci.org/paulasmuth/fnordmetric.png) ](http://travis-ci.org/paulasmuth/fnordmetric)
+
+
+[ ![Screenshot](https://github.com/paulasmuth/fnordmetric/raw/v1.0-alpha/doc/preview3.png) ](http://github.com/paulasmuth/fnordmetric)
+
+
+
+Getting Started
+---------------
+
+Simple Example: this will listen for json-events with `type=unicorn_seen` 
+and render a timeline-plot showing the number of received events per hour.
+
+```ruby
+require "fnordmetric"
+
+FnordMetric.namespace :myapp do
+
+  # timeline+plot and punchcard
+  timeseries_gauge :unicorns_seen_per_hour, 
+    :resolution => 5.minutes,
+    :title => "Unicorns seenper Hour",
+    :punchcard => true
+
+  # on every event like { _type: 'unicorn_seen' }
+  event(:unicorn_seen) do
+    # increment the unicorns_seen_per_hour gauge by 1
+    incr :unicorns_seen_per_hour
+  end
+
+end
+
+FnordMetric.standalone
+```
+
+This is the easiest way to submit an event:
+
+    echo '{"_type": "unicorn_seen"}' | nc localhost 1337
+
+
+
+Installation
+------------
+
+    gem install fnordmetric
+
+or in your Gemfile:
+
+    gem 'fnordmetric', '>= 0.9.6'
+
+
+Documentation
+-------------
+
+Check out the docs in [the wiki](http://github.com/paulasmuth/fnordmetric/wiki) or have a look at `doc/full_example.rb`. There are also a few blogs/howtos:
+
++ [Blog: Monitor your Python App With FnordMetric](http://stephenholiday.com/articles/2012/monitor-your-python-app-with-fnordmetric/)
++ [Blog: FnordMetric and C (Blog/Howto)](http://johnmurray.io/log/2012/01/19/FnordMetrics-and-C%23.md)
++ [Blog: FnordMetric HowTo (russian)](http://www.pvsm.ru/ruby/2723)
+
+
+### More Resources
+
++ [PHP API for FnordMetric (github.com/leemachin/fnordmetric-php-api)](https://github.com/leemachin/fnordmetric-php-api)
++ [Python API for FnordMetric (github.com/sholiday/pyfnordmetric)](https://github.com/sholiday/pyfnordmetric)
++ [Another Ruby API for FnordMetric](https://github.com/savonarola/fnordmetric-client)
++ [Beanstalk Monitoring with FnordMetric](https://github.com/sholiday/fnordstalk)
++ [The "FnordMetric" google group](http://groups.google.com/group/fnordmetric)
+
+
+
+Full Example
+------------
+
+```ruby
+require "fnordmetric"
+
+FnordMetric.namespace :myapp do
+
+  hide_overview
+  hide_active_users
+
+
+  # NEW DSL (v1.0 upwards)
+
+  timeseries_gauge :number_of_signups,
+    :group => "My Group",
+    :title => "Number of Signups",
+    :key_nouns => ["Singup", "Signups"],
+    :series => [:via_twitter, :via_facebook],
+    :resolution => 2.minutes
+
+
+  distribution_gauge :user_age_distribution,
+    :title => "User Age Distribution",
+    :value_ranges => [(10..16), (16..20), (20..24), (24..28), (28..32), (32..36), (40..44), (44..48),
+                      (48..52), (52..56), (60..64), (64..68), (68..72), (72..76), (70..74), (74..78)],
+    :value_scale  => 1,
+    :resolution => 2.minutes
+
+
+  toplist_gauge :popular_keywords,
+    :title => "Popular Keywords",
+    :resolution => 2.minutes
+
+
+  event :search do
+    observe :popular_keywords, data[:keyword]
+  end
+
+  event :signup do
+    if data[:referrer] == "facebook"
+      incr :number_of_signups, :via_facebook, 1
+    elsif data[:referrer] == "twitter"
+      incr :number_of_signups, :via_twitter, 1
+    end
+  end
+
+
+  # OLD DSL (will be supported forever, allows finer-grained control)
+
+  gauge :events_per_hour, :tick => 1.hour
+  gauge :events_per_second, :tick => 1.second
+  gauge :events_per_minute, :tick => 1.minute
+
+  event :"*" do
+    incr :events_per_hour
+    incr :events_per_minute
+    incr :events_per_second
+  end
+
+
+  gauge :pageviews_daily_unique, :tick => 1.day.to_i, :unique => true, :title => "Unique Visits (Daily)"
+  gauge :pageviews_hourly_unique, :tick => 1.hour.to_i, :unique => true, :title => "Unique Visits (Hourly)"
+  gauge :pageviews_monthly_unique, :tick => 40.days.to_i, :unique => true, :title => "Unique Visits (Month)"
+
+  gauge :messages_sent, :tick => 1.day.to_i, :title => "Messages (sent)"
+  gauge :messages_read, :tick => 1.day.to_i, :title => "Messages (read)"
+  gauge :winks_sent, :tick => 1.day.to_i, :title => "Winks sent"
+
+  gauge :pageviews_per_url_daily,
+    :tick => 1.day.to_i,
+    :title => "Daily Pageviews per URL",
+    :three_dimensional => true
+
+  gauge :pageviews_per_url_monthly,
+    :tick => 30.days.to_i,
+    :title => "Monthly Pageviews per URL",
+    :three_dimensional => true
+
+  event :_pageview do
+    incr :pageviews_daily_unique
+    incr :pageviews_hourly_unique
+    incr :pageviews_monthly_unique
+    incr_field :pageviews_per_url_daily, data[:url]
+    incr_field :pageviews_per_url_monthly, data[:url]
+  end
+
+
+  widget 'TechStats', {
+    :title => "Events per Minute",
+    :type => :timeline,
+    :width => 100,
+    :gauges => :events_per_minute,
+    :include_current => true,
+    :autoupdate => 30
+  }
+
+  widget 'TechStats', {
+    :title => "Events per Hour",
+    :type => :timeline,
+    :width => 50,
+    :gauges => :events_per_hour,
+    :include_current => true,
+    :autoupdate => 30
+  }
+
+
+  widget 'TechStats', {
+    :title => "Events/Second",
+    :type => :timeline,
+    :width => 50,
+    :gauges => :events_per_second,
+    :include_current => true,
+    :plot_style => :areaspline,
+    :autoupdate => 1
+  }
+
+  widget 'TechStats', {
+    :title => "Events Numbers",
+    :type => :numbers,
+    :width => 100,
+    :gauges => [:events_per_second, :events_per_minute, :events_per_hour],
+    :offsets => [1,3,5,10],
+    :autoupdate => 1
+  }
+
+
+  gauge :age_distribution_female_monthly,
+    :tick => 1.month.to_i,
+    :three_dimensional => true,
+    :title => "Age Distribution (female) monthly"
+
+  gauge :age_distribution_male_monthly,
+    :tick => 1.month.to_i,
+    :three_dimensional => true,
+    :title => "Age Distribution (male) monthly"
+
+  gauge :age_distribution_female_daily,
+    :tick => 1.day.to_i,
+    :three_dimensional => true,
+    :title => "Age Distribution (female) daily"
+
+  gauge :age_distribution_male_daily,
+    :tick => 1.day.to_i,
+    :three_dimensional => true,
+    :title => "Age Distribution (male) daily"
+
+
+  widget 'Demography', {
+    :title => "Age Distribution: Female Users (Monthly)",
+    :type => :bars,
+    :width => 50,
+    :autoupdate => 5,
+    :order_by => :field,
+    :gauges => [ :age_distribution_female_monthly ]
+  }
+
+  widget 'Demography', {
+    :title => "Age Distribution: Male Users (Monthly)",
+    :type => :bars,
+    :width => 50,
+    :autoupdate => 5,
+    :order_by => :field,
+    :gauges => [ :age_distribution_male_monthly ]
+  }
+
+
+  widget 'Demography', {
+    :title => "Age Distribution: Female Users",
+    :type => :toplist,
+    :width => 50,
+    :autoupdate => 5,
+    :gauges => [ :age_distribution_female_monthly, :age_distribution_female_daily ]
+  }
+
+  widget 'Demography', {
+    :title => "Age Distribution: Male Users",
+    :type => :toplist,
+    :width => 50,
+    :autoupdate => 5,
+    :gauges => [ :age_distribution_male_monthly, :age_distribution_male_daily ]
+  }
+
+  event "user_demography" do
+    if data[:gender] == "female"
+      incr_field(:age_distribution_female_monthly, data[:age], 1)
+      incr_field(:age_distribution_female_daily, data[:age], 1)
+    end
+    if data[:gender] == "male"
+      incr_field(:age_distribution_male_monthly, data[:age], 1)
+      incr_field(:age_distribution_male_daily, data[:age], 1)
+    end
+    observe :user_age_distribution, data[:age]
+  end
+
+
+end
+
+FnordMetric.options = {
+  :event_queue_ttl  => 10, # all data that isn't processed within 10s is discarded to prevent memory overruns
+  :event_data_ttl   => 10,
+  :session_data_ttl => 1,  # we don't care about session data for now
+  :redis_prefix => "fnordmetric" 
+}
+
+def start_example_data_generator
+
+  api = FnordMetric::API.new
+  Thread.new do
+    loop do
+      api.event(:_type => :signup, :referrer => (rand(3) == 1 ? :twitter : :facebook))
+      api.event(:_type => :search, :keyword => (%w(Donau Dampf Schiff Fahrts Kaptitaens Muetzen Staender).shuffle[0..2] * ""))
+      api.event(:_type => :user_demography, :age => rand(15..85), :gender => (rand(2)==1 ? :female : :male) )
+      sleep (rand(10)/10.to_f)
+    end
+  end
+
+end
+
+start_example_data_generator
+
+FnordMetric::Web.new(:port => 4242)
+FnordMetric::Acceptor.new(:protocol => :tcp, :port => 2323)
+FnordMetric::Worker.new
+FnordMetric.run
+```
+
+
+
+
+Contributors
+------------
+
+http://github.com/paulasmuth/fnordmetric/graphs/contributors
+
+(One patch or more)
+
++ Simon Menke (http://github.com/fd)
++ Bruno Michel (http://github.com/nono)
++ Marco Borromeo (http://github.com/mborromeo)
++ Leo Lou (http://github.com/l4u)
++ Andy Lindeman (http://github.com/alindeman)
++ Jurriaan Pruis (http://github.com/jurriaan)
++ Kacper Bielecki (http://github.com/kazjote)
++ John Murray (http://github.com/JohnMurray)
++ Lars Gierth (http://github.com/lgierth)
++ Ross Kaffenberger (http://github.com/rossta)
++ Kunal Modi (http://github.com/kunalmodi)
++ Michael Fairchild (http://github.com/fairchild)
++ James Cox (http://github.com/imajes)
++ Pieter Noordhuis (http://github.com/pietern)
+
+To contribute, please fork this repository, make your changes and run the 
+specs, commit them to your github repository and send me a pull request.
+Need help, head on over to our [Google Groups][1]  page to discuss any ideas
+that you might have.
+
+
+License
+-------
+
+Copyright (c) 2011 Paul Asmuth
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to use, copy and modify copies of the Software, subject 
+to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+  [1]: http://groups.google.com/group/fnordmetric
+  [2]: http://www.screenr.com/KiJs
+  [3]: https://secure.travis-ci.org/paulasmuth/fnordmetric.png
+  [4]: http://travis-ci.org/paulasmuth/fnordmetric

data/spec/gauge_like_shared.rb

@@ -0,0 +1,54 @@
+share_examples_for FnordMetric::GaugeLike do
+
+  it "should remember its own name" do
+    gauge = @gauge_klass.new({:key_prefix => "foo", :key => "fnordgauge", :ticks => [1.hour], :series => [:fu]}) 
+    gauge.name.should == "fnordgauge"
+  end
+
+  it "should return its key as title if none specified" do
+    gauge = @gauge_klass.new({:key_prefix => "foo", :key => "fnordgauge", :ticks => [1.hour], :series => [:fu]}) 
+    gauge.title.should == "fnordgauge"
+  end
+
+  it "should return its title as title if none specified" do
+    gauge = @gauge_klass.new({:key_prefix => "foo", :key => "fnordgauge", :ticks => [1.hour], :title => "Fnord Gauge", :series => [:fu]}) 
+    gauge.title.should == "Fnord Gauge"
+  end
+
+  it "should raise an error when initialize without key" do
+    lambda{ 
+      @gauge_klass.new({:key_prefix => "foo", :ticks => [1.hour], :series => [:fu]})
+    }.should raise_error(key_error_klass)
+  end
+
+  it "should raise an error when initialize without key_prefix" do
+    lambda{ 
+      @gauge_klass.new({:key => "foo", :ticks => [1.hour], :series => [:fu]})
+    }.should raise_error(key_error_klass)
+  end
+
+  it "should generate the correct key without append" do
+    gauge = @gauge_klass.new({:key_prefix => "fnordmetrics-myns", :key => "mygauge", :tick => 23, :ticks => [1.hour], :series => [ :fnord ]})
+    if [@gauge_klass, @gauge_klass.superclass].include?(FnordMetric::MultiGauge)
+      gauge.key.should == "fnordmetrics-myns-multigauge-mygauge"
+    else
+      gauge.key.should == "fnordmetrics-myns-gauge-mygauge-23"
+    end
+  end
+
+  it "should generate the correct key with append" do
+    gauge = @gauge_klass.new({:key_prefix => "fnordmetrics-myns", :key => "mygauge", :tick => 23, :ticks => [1.hour], :series => [ :fnord ]})
+    if [@gauge_klass, @gauge_klass.superclass].include?(FnordMetric::MultiGauge)
+      gauge.key(:fnord).should == "fnordmetrics-myns-multigauge-mygauge-fnord"
+    else
+      gauge.key(:fnord).should == "fnordmetrics-myns-gauge-mygauge-23-fnord"
+    end
+  end
+
+  it "should add redis" do
+    gauge = @gauge_klass.new({:key_prefix => "fnordmetrics-myns", :key => "mygauge", :tick => 23, :ticks => [1.hour], :series => [ :fnord ]})
+    gauge.add_redis("FNORD")
+    gauge.instance_variable_get(:"@opts")[:redis].should == "FNORD"
+  end
+
+end

data/spec/gauge_spec.rb

@@ -6,48 +6,14 @@ describe FnordMetric::Gauge do
     @now = Time.utc(1992,01,13,5,23,23).to_i    
     @redis = Redis.new
     @redis_wrap = RedisWrap.new(@redis, false)
+    @gauge_klass = FnordMetric::Gauge
   end
 
   before(:each) do
     @redis.keys("fnordmetric-myns*").each { |k| @redis.del(k) }  
   end
 
-  it "should remember its own name" do
-    gauge = FnordMetric::Gauge.new({:key_prefix => "foo", :key => "fnordgauge"}) 
-    gauge.name.should == "fnordgauge"
-  end
-
-  it "should return its key as title if none specified" do
-    gauge = FnordMetric::Gauge.new({:key_prefix => "foo", :key => "fnordgauge"}) 
-    gauge.title.should == "fnordgauge"
-  end
-
-  it "should return its title as title if none specified" do
-    gauge = FnordMetric::Gauge.new({:key_prefix => "foo", :key => "fnordgauge", :title => "Fnord Gauge"}) 
-    gauge.title.should == "Fnord Gauge"
-  end
-
-  it "should raise an error when initialize without key" do
-    lambda{ 
-      FnordMetric::Gauge.new({:key_prefix => "foo"})
-    }.should raise_error(key_error_klass)
-  end
-
-  it "should raise an error when initialize without key_prefix" do
-    lambda{ 
-      FnordMetric::Gauge.new({:key => "foo"})
-    }.should raise_error(key_error_klass)
-  end
-
-  it "should generate the correct key without append" do
-    gauge = FnordMetric::Gauge.new({:key_prefix => "fnordmetrics-myns", :key => "mygauge", :tick => 23})
-    gauge.key.should == "fnordmetrics-myns-gauge-mygauge-23"
-  end
-
-  it "should generate the correct key with append" do
-    gauge = FnordMetric::Gauge.new({:key_prefix => "fnordmetrics-myns", :key => "mygauge", :tick => 23})
-    gauge.key(:fnord).should == "fnordmetrics-myns-gauge-mygauge-23-fnord"
-  end
+  it_should_behave_like FnordMetric::GaugeLike
 
   describe "ticks" do
 

data/spec/namespace_spec.rb

@@ -2,7 +2,6 @@ require ::File.expand_path('../spec_helper.rb', __FILE__)
 
 describe FnordMetric::Namespace do
 
-
   before(:all) do
     @redis = Redis.new
     @redis_wrap = RedisWrap.new(@redis)
@@ -10,13 +9,12 @@ describe FnordMetric::Namespace do
 
   before(:each) do
     @redis.keys("fnordmetric-myns*").each { |k| @redis.del(k) }  
+    @namespace = FnordMetric::Namespace.new(:myns_213, :redis_prefix => "fnordmetric")
   end
 
-  describe "instance methods" do
+  it "should generate the correct redis prefix"
 
-    before(:each) do
-      @namespace = FnordMetric::Namespace.new(:myns_213, :redis_prefix => "fnordmetric")
-    end
+  describe "instance methods" do
 
     it "should create a new dashboard if a widget is added" do
       @namespace.widget("MyFooDash", nil)
@@ -33,18 +31,34 @@ describe FnordMetric::Namespace do
 
   end
 
+  describe "registering gauges" do
 
-  it "should generate the correct redis prefix"
+    it "should register a simple gauge" do
+      @namespace.gauge(:fugauge, {:fnord => 23})
+      @namespace.gauges[:fugauge].should be_a(FnordMetric::Gauge) 
+    end
 
-  it "should register a gauge"
-  it "should register a gauge and pass options"
+    it "should register a simple gauge and pass options" do
+      @namespace.gauge(:fugauge2, {:fnord => 23})
+      @namespace.gauges[:fugauge2].instance_variable_get(:@opts).should include({:fnord => 23})
+      @namespace.gauges[:fugauge2].instance_variable_get(:@opts).should include({:key => :fugauge2})
+    end
 
-  describe "registering event handlers" do
+    it "should register a multi gauge" do
+      @namespace.numeric_gauge(:multigauge, {:fnord => 23, :ticks => [1.hour], :series => ["fnord"]})
+      @namespace.gauges[:multigauge].should be_a(FnordMetric::NumericGauge) 
+    end
 
-    before(:each) do
-      @namespace = FnordMetric::Namespace.new(:myns_213, :redis_prefix => "fnordmetric")
+    it "should register a multi gauge and pass options" do
+      @namespace.numeric_gauge(:multigauge2, {:fnord => 42, :ticks => [1.hour], :series => ["fnord"]})
+      @namespace.gauges[:multigauge2].instance_variable_get(:@opts).should include({:fnord => 42})
+      @namespace.gauges[:multigauge2].instance_variable_get(:@opts).should include({:key => :multigauge2})
     end
 
+  end
+
+  describe "registering event handlers" do
+
     it "should register an event handler" do
       @namespace.handlers.length.should == 0
       @namespace.event(:foobar){}

data/spec/spec_helper.rb

@@ -13,6 +13,10 @@ require "fnordmetric"
 require "eventmachine"
 require 'em-hiredis'
 
+$: << ::File.expand_path('../../spec', __FILE__)
+module FnordMetric::GaugeLike; end
+require "gauge_like_shared"
+
 def EM.defer; yield; end
 def EM.next_tick; yield; end
 

data/spec/{inbound_stream_spec.rb → tcp_acceptor_spec.rb}

@@ -1,16 +1,16 @@
 require ::File.expand_path('../spec_helper.rb', __FILE__)
 
-describe FnordMetric::InboundStream do
+describe FnordMetric::TCPAcceptor do
 
   before(:all) do
     @redis = Redis.new
     @redis_wrap = RedisWrap.new(@redis)
-    FnordMetric::InboundStream.options(
+    FnordMetric::TCPAcceptor.options(
       :redis_url => "redis://localhost:6379",
       :redis_prefix => "fnordmetric-test",
       :event_queue_ttl => 120
     )
-    @inbound_stream = FnordMetric::InboundStream.new(nil)
+    @inbound_stream = FnordMetric::TCPAcceptor.new(nil)
   end
 
   describe "pushing new events" do

data/spec/timeseries_gauge_spec.rb

@@ -0,0 +1,54 @@
+require ::File.expand_path('../spec_helper.rb', __FILE__)
+
+describe FnordMetric::TimeseriesGauge do
+
+  before(:all) do    
+    @now = Time.utc(1992,01,13,5,23,23).to_i    
+    @redis = Redis.new
+    @redis_wrap = RedisWrap.new(@redis, false)
+    @gauge_klass = FnordMetric::TimeseriesGauge
+  end
+
+  before(:each) do
+    @redis.keys("fnordmetric-myns*").each { |k| @redis.del(k) }  
+  end
+
+  it_should_behave_like FnordMetric::GaugeLike
+
+  describe "option validation" do
+
+    it "should raise when initialized with non-unique series tokens" do
+      lambda{
+        FnordMetric::TimeseriesGauge.new(
+          :series => [:fnord, :fnord], 
+          :ticks => [1.hour],
+          :key => 123, 
+          :key_prefix => 123
+        )
+      }.should raise_exception
+    end
+
+    it "should raise when initialized without series tokens key" do
+      lambda{
+        FnordMetric::TimeseriesGauge.new(
+          :key => 123, 
+          :ticks => [1.hour],
+          :key_prefix => 123
+        )
+      }.should raise_exception
+    end
+
+    it "should raise when initialized with emtpy series tokens" do
+      lambda{
+        FnordMetric::TimeseriesGauge.new(
+          :series => [],
+          :key => 123,
+          :ticks => [1.hour],
+          :key_prefix => 123
+        )
+      }.should raise_exception
+    end
+
+  end
+
+end

data/spec/{inbound_datagram_spec.rb → udp_acceptor_spec.rb}

@@ -1,13 +1,13 @@
 require ::File.expand_path('../spec_helper.rb', __FILE__)
 
-describe FnordMetric::InboundDatagram do
+describe FnordMetric::UDPAcceptor do
 
-  let(:inbound_datagram) { FnordMetric::InboundDatagram.new(nil) }
+  let(:inbound_datagram) { FnordMetric::UDPAcceptor.new(nil) }
 
   before(:all) do
     @redis = Redis.new
     @redis_wrap = RedisWrap.new(@redis)
-    FnordMetric::InboundDatagram.opts = {
+    FnordMetric::UDPAcceptor.opts = {
       :redis_url => "redis://localhost:6379",
       :redis_prefix => "fnordmetric-test",
       :event_queue_ttl => 120