visage-app 2.0.5 → 2.1.0

data/.gitignore

@@ -10,3 +10,4 @@ _site
 features/data/config/*/*.yaml
 vendor
 .bundle
+.rbenv-version

data/AUTHORS

@@ -7,6 +7,7 @@ Contributors include
 Jeffrey Lim <jfs.world@gmail.com>
 Andrew Harvey <andrew@mootpointer.com>
 Xavier Mehrenberger <xavier.mehrenberger@gmail.com>
+Jesse Reynolds <jesse@bulletproof.net>
 
 Inspiration given by
 --------------------

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+2.1.0 - 2012/04/20
+ - Add support for drawing 95th percentile lines on graphs
+   Thanks to Jesse Reynolds for the feature addition!
+
 2.0.5 - 2012/03/30
  - Fix a bug where metrics occasionally won't appear in the builder interface.
    Thanks to Jesse Reynolds for the patch!

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    visage-app (2.0.5)
+    visage-app (2.1.0)
       errand (= 0.7.3)
       haml (= 3.1.4)
       sinatra (= 1.3.2)

data/README.md

@@ -227,11 +227,11 @@ rake install
 Releasing
 ---------
 
-# Bump the version in lib/visage-app/version.rb
-# Add an entry to CHANGELOG.md
-# `git commit` everything.
-# Build the gem with `rake build`
-# Push the gem to RubyGems.org with `rake push`
+1. Bump the version in lib/visage-app/version.rb
+2. Add an entry to CHANGELOG.md
+3. `git commit` everything.
+4. Build the gem with `rake build`
+5. Push the gem to RubyGems.org with `rake push`
 
 Licencing
 ---------

data/Rakefile

@@ -56,7 +56,7 @@ namespace :verify do
     @root = Pathname.new(File.dirname(__FILE__)).expand_path
     javascripts_path = @root.join('lib/visage-app/public/javascripts')
 
-    javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /mootools|src\.js/ }
+    javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /highcharts|mootools|src\.js/ }
     javascripts.each do |filename|
       puts "Checking #{filename}".green
       count = `grep -c 'console.log' #{filename}`.strip.to_i

data/features/json.feature

@@ -10,12 +10,6 @@ Feature: Export data
     Then I should receive valid JSON
     And the JSON should have a list of hosts
 
-  Scenario: Retreive a list of hosts
-    When I visit the first available host
-    Then the request should succeed
-    Then I should receive valid JSON
-    And the JSON should have a list of plugins
-
   Scenario: Get a list of available metrics on a host
     When I visit the first available host
     Then the request should succeed
@@ -86,3 +80,13 @@ Feature: Export data
       | start     | 1321769692 |
       | finish    | 1321773292 |
 
+  Scenario: Retrieve data for a defined period including 95th percentile calculations
+    Given a list of hosts exist
+    When I visit "disk*/disk_ops" on the first available host with the following query parameters:
+      | parameter   | value      |
+      | start       | 1 hour ago |
+      | finish      | now        |
+      | percentiles | 95         |
+    Then the request should succeed
+    Then I should receive valid JSON
+    And I should see a 95th percentile value for each plugin instance

data/features/profiles.feature

@@ -5,23 +5,18 @@ Feature: Visit site
 
   Scenario: List profiles
     When I go to /profiles
-    Then I should see a list of profiles
-    When I follow "created"
-    Then I should see a list of profiles
-    When I follow "name"
     Then I should see a list of profiles sorted alphabetically
 
   Scenario: Show profile
     When I go to /profiles
     And I visit the first profile
-    Then I should see a list of graphs
-    And I should see a profile heading
+    Then I should see a profile heading
+    And I should see "Back to profiles"
 
   Scenario: Navigate profiles
     When I go to /profiles
     And I visit the first profile
-    Then I should see a list of graphs
-    And I should see a profile heading
+    Then I should see a profile heading
     When I follow "back to profiles"
-    Then I should see a list of profiles
+    Then I should see a list of profiles sorted alphabetically
 

data/features/step_definitions/cli_steps.rb

@@ -20,7 +20,7 @@ Then /^a visage web server should be running$/ do
 end
 
 Then /^I should see "([^"]*)" on the terminal$/ do |string|
-  output = @pipe.read(250)
+  output = @pipe.read(350)
   output.should =~ /#{string}/
 end
 
@@ -42,5 +42,5 @@ Then /^I should see a file at "([^"]*)"$/ do |filename|
 end
 
 Then /^show me the output$/ do
-  puts @pipe.read(250)
+  puts @pipe.read(350)
 end

data/features/step_definitions/json_steps.rb

@@ -103,7 +103,7 @@ end
 When /^I visit "([^"]*)" on the first available host$/ do |glob|
   host = @response["hosts"].first
   url  = "/data/#{host}/#{glob}"
-  When "I go to #{url}"
+  step "I go to #{url}"
 end
 
 
@@ -115,7 +115,7 @@ When /^I visit the first available host$/ do
 
   host = @response["hosts"].first
   url  = "/data/#{host}"
-  When "I go to #{url}"
+  step "I go to #{url}"
 end
 
 When /^I visit the first two available hosts$/ do
@@ -126,7 +126,7 @@ When /^I visit the first two available hosts$/ do
 
   host = @response["hosts"][0..1].join(',')
   url  = "/data/#{host}"
-  When "I go to #{url}"
+  step "I go to #{url}"
 end
 
 Then /^the JSON should have a list of plugins$/ do
@@ -150,6 +150,14 @@ When /^I visit "([^"]*)" on the first available host with the following query pa
   url  = "/data/#{host}/#{glob}"
 
   params = Hash[table.hashes.map { |hash| [hash["parameter"], hash["value"]] }]
+  params.each do |key, value|
+    if value == "1 hour ago"
+      params[key] = (Time.now() - 3600).to_i.to_s
+    end
+    if value == "now"
+      params[key] = Time.now().to_i.to_s
+    end
+  end
   query  = params.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}.join("&")
   url   += "?#{query}"
 
@@ -175,3 +183,18 @@ Then /^I should see the following parameters for each plugin instance:$/ do |tab
 
 end
 
+Then /^I should see a 95th percentile value for each plugin instance$/ do
+  @response.should_not be_nil
+
+  @response.each_pair do |host, plugin|
+    plugin.each_pair do |instance, metric|
+      metric.each_pair do |k, series|
+        series.each_pair do |k, data|
+          data['percentile_95'].should >= 0
+        end
+      end
+    end
+  end
+
+end
+

data/features/step_definitions/site_steps.rb

@@ -10,7 +10,6 @@ Then /^I should see a list of graphs$/ do
     follow_redirect!
   rescue Rack::Test::Error
   end
-
   doc = Nokogiri::HTML(response_body)
   doc.search('div#profile div.graph').size.should > 0
 end

data/lib/visage-app.rb

@@ -21,6 +21,8 @@ module Visage
     set :public_folder, @root.join('lib/visage-app/public')
     set :views,         @root.join('lib/visage-app/views')
 
+    enable :logging
+
     helpers Sinatra::LinkToHelper
     helpers Sinatra::PageTitleHelper
     helpers Sinatra::RequireJSHelper
@@ -34,6 +36,7 @@ module Visage
 
       # Load up the profiles.yaml. Creates it if it doesn't already exist.
       Visage::Profile.load
+
     end
   end
 
@@ -107,18 +110,23 @@ module Visage
 
     # /data/:host/:plugin/:optional_plugin_instance
     get %r{/data/([^/]+)/([^/]+)((/[^/]+)*)} do
-      host      = params[:captures][0].gsub("\0", "")
-      plugin    = params[:captures][1].gsub("\0", "")
-      instances = params[:captures][2].gsub("\0", "")
-      start     = params[:start]
-      finish    = params[:finish]
+      host        = params[:captures][0].gsub("\0", "")
+      plugin      = params[:captures][1].gsub("\0", "")
+      instances   = params[:captures][2].gsub("\0", "")
+      start       = params[:start]
+      finish      = params[:finish]
+      percentiles = params[:percentiles] ||= "false"
+      resolution  = params[:resolution]
 
       collectd = Visage::Collectd::JSON.new(:rrddir => Visage::Config.rrddir)
-      json = collectd.json(:host      => host,
-                           :plugin    => plugin,
-                           :instances => instances,
-                           :start     => start,
-                           :finish    => finish)
+
+      json = collectd.json(:host        => host,
+                           :plugin      => plugin,
+                           :instances   => instances,
+                           :start       => start,
+                           :finish      => finish,
+                           :percentiles => percentiles,
+                           :resolution  => resolution)
 
       # If the request is cross-domain, we need to serve JSON-P.
       maybe_wrap_with_callback(json)

data/lib/visage-app/collectd/json.rb

@@ -6,6 +6,48 @@ require 'lib/visage-app/patches'
 require 'errand'
 require 'yajl'
 
+class Array
+  def in_groups(number, fill_with = nil)
+    raise "Error - in_groups of zero doesn't make sense" unless number > 0
+    # size / number gives minor group size;
+    # size % number gives how many objects need extra accomodation;
+    # each group hold either division or division + 1 items.
+    division = size / number
+    modulo = size % number
+
+    # create a new array avoiding dup
+    groups = []
+    start = 0
+
+    number.times do |index|
+      length = division + (modulo > 0 && modulo > index ? 1 : 0)
+      padding = fill_with != false &&
+        modulo > 0 && length == division ? 1 : 0
+      groups << slice(start, length).concat([fill_with] * padding)
+      start += length
+    end
+
+    if block_given?
+      groups.each{|g| yield(g) }
+    else
+      groups
+    end
+  end
+
+  def sum
+    inject( nil ) { |sum,x| sum ? sum+x : x }
+  end
+
+  def mean
+    if size > 0
+      sum / size
+    else
+      nil
+    end
+  end
+
+end
+
 # Exposes RRDs as JSON.
 #
 # A loose shim onto RRDtool, with some extra logic to normalise the data.
@@ -32,14 +74,16 @@ module Visage
 
       # Entry point.
       def json(opts={})
-        host      = opts[:host]
-        plugin    = opts[:plugin]
-        instances = opts[:instances][/\w.*/]
-        instances = instances.blank? ? '*' : '{' + instances.split('/').join(',') + '}'
-        rrdglob   = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
-        finish    = parse_time(opts[:finish])
-        start     = parse_time(opts[:start],  :default => (finish - 3600 || (Time.now - 3600).to_i))
-        data      = []
+        host        = opts[:host]
+        plugin      = opts[:plugin]
+        instances   = opts[:instances][/\w.*/]
+        instances   = instances.blank? ? '*' : '{' + instances.split('/').join(',') + '}'
+        percentiles = opts[:percentiles] !~ /^$|^false$/ ? true : false
+        resolution  = opts[:resolution] || ""
+        rrdglob     = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
+        finish      = parse_time(opts[:finish])
+        start       = parse_time(opts[:start],  :default => (finish - 3600 || (Time.now - 3600).to_i))
+        data        = []
 
         Dir.glob(rrdglob).map do |rrdname|
           parts         = rrdname.gsub(/#{@rrddir}\//, '').split('/')
@@ -48,16 +92,45 @@ module Visage
           instance_name = File.basename(parts[2], '.rrd')
           rrd           = Errand.new(:filename => rrdname)
 
-          data << {  :plugin => plugin_name, :instance => instance_name,
-                     :host   => host_name,
-                     :start  => start,
-                     :finish => finish,
-                     :rrd    => rrd }
+          data << {  :plugin      => plugin_name, :instance => instance_name,
+                     :host        => host_name,
+                     :start       => start,
+                     :finish      => finish,
+                     :rrd         => rrd,
+                     :percentiles => percentiles,
+                     :resolution  => resolution}
+
         end
 
         encode(data)
       end
 
+      def percentile_of_array(samples, percentage)
+        if samples
+          samples.sort[ (samples.length.to_f * ( percentage.to_f / 100.to_f ) ).to_i - 1 ]
+        else
+          raise "I can't work out percentiles on a nil sample set"
+        end
+      end
+
+      def downsample_array(samples, old_resolution, new_resolution)
+        return samples unless samples.length > 0
+        timer_start = Time.now
+        new_samples = []
+        if (new_resolution > 0) and (old_resolution > 0) and (new_resolution % old_resolution == 0)
+          groups_of = samples.length / (new_resolution / old_resolution)
+          return samples unless groups_of > 0
+          samples.in_groups(groups_of, false) {|group|
+            new_samples << group.compact.mean
+          }
+        else
+          raise "downsample_array: cowardly refusing to downsample as old_resolution (#{old_resolution.to_s}) doesn't go into new_resolution (#{new_resolution.to_s}) evenly, or new_resolution or old_resolution are zero."
+        end
+        timer = Time.now - timer_start
+
+        new_samples
+      end
+
       private
       # Attempt to structure the JSON reasonably sanely, so the consumer (i.e. a
       # browser) doesn't have to do a lot of computationally expensive work.
@@ -65,17 +138,25 @@ module Visage
 
         structure = {}
         datas.each do |data|
-          fetch    = data[:rrd].fetch(:function => "AVERAGE",
-                                      :start    => data[:start],
-                                      :finish   => data[:finish])
+
+          start      = data[:start].to_i
+          finish     = data[:finish].to_i
+          resolution = data[:resolution].to_i || 0
+
+          fetch    = data[:rrd].fetch(:function   => "AVERAGE",
+                                      :start      => start.to_s,
+                                      :finish     => finish.to_s)
+
           rrd_data = fetch[:data]
+          percentiles = data[:percentiles]
 
           # A single rrd can have multiple data sets (multiple metrics within
           # the same file). Separate the metrics.
           rrd_data.each_pair do |source, metric|
 
             # Filter out NaNs and weirdly massive values so yajl doesn't choke
-            metric.map! do |datapoint|
+            # FIXME: does this actually do anything?
+            metric = metric.map do |datapoint|
               case
               when datapoint && datapoint.nan?
                 @tripped = true
@@ -87,21 +168,40 @@ module Visage
               end
             end
 
-            # Last value is always wack. Set to 0, so the timescale isn't off by 1.
-            metric[-1] = 0.0
+            # Last value is always wack. Remove it.
+            metric   = metric[0...metric.length-1]
             host     = data[:host]
             plugin   = data[:plugin]
             instance = data[:instance]
-            start    = data[:start].to_i
-            finish   = data[:finish].to_i
+
+            # only calculate percentiles if requested
+            if percentiles
+              timeperiod = finish.to_f - start.to_f
+              interval = (timeperiod / metric.length.to_f).round
+              resolution = 300
+              if (interval < resolution) and (resolution > 0)
+                metric_for_percentiles = downsample_array(metric, interval, resolution)
+              else
+                metric_for_percentiles = metric
+              end
+              metric_for_percentiles.compact!
+              percentiles = false unless metric_for_percentiles.length > 0
+            end
+
+            if metric.length > 2000
+              metric = downsample_array(metric, 1, metric.length / 1000)
+            end
 
             structure[host] ||= {}
             structure[host][plugin] ||= {}
             structure[host][plugin][instance] ||= {}
             structure[host][plugin][instance][source] ||= {}
-            structure[host][plugin][instance][source][:start]  ||= start
-            structure[host][plugin][instance][source][:finish] ||= finish
-            structure[host][plugin][instance][source][:data]   ||= metric
+            structure[host][plugin][instance][source][:start]         ||= start
+            structure[host][plugin][instance][source][:finish]        ||= finish
+            structure[host][plugin][instance][source][:data]          ||= metric
+            structure[host][plugin][instance][source][:percentile_95] ||= percentile_of_array(metric_for_percentiles, 95).round if percentiles
+            structure[host][plugin][instance][source][:percentile_50] ||= percentile_of_array(metric_for_percentiles, 50).round if percentiles
+            structure[host][plugin][instance][source][:percentile_5]  ||= percentile_of_array(metric_for_percentiles,  5).round if percentiles
 
           end
         end