visage-app 0.1.0

data/.gitignore

@@ -0,0 +1,9 @@
+config/profiles.yaml
+.DS_Store
+log/*
+tmp/*
+pkg/*
+*~
+.#*
+webrat*
+visage-app.gemspec

data/README.md

@@ -0,0 +1,138 @@
+Visage
+======
+
+Visage is a web interface for viewing [collectd](http://collectd.org) statistics.
+
+It also provides a [JSON](http://json.org) interface onto `collectd`'s RRD data,
+giving you an easy way to mash up the data.
+
+Features
+--------
+
+ * renders graphs in the browser, and retrieves data asynchronously
+ * interactive graph keys, to highlight lines and toggle line visibility
+ * drop-down or mouse selection of timeframes (also rendered asynchronously)
+ * JSON interface onto `collectd` RRDs
+
+Check out [a demo](http://visage.unstated.net/nadia/cpu+load).
+
+Installing
+----------
+
+On Ubuntu, to install dependencies run:
+
+    $ sudo apt-get install -y librrd-ruby ruby rubygems
+
+On CentOS, to install dependencies run:
+
+    $ sudo yum install -y rrdtool ruby rubygems
+
+Then install the app with:
+
+    $ gem install visage
+
+
+Running
+-------
+
+You can check out the application quickly with:
+
+    $ visage start
+
+
+Configuring
+-----------
+
+Config lives in several files under `config/`.
+
+ * `profiles.yaml` - groups of graphs Visage is to display
+ * `plugin-colors.yaml` - colors for specific plugins/plugin instances
+ * `fallback-colors.yaml` - ordered list of fallback colors
+ * `init.rb` - bootstrapping code, specifies collectd's RRD directory
+
+`profiles.yaml` isn't setup by default, but you can copy `profiles.yaml.sample`
+across and edit to taste. The plugins are in the format of
+`plugin/plugin-instance`, with `plugins-instance` being optional.
+
+If you don't specify a `plugin-instance` Visage will attempt to graph all plugin
+instances under the specified `plugin`, e.g. `cpu-0` will display `cpu-idle`,
+`cpu-interrupt`, `cpu-nice`, etc, whereas `cpu-0/cpu-wait` will only show
+`cpu-wait`. You can also choose a specific group of plugin instances to graph,
+with something like `cpu-0/cpu-system/cpu-user/cpu-wait`.
+
+It should be pretty easy to deduce the config format from the existing file
+(it's simple nested key-value data).
+
+Make sure collectd's RRD directory is readable by whatever user the web server
+is running as. You can specify where collectd's rrd directory is in `init.rb`,
+with the `c['rrddir']` key.
+
+Deploying
+---------
+
+With Passenger, create an Apache vhost with the `DocumentRoot` set to the
+`public/` directory of where you have deployed the checked out code, e.g.
+
+    <VirtualHost *>
+      ServerName visage.example.org
+      ServerAdmin contact@visage.example.org
+
+      DocumentRoot /srv/www/visage.example.org/root/public/
+
+      <Directory "/srv/www/visage.example.org/root/public/">
+         Options FollowSymLinks Indexes
+         AllowOverride None
+         Order allow,deny
+         Allow from all
+       </Directory>
+
+       ErrorLog /srv/www/visage.example.org/log/apache_errors_log
+       CustomLog /srv/www/visage.example.org/log/apache_app_log combined
+
+    </VirtualHost>
+
+This assumes you have a checkout of the code at `/srv/www/visage.example.org/root`.
+
+If you don't want to use Apache + Passenger, you can install the `thin` or
+`mongrel` gems and run up a web server yourself.
+
+Ubuntu users looking for Passenger packages should add John Ferlito's
+[mod-passenger PPA](https://launchpad.net/~johnf-inodes/+archive/mod-passenger)
+to their apt sources.
+
+Developing + testing
+--------------------
+
+Check out the code with:
+
+    $ git clone git://github.com/auxesis/visage.git
+
+Install the development dependencies with
+
+    $ gem install shotgun rack-test rspec cucumber webrat
+
+And run the app with:
+
+    $ shotgun visage.rb
+
+Create and install a new gem from the current source tree:
+
+    $ rake install
+
+Run all cucumber features:
+
+    $ rake cucumber
+
+Specific features:
+
+    $ bin/cucumber --require features/ features/something.feature
+
+TODO
+----
+
+ * refactor tests to work on hosts other than my laptop
+ * interface to build custom graph profiles
+ * combine graphs from different hosts
+ * detailed point-in-time data on hover
+ * comment on time periods
+ * view list of comments

data/Rakefile

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+
+begin
+  require 'cucumber/rake/task'
+
+  Cucumber::Rake::Task.new do |t|
+    t.binary = "bin/cucumber"
+    t.cucumber_opts = "--require features/ features/"
+  end
+rescue LoadError
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "visage-app"
+    gemspec.summary = "a web (interface | service) for viewing collectd statistics"
+    gemspec.description = "Visage is a web interface for viewing collectd statistics. It also provides a JSON interface onto collectd's RRD data, giving you an easy way to mash up the data."
+    gemspec.email = "lindsay@holmwood.id.au"
+    gemspec.homepage = "http://github.com/auxesis/visage"
+    gemspec.authors = ["Lindsay Holmwood"]
+
+    gemspec.add_dependency "sinatra", "1.0"
+    gemspec.add_dependency "tilt", "1.0.1"
+    gemspec.add_dependency "haml", "3.0.13"
+    gemspec.add_dependency "errand", "0.7.2"
+    gemspec.add_dependency "yajl-ruby", "0.7.6"
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/visage

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+
+@root = Pathname.new(File.dirname(__FILE__)).parent.expand_path
+action = ARGV[0]
+
+case action
+when "start"
+  require 'rubygems'
+  require 'rack'
+  config = @root.join('config.ru').to_s
+  Rack::Server.start(:config => config, :Port => 9292)
+else
+  puts "Usage: visage start"
+end
+

data/config.ru

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$0 = "visage"
+
+require 'lib/visage-app'
+run Sinatra::Application
+

data/features/json.feature

@@ -0,0 +1,66 @@
+Feature: Export data
+  To perform analysis
+  A user
+  Must be able to extract data
+  From the application
+
+  Scenario: Retrieve single plugin instance
+    When I go to /data/theodor/memory/memory-free
+    Then the request should succeed
+    Then I should receive valid JSON
+    And the JSON should have a plugin instance named "memory-free"
+
+  Scenario: Retrieve multiple plugin instances
+    When I go to /data/theodor/memory
+    Then the request should succeed
+    Then I should receive valid JSON
+    And the JSON should have a plugin named "memory"
+    And the JSON should have multiple plugin instances under the "memory" plugin
+
+  Scenario: Make cross-domain requests
+    When I go to /data/theodor/cpu-0?callback=foobar
+    Then I should receive JSON wrapped in a callback named "foobar"
+
+  Scenario: Retrieve multiple plugin instances without color definition
+    When I go to /data/theodor/memory
+    Then the request should succeed
+    Then I should receive valid JSON
+    And each plugin instance should have a different color
+  
+  Scenario Outline: Return only one colour per metric
+    When I go to /data/theodor/<path> 
+    Then the request should succeed
+    Then I should receive valid JSON
+    And each plugin instance should have a different color
+
+  Examples: 
+    | path           |
+    | cpu-0/cpu-user |
+    | df/df-root     |
+
+  Scenario: Retrieve single plugin instance with a color definition
+    When I go to /data/theodor/tcpconns-80-local/tcp_connections-LISTEN
+    Then the request should succeed
+    Then I should receive valid JSON
+    And the plugin instance should have a color
+
+  Scenario: Retrieve multiple plugins through a glob
+    Given I have the "tcpconns" plugin collecting data on multiple ports
+    When I go to /data/theodor/tcpconns-*-local/tcp_connections-LISTEN
+    Then the request should succeed
+    Then I should receive valid JSON
+    And I should see multiple plugins
+
+  Scenario Outline: Retrieve multple hosts through a glob
+    Given I have the "memory" plugin collecting data on multiple ports
+    When I go to /data/<host>/libvirt/virt_cpu_total
+    Then the request should succeed
+    Then I should receive valid JSON
+    And I should see multiple hosts
+
+  Examples: 
+    | host                    |
+    | *                       |
+    | %7Brgh,flapjack-test%7D |
+
+

data/features/site.feature

@@ -0,0 +1,9 @@
+Feature: Visit site
+  To find out how systems are performing
+  A user
+  Must be able to visualise the data
+
+  Scenario: Show available hosts
+    When I go to /
+    Then I should see a list of available hosts
+    When I follow the first host

data/features/step_definitions/form_steps.rb

@@ -0,0 +1,6 @@
+
+Then /^the "(.+)" select should have "(.+)" selected$/ do |input, value|
+  response_body.to_s.should have_xpath("//select[@name='#{input}']//option[contains(.,'#{value}')][@selected]")
+end
+
+

data/features/step_definitions/json_steps.rb

@@ -0,0 +1,79 @@
+Then /^I should receive valid JSON$/ do
+  yajl = Yajl::Parser.new
+  lambda {
+    @response = yajl.parse(response_body)
+  }.should_not raise_error
+ 
+  host   = @response.keys.first
+  plugin = @response[host].keys.first
+  metric = @response[host][plugin].keys.first
+  
+  host.should_not be_nil
+  plugin.should_not be_nil
+  metric.should_not be_nil
+
+  data = @response[host][plugin][metric]["data"]
+  #.values.first.size.should > 0
+end
+
+Then /^I should receive JSON wrapped in a callback named "([^\"]*)"$/ do |callback|
+  response_body.should =~ /^#{callback}\(.+\)$/
+end
+
+Then /^the JSON should have a plugin instance named "([^\"]*)"$/ do |plugin_instance|
+  yajl = Yajl::Parser.new
+  data = yajl.parse(response_body)
+
+  data.values.map { |k,v| k.values }.map {|k,v| k.keys }.flatten.include?(plugin_instance).should be_true
+end
+
+Then /^the JSON should have a plugin named "([^\"]*)"$/ do |plugin|
+  yajl = Yajl::Parser.new
+  data = yajl.parse(response_body)
+
+  data.values.map { |k,v| k.keys }.flatten.include?(plugin).should be_true
+end
+
+Then /^the JSON should have multiple plugin instances under the "([^\"]*)" plugin$/ do |arg1|
+  yajl = Yajl::Parser.new
+  data = yajl.parse(response_body)
+
+  data.values.map { |k,v| k[arg1] }.map { |k,v| k.keys }.flatten.size.should > 1
+end
+
+Then /^each plugin instance should have a different color$/ do
+  yajl = Yajl::Parser.new
+  data = yajl.parse(response_body)
+
+  @colours = []
+  data.values.map { |k,v| k.values }.map {|k,v| k.values }.map {|k,v| k.values }.map do |a|
+    a.each do |b|
+      string = b["color"]
+      string.should =~ /^#[0-9a-fA-F]+$/
+      @colours << string
+    end
+  end
+
+  @colours.uniq.size.should == @colours.size
+
+end
+
+Then /^the plugin instance should have a color$/ do
+  Then "each plugin instance should have a different color"
+end
+
+
+Given /^I have the "([^\"]*)" plugin collecting data on multiple ports$/ do |plugin|
+  Dir.glob("/var/lib/collectd/rrd/theodor/tcpconns*").size.should > 1
+end
+
+Then /^I should see multiple plugins$/ do
+  @response.should_not be_nil
+  host = @response.keys.first
+  @response[host].keys.size.should > 1
+end
+
+Then /^I should see multiple hosts$/ do
+  @response.should_not be_nil
+  @response.keys.size.should > 1
+end

data/features/step_definitions/result_steps.rb

@@ -0,0 +1,19 @@
+Then /^I should see "(.*)"$/ do |text|
+  response_body.to_s.should =~ /#{text}/m
+end
+
+Then /^I should not see "(.*)"$/ do |text|
+  response_body.to_s.should_not =~ /#{text}/m
+end
+
+Then /^I should see an? (\w+) message$/ do |message_type|
+  response.should have_xpath("//*[@class='#{message_type}']")
+end
+
+Then /^the (.*) ?request should succeed/ do |_|
+  response_code.should < 400
+end
+
+Then /^the (.*) ?request should fail/ do |_|
+  response_code.should >= 400
+end

data/features/step_definitions/site_steps.rb

@@ -0,0 +1,4 @@
+Then /^I should see a list of available hosts$/ do
+  doc = Nokogiri::HTML(response_body)
+  doc.search('div#hosts li').size.should > 0
+end

data/features/step_definitions/visage_steps.rb

@@ -0,0 +1,20 @@
+Given /^the "([^"]*)" gem is installed$/ do |name|
+  `gem list #{name} |grep #{name}`.size.should > 0
+end
+
+When /^I start the visage server helper with "([^"]*)"$/ do |cmd|
+  @root = Pathname.new(File.dirname(__FILE__)).parent.parent.expand_path
+  command = "#{@root.join('bin')}/#{cmd}"
+
+  pipe = IO.popen(command)
+  sleep 2 # so the visage server has a chance to boot
+
+  # clean up the visage server when the tests finish
+  at_exit do
+    Process.kill("KILL", pipe.pid)
+  end
+end
+
+Then /^a visage web server should be running$/ do
+  `ps -eo cmd |grep ^visage`.size.should > 0
+end

data/features/step_definitions/webrat_steps.rb

@@ -0,0 +1,42 @@
+# Commonly used webrat steps
+# http://github.com/brynary/webrat
+
+When /^I go to (.*)$/ do |path|
+  visit path
+end
+
+When /^I press "(.*)"$/ do |button|
+  click_button(button)
+end
+
+When /^I follow "(.*)"$/ do |link|
+  click_link(link)
+end
+
+When /^I fill in "(.*)" with "(.*)"$/ do |field, value|
+  fill_in(field, :with => value) 
+end
+
+When /^I select "(.*)" from "(.*)"$/ do |value, field|
+  select(value, :from => field) 
+end
+
+When /^I check "(.*)"$/ do |field|
+  check(field) 
+end
+
+When /^I uncheck "(.*)"$/ do |field|
+  uncheck(field) 
+end
+
+When /^I choose "(.*)"$/ do |field|
+  choose(field)
+end
+
+When /^I attach the file at "(.*)" to "(.*)" $/ do |path, field|
+  attach_file(field, path)
+end
+
+Then /^show me the page$/ do
+  save_and_open_page
+end

data/features/support/env.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+
+@root = Pathname.new(File.dirname(__FILE__)).parent.parent.expand_path
+app_file = @root.join('lib/visage')
+
+require 'rubygems'
+require 'spec/expectations'
+require 'rack/test'
+require 'webrat'
+
+require app_file
+# Force the application name because polyglot breaks the auto-detection logic.
+Sinatra::Application.app_file = app_file
+
+Webrat.configure do |config|
+  config.mode = :rack
+end
+
+class SinatraWorld
+  include Rack::Test::Methods
+  include Webrat::Methods
+  include Webrat::Matchers
+
+  Webrat::Methods.delegate_to_session :response_code, :response_body
+
+  def app
+    Sinatra::Application
+  end
+end
+
+World do
+  SinatraWorld.new
+end
+

data/features/visage.feature

@@ -0,0 +1,11 @@
+Feature: command line utility
+  As a systems administrator
+  Or a hard core developer
+  I want to get Visage up and running
+  With the least hassle possible
+
+  Scenario: Command line tool
+    Given the "visage" gem is installed
+    When I start the visage server helper with "visage start"
+    Then a visage web server should be running
+

data/lib/visage/collectd/json.rb

@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby
+
+$: << File.expand_path(File.dirname(__FILE__))
+$: << File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+require 'errand'
+require 'yajl'
+require 'patches'
+
+# Exposes RRDs as JSON.
+#
+# A loose shim onto RRDtool, with some extra logic to normalise the data.
+# Also provides a recommended color for rendering the data in a line graph.
+#
+class CollectdJSON
+
+  def initialize(opts={})
+    @rrddir = opts[:rrddir] || CollectdJSON.rrddir
+    @fallback_colors = opts[:fallback_colors] || {}
+    @used_fallbacks = []
+  end
+
+  # Entry point.
+  def json(opts={})
+    host             = opts[:host]
+    plugin           = opts[:plugin]
+    plugin_instances = opts[:plugin_instances][/\w.*/]
+    instances        = plugin_instances.blank? ? '*' : '{' + plugin_instances.split('/').join(',') + '}'
+    @colors          = opts[:plugin_colors]
+    @plugin_names = []
+
+    rrdglob = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
+    plugin_offset = @rrddir.size + 1 + host.size + 1
+
+    data = []
+
+    Dir.glob(rrdglob).map do |rrdname|
+      parts         = rrdname.gsub(/#{@rrddir}\//, '').split('/')
+      host          = parts[0]
+      plugin_name   = parts[1]
+      instance_name = parts[2].split('.').first
+      rrd = Errand.new(:filename => rrdname)
+
+      data << { :plugin  => plugin_name, :instance => instance_name,
+                 :host   => host,
+                 :start  => opts[:start] || (Time.now - 3600).to_i,
+                 :finish => opts[:finish] || Time.now.to_i,
+                 :rrd    => rrd }
+    end
+
+    encode(data)
+  end
+
+  private
+  # Attempt to structure the JSON reasonably sanely, so the consumer (i.e. a
+  # browser) doesn't have to do a lot of computationally expensive work.
+  def encode(datas)
+
+    structure = {}
+    datas.each do |data|
+      fetch = data[:rrd].fetch(:function => "AVERAGE",
+                                  :start => data[:start],
+                                  :finish => data[:finish])
+      rrd_data = fetch[:data]
+
+      # A single rrd can have multiple data sets (multiple metrics within
+      # the same file). Separate the metrics.
+      rrd_data.each_pair do |source, metric|
+
+        # filter out NaNs, so yajl doesn't choke
+        metric.map! do |datapoint|
+          (!datapoint || datapoint.nan?) ? 0.0 : datapoint
+        end
+
+        color = color_for(:host => data[:host],
+                          :plugin => data[:plugin],
+                          :instance => data[:instance],
+                          :metric => source)
+
+        structure[data[:host]] ||= {}
+        structure[data[:host]][data[:plugin]] ||= {}
+        structure[data[:host]][data[:plugin]][data[:instance]] ||= {}
+        structure[data[:host]][data[:plugin]][data[:instance]][source] ||= {}
+        structure[data[:host]][data[:plugin]][data[:instance]][source][:start]  ||= data[:start]
+        structure[data[:host]][data[:plugin]][data[:instance]][source][:finish] ||= data[:finish]
+        structure[data[:host]][data[:plugin]][data[:instance]][source][:data]   ||= metric
+        structure[data[:host]][data[:plugin]][data[:instance]][source][:color]  ||= color
+      end
+    end
+
+    encoder = Yajl::Encoder.new
+    encoder.encode(structure)
+  end
+
+  # We append the recommended line color onto data set, so the javascript
+  # doesn't try and have to work it out. This lets us use all sorts of funky
+  # fallback logic when determining what colours should be used.
+  def color_for(opts={})
+
+    plugin   = opts[:plugin]
+    instance = opts[:instance]
+    metric   = opts[:metric]
+
+    return fallback_color unless plugin
+    return color_for(opts.merge(:plugin => plugin[/(.+)-.+$/, 1])) unless @colors[plugin]
+    return color_for(opts.merge(:instance => instance[/(.+)-.+$/, 1])) unless @colors[plugin][instance]
+    return @colors[plugin][instance][metric] if @colors[plugin][instance]
+    return fallback_color
+  end
+
+  def fallback_color
+    fallbacks = @fallback_colors.to_a.sort_by {|pair| pair[1]['fallback_order'] }
+    fallback = fallbacks.find { |color| !@used_fallbacks.include?(color) }
+    unless fallback
+      @used_fallbacks = []
+      fallback = fallbacks.find { |color| !@used_fallbacks.include?(color) }
+    end
+    @used_fallbacks << fallback
+    fallback[1]['color'] || "#000"
+  end
+
+  class << self
+    attr_writer :rrddir
+
+    def rrddir
+      @rrddir || @rrddir = "/var/lib/collectd/rrd"
+    end
+
+    def hosts
+      if @rrddir
+        Dir.glob("#{@rrddir}/*").map {|e| e.split('/').last }.sort
+      end
+    end
+
+    def plugins(opts={})
+      host = opts[:host] || '*'
+      Dir.glob("#{@rrddir}/#{host}/*").map {|e| e.split('/').last }.sort
+    end
+
+  end
+
+end