visage-app 2.0.5 → 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore CHANGED
@@ -10,3 +10,4 @@ _site
10
10
  features/data/config/*/*.yaml
11
11
  vendor
12
12
  .bundle
13
+ .rbenv-version
data/AUTHORS CHANGED
@@ -7,6 +7,7 @@ Contributors include
7
7
  Jeffrey Lim <jfs.world@gmail.com>
8
8
  Andrew Harvey <andrew@mootpointer.com>
9
9
  Xavier Mehrenberger <xavier.mehrenberger@gmail.com>
10
+ Jesse Reynolds <jesse@bulletproof.net>
10
11
 
11
12
  Inspiration given by
12
13
  --------------------
@@ -1,3 +1,7 @@
1
+ 2.1.0 - 2012/04/20
2
+ - Add support for drawing 95th percentile lines on graphs
3
+ Thanks to Jesse Reynolds for the feature addition!
4
+
1
5
  2.0.5 - 2012/03/30
2
6
  - Fix a bug where metrics occasionally won't appear in the builder interface.
3
7
  Thanks to Jesse Reynolds for the patch!
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- visage-app (2.0.5)
4
+ visage-app (2.1.0)
5
5
  errand (= 0.7.3)
6
6
  haml (= 3.1.4)
7
7
  sinatra (= 1.3.2)
data/README.md CHANGED
@@ -227,11 +227,11 @@ rake install
227
227
  Releasing
228
228
  ---------
229
229
 
230
- # Bump the version in lib/visage-app/version.rb
231
- # Add an entry to CHANGELOG.md
232
- # `git commit` everything.
233
- # Build the gem with `rake build`
234
- # Push the gem to RubyGems.org with `rake push`
230
+ 1. Bump the version in lib/visage-app/version.rb
231
+ 2. Add an entry to CHANGELOG.md
232
+ 3. `git commit` everything.
233
+ 4. Build the gem with `rake build`
234
+ 5. Push the gem to RubyGems.org with `rake push`
235
235
 
236
236
  Licencing
237
237
  ---------
data/Rakefile CHANGED
@@ -56,7 +56,7 @@ namespace :verify do
56
56
  @root = Pathname.new(File.dirname(__FILE__)).expand_path
57
57
  javascripts_path = @root.join('lib/visage-app/public/javascripts')
58
58
 
59
- javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /mootools|src\.js/ }
59
+ javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /highcharts|mootools|src\.js/ }
60
60
  javascripts.each do |filename|
61
61
  puts "Checking #{filename}".green
62
62
  count = `grep -c 'console.log' #{filename}`.strip.to_i
@@ -10,12 +10,6 @@ Feature: Export data
10
10
  Then I should receive valid JSON
11
11
  And the JSON should have a list of hosts
12
12
 
13
- Scenario: Retreive a list of hosts
14
- When I visit the first available host
15
- Then the request should succeed
16
- Then I should receive valid JSON
17
- And the JSON should have a list of plugins
18
-
19
13
  Scenario: Get a list of available metrics on a host
20
14
  When I visit the first available host
21
15
  Then the request should succeed
@@ -86,3 +80,13 @@ Feature: Export data
86
80
  | start | 1321769692 |
87
81
  | finish | 1321773292 |
88
82
 
83
+ Scenario: Retrieve data for a defined period including 95th percentile calculations
84
+ Given a list of hosts exist
85
+ When I visit "disk*/disk_ops" on the first available host with the following query parameters:
86
+ | parameter | value |
87
+ | start | 1 hour ago |
88
+ | finish | now |
89
+ | percentiles | 95 |
90
+ Then the request should succeed
91
+ Then I should receive valid JSON
92
+ And I should see a 95th percentile value for each plugin instance
@@ -5,23 +5,18 @@ Feature: Visit site
5
5
 
6
6
  Scenario: List profiles
7
7
  When I go to /profiles
8
- Then I should see a list of profiles
9
- When I follow "created"
10
- Then I should see a list of profiles
11
- When I follow "name"
12
8
  Then I should see a list of profiles sorted alphabetically
13
9
 
14
10
  Scenario: Show profile
15
11
  When I go to /profiles
16
12
  And I visit the first profile
17
- Then I should see a list of graphs
18
- And I should see a profile heading
13
+ Then I should see a profile heading
14
+ And I should see "Back to profiles"
19
15
 
20
16
  Scenario: Navigate profiles
21
17
  When I go to /profiles
22
18
  And I visit the first profile
23
- Then I should see a list of graphs
24
- And I should see a profile heading
19
+ Then I should see a profile heading
25
20
  When I follow "back to profiles"
26
- Then I should see a list of profiles
21
+ Then I should see a list of profiles sorted alphabetically
27
22
 
@@ -20,7 +20,7 @@ Then /^a visage web server should be running$/ do
20
20
  end
21
21
 
22
22
  Then /^I should see "([^"]*)" on the terminal$/ do |string|
23
- output = @pipe.read(250)
23
+ output = @pipe.read(350)
24
24
  output.should =~ /#{string}/
25
25
  end
26
26
 
@@ -42,5 +42,5 @@ Then /^I should see a file at "([^"]*)"$/ do |filename|
42
42
  end
43
43
 
44
44
  Then /^show me the output$/ do
45
- puts @pipe.read(250)
45
+ puts @pipe.read(350)
46
46
  end
@@ -103,7 +103,7 @@ end
103
103
  When /^I visit "([^"]*)" on the first available host$/ do |glob|
104
104
  host = @response["hosts"].first
105
105
  url = "/data/#{host}/#{glob}"
106
- When "I go to #{url}"
106
+ step "I go to #{url}"
107
107
  end
108
108
 
109
109
 
@@ -115,7 +115,7 @@ When /^I visit the first available host$/ do
115
115
 
116
116
  host = @response["hosts"].first
117
117
  url = "/data/#{host}"
118
- When "I go to #{url}"
118
+ step "I go to #{url}"
119
119
  end
120
120
 
121
121
  When /^I visit the first two available hosts$/ do
@@ -126,7 +126,7 @@ When /^I visit the first two available hosts$/ do
126
126
 
127
127
  host = @response["hosts"][0..1].join(',')
128
128
  url = "/data/#{host}"
129
- When "I go to #{url}"
129
+ step "I go to #{url}"
130
130
  end
131
131
 
132
132
  Then /^the JSON should have a list of plugins$/ do
@@ -150,6 +150,14 @@ When /^I visit "([^"]*)" on the first available host with the following query pa
150
150
  url = "/data/#{host}/#{glob}"
151
151
 
152
152
  params = Hash[table.hashes.map { |hash| [hash["parameter"], hash["value"]] }]
153
+ params.each do |key, value|
154
+ if value == "1 hour ago"
155
+ params[key] = (Time.now() - 3600).to_i.to_s
156
+ end
157
+ if value == "now"
158
+ params[key] = Time.now().to_i.to_s
159
+ end
160
+ end
153
161
  query = params.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}.join("&")
154
162
  url += "?#{query}"
155
163
 
@@ -175,3 +183,18 @@ Then /^I should see the following parameters for each plugin instance:$/ do |tab
175
183
 
176
184
  end
177
185
 
186
+ Then /^I should see a 95th percentile value for each plugin instance$/ do
187
+ @response.should_not be_nil
188
+
189
+ @response.each_pair do |host, plugin|
190
+ plugin.each_pair do |instance, metric|
191
+ metric.each_pair do |k, series|
192
+ series.each_pair do |k, data|
193
+ data['percentile_95'].should >= 0
194
+ end
195
+ end
196
+ end
197
+ end
198
+
199
+ end
200
+
@@ -10,7 +10,6 @@ Then /^I should see a list of graphs$/ do
10
10
  follow_redirect!
11
11
  rescue Rack::Test::Error
12
12
  end
13
-
14
13
  doc = Nokogiri::HTML(response_body)
15
14
  doc.search('div#profile div.graph').size.should > 0
16
15
  end
@@ -21,6 +21,8 @@ module Visage
21
21
  set :public_folder, @root.join('lib/visage-app/public')
22
22
  set :views, @root.join('lib/visage-app/views')
23
23
 
24
+ enable :logging
25
+
24
26
  helpers Sinatra::LinkToHelper
25
27
  helpers Sinatra::PageTitleHelper
26
28
  helpers Sinatra::RequireJSHelper
@@ -34,6 +36,7 @@ module Visage
34
36
 
35
37
  # Load up the profiles.yaml. Creates it if it doesn't already exist.
36
38
  Visage::Profile.load
39
+
37
40
  end
38
41
  end
39
42
 
@@ -107,18 +110,23 @@ module Visage
107
110
 
108
111
  # /data/:host/:plugin/:optional_plugin_instance
109
112
  get %r{/data/([^/]+)/([^/]+)((/[^/]+)*)} do
110
- host = params[:captures][0].gsub("\0", "")
111
- plugin = params[:captures][1].gsub("\0", "")
112
- instances = params[:captures][2].gsub("\0", "")
113
- start = params[:start]
114
- finish = params[:finish]
113
+ host = params[:captures][0].gsub("\0", "")
114
+ plugin = params[:captures][1].gsub("\0", "")
115
+ instances = params[:captures][2].gsub("\0", "")
116
+ start = params[:start]
117
+ finish = params[:finish]
118
+ percentiles = params[:percentiles] ||= "false"
119
+ resolution = params[:resolution]
115
120
 
116
121
  collectd = Visage::Collectd::JSON.new(:rrddir => Visage::Config.rrddir)
117
- json = collectd.json(:host => host,
118
- :plugin => plugin,
119
- :instances => instances,
120
- :start => start,
121
- :finish => finish)
122
+
123
+ json = collectd.json(:host => host,
124
+ :plugin => plugin,
125
+ :instances => instances,
126
+ :start => start,
127
+ :finish => finish,
128
+ :percentiles => percentiles,
129
+ :resolution => resolution)
122
130
 
123
131
  # If the request is cross-domain, we need to serve JSON-P.
124
132
  maybe_wrap_with_callback(json)
@@ -6,6 +6,48 @@ require 'lib/visage-app/patches'
6
6
  require 'errand'
7
7
  require 'yajl'
8
8
 
9
+ class Array
10
+ def in_groups(number, fill_with = nil)
11
+ raise "Error - in_groups of zero doesn't make sense" unless number > 0
12
+ # size / number gives minor group size;
13
+ # size % number gives how many objects need extra accomodation;
14
+ # each group hold either division or division + 1 items.
15
+ division = size / number
16
+ modulo = size % number
17
+
18
+ # create a new array avoiding dup
19
+ groups = []
20
+ start = 0
21
+
22
+ number.times do |index|
23
+ length = division + (modulo > 0 && modulo > index ? 1 : 0)
24
+ padding = fill_with != false &&
25
+ modulo > 0 && length == division ? 1 : 0
26
+ groups << slice(start, length).concat([fill_with] * padding)
27
+ start += length
28
+ end
29
+
30
+ if block_given?
31
+ groups.each{|g| yield(g) }
32
+ else
33
+ groups
34
+ end
35
+ end
36
+
37
+ def sum
38
+ inject( nil ) { |sum,x| sum ? sum+x : x }
39
+ end
40
+
41
+ def mean
42
+ if size > 0
43
+ sum / size
44
+ else
45
+ nil
46
+ end
47
+ end
48
+
49
+ end
50
+
9
51
  # Exposes RRDs as JSON.
10
52
  #
11
53
  # A loose shim onto RRDtool, with some extra logic to normalise the data.
@@ -32,14 +74,16 @@ module Visage
32
74
 
33
75
  # Entry point.
34
76
  def json(opts={})
35
- host = opts[:host]
36
- plugin = opts[:plugin]
37
- instances = opts[:instances][/\w.*/]
38
- instances = instances.blank? ? '*' : '{' + instances.split('/').join(',') + '}'
39
- rrdglob = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
40
- finish = parse_time(opts[:finish])
41
- start = parse_time(opts[:start], :default => (finish - 3600 || (Time.now - 3600).to_i))
42
- data = []
77
+ host = opts[:host]
78
+ plugin = opts[:plugin]
79
+ instances = opts[:instances][/\w.*/]
80
+ instances = instances.blank? ? '*' : '{' + instances.split('/').join(',') + '}'
81
+ percentiles = opts[:percentiles] !~ /^$|^false$/ ? true : false
82
+ resolution = opts[:resolution] || ""
83
+ rrdglob = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
84
+ finish = parse_time(opts[:finish])
85
+ start = parse_time(opts[:start], :default => (finish - 3600 || (Time.now - 3600).to_i))
86
+ data = []
43
87
 
44
88
  Dir.glob(rrdglob).map do |rrdname|
45
89
  parts = rrdname.gsub(/#{@rrddir}\//, '').split('/')
@@ -48,16 +92,45 @@ module Visage
48
92
  instance_name = File.basename(parts[2], '.rrd')
49
93
  rrd = Errand.new(:filename => rrdname)
50
94
 
51
- data << { :plugin => plugin_name, :instance => instance_name,
52
- :host => host_name,
53
- :start => start,
54
- :finish => finish,
55
- :rrd => rrd }
95
+ data << { :plugin => plugin_name, :instance => instance_name,
96
+ :host => host_name,
97
+ :start => start,
98
+ :finish => finish,
99
+ :rrd => rrd,
100
+ :percentiles => percentiles,
101
+ :resolution => resolution}
102
+
56
103
  end
57
104
 
58
105
  encode(data)
59
106
  end
60
107
 
108
+ def percentile_of_array(samples, percentage)
109
+ if samples
110
+ samples.sort[ (samples.length.to_f * ( percentage.to_f / 100.to_f ) ).to_i - 1 ]
111
+ else
112
+ raise "I can't work out percentiles on a nil sample set"
113
+ end
114
+ end
115
+
116
+ def downsample_array(samples, old_resolution, new_resolution)
117
+ return samples unless samples.length > 0
118
+ timer_start = Time.now
119
+ new_samples = []
120
+ if (new_resolution > 0) and (old_resolution > 0) and (new_resolution % old_resolution == 0)
121
+ groups_of = samples.length / (new_resolution / old_resolution)
122
+ return samples unless groups_of > 0
123
+ samples.in_groups(groups_of, false) {|group|
124
+ new_samples << group.compact.mean
125
+ }
126
+ else
127
+ raise "downsample_array: cowardly refusing to downsample as old_resolution (#{old_resolution.to_s}) doesn't go into new_resolution (#{new_resolution.to_s}) evenly, or new_resolution or old_resolution are zero."
128
+ end
129
+ timer = Time.now - timer_start
130
+
131
+ new_samples
132
+ end
133
+
61
134
  private
62
135
  # Attempt to structure the JSON reasonably sanely, so the consumer (i.e. a
63
136
  # browser) doesn't have to do a lot of computationally expensive work.
@@ -65,17 +138,25 @@ module Visage
65
138
 
66
139
  structure = {}
67
140
  datas.each do |data|
68
- fetch = data[:rrd].fetch(:function => "AVERAGE",
69
- :start => data[:start],
70
- :finish => data[:finish])
141
+
142
+ start = data[:start].to_i
143
+ finish = data[:finish].to_i
144
+ resolution = data[:resolution].to_i || 0
145
+
146
+ fetch = data[:rrd].fetch(:function => "AVERAGE",
147
+ :start => start.to_s,
148
+ :finish => finish.to_s)
149
+
71
150
  rrd_data = fetch[:data]
151
+ percentiles = data[:percentiles]
72
152
 
73
153
  # A single rrd can have multiple data sets (multiple metrics within
74
154
  # the same file). Separate the metrics.
75
155
  rrd_data.each_pair do |source, metric|
76
156
 
77
157
  # Filter out NaNs and weirdly massive values so yajl doesn't choke
78
- metric.map! do |datapoint|
158
+ # FIXME: does this actually do anything?
159
+ metric = metric.map do |datapoint|
79
160
  case
80
161
  when datapoint && datapoint.nan?
81
162
  @tripped = true
@@ -87,21 +168,40 @@ module Visage
87
168
  end
88
169
  end
89
170
 
90
- # Last value is always wack. Set to 0, so the timescale isn't off by 1.
91
- metric[-1] = 0.0
171
+ # Last value is always wack. Remove it.
172
+ metric = metric[0...metric.length-1]
92
173
  host = data[:host]
93
174
  plugin = data[:plugin]
94
175
  instance = data[:instance]
95
- start = data[:start].to_i
96
- finish = data[:finish].to_i
176
+
177
+ # only calculate percentiles if requested
178
+ if percentiles
179
+ timeperiod = finish.to_f - start.to_f
180
+ interval = (timeperiod / metric.length.to_f).round
181
+ resolution = 300
182
+ if (interval < resolution) and (resolution > 0)
183
+ metric_for_percentiles = downsample_array(metric, interval, resolution)
184
+ else
185
+ metric_for_percentiles = metric
186
+ end
187
+ metric_for_percentiles.compact!
188
+ percentiles = false unless metric_for_percentiles.length > 0
189
+ end
190
+
191
+ if metric.length > 2000
192
+ metric = downsample_array(metric, 1, metric.length / 1000)
193
+ end
97
194
 
98
195
  structure[host] ||= {}
99
196
  structure[host][plugin] ||= {}
100
197
  structure[host][plugin][instance] ||= {}
101
198
  structure[host][plugin][instance][source] ||= {}
102
- structure[host][plugin][instance][source][:start] ||= start
103
- structure[host][plugin][instance][source][:finish] ||= finish
104
- structure[host][plugin][instance][source][:data] ||= metric
199
+ structure[host][plugin][instance][source][:start] ||= start
200
+ structure[host][plugin][instance][source][:finish] ||= finish
201
+ structure[host][plugin][instance][source][:data] ||= metric
202
+ structure[host][plugin][instance][source][:percentile_95] ||= percentile_of_array(metric_for_percentiles, 95).round if percentiles
203
+ structure[host][plugin][instance][source][:percentile_50] ||= percentile_of_array(metric_for_percentiles, 50).round if percentiles
204
+ structure[host][plugin][instance][source][:percentile_5] ||= percentile_of_array(metric_for_percentiles, 5).round if percentiles
105
205
 
106
206
  end
107
207
  end