visage-app 2.0.5 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore CHANGED
@@ -10,3 +10,4 @@ _site
10
10
  features/data/config/*/*.yaml
11
11
  vendor
12
12
  .bundle
13
+ .rbenv-version
data/AUTHORS CHANGED
@@ -7,6 +7,7 @@ Contributors include
7
7
  Jeffrey Lim <jfs.world@gmail.com>
8
8
  Andrew Harvey <andrew@mootpointer.com>
9
9
  Xavier Mehrenberger <xavier.mehrenberger@gmail.com>
10
+ Jesse Reynolds <jesse@bulletproof.net>
10
11
 
11
12
  Inspiration given by
12
13
  --------------------
@@ -1,3 +1,7 @@
1
+ 2.1.0 - 2012/04/20
2
+ - Add support for drawing 95th percentile lines on graphs
3
+ Thanks to Jesse Reynolds for the feature addition!
4
+
1
5
  2.0.5 - 2012/03/30
2
6
  - Fix a bug where metrics occasionally won't appear in the builder interface.
3
7
  Thanks to Jesse Reynolds for the patch!
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- visage-app (2.0.5)
4
+ visage-app (2.1.0)
5
5
  errand (= 0.7.3)
6
6
  haml (= 3.1.4)
7
7
  sinatra (= 1.3.2)
data/README.md CHANGED
@@ -227,11 +227,11 @@ rake install
227
227
  Releasing
228
228
  ---------
229
229
 
230
- # Bump the version in lib/visage-app/version.rb
231
- # Add an entry to CHANGELOG.md
232
- # `git commit` everything.
233
- # Build the gem with `rake build`
234
- # Push the gem to RubyGems.org with `rake push`
230
+ 1. Bump the version in lib/visage-app/version.rb
231
+ 2. Add an entry to CHANGELOG.md
232
+ 3. `git commit` everything.
233
+ 4. Build the gem with `rake build`
234
+ 5. Push the gem to RubyGems.org with `rake push`
235
235
 
236
236
  Licencing
237
237
  ---------
data/Rakefile CHANGED
@@ -56,7 +56,7 @@ namespace :verify do
56
56
  @root = Pathname.new(File.dirname(__FILE__)).expand_path
57
57
  javascripts_path = @root.join('lib/visage-app/public/javascripts')
58
58
 
59
- javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /mootools|src\.js/ }
59
+ javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /highcharts|mootools|src\.js/ }
60
60
  javascripts.each do |filename|
61
61
  puts "Checking #{filename}".green
62
62
  count = `grep -c 'console.log' #{filename}`.strip.to_i
@@ -10,12 +10,6 @@ Feature: Export data
10
10
  Then I should receive valid JSON
11
11
  And the JSON should have a list of hosts
12
12
 
13
- Scenario: Retreive a list of hosts
14
- When I visit the first available host
15
- Then the request should succeed
16
- Then I should receive valid JSON
17
- And the JSON should have a list of plugins
18
-
19
13
  Scenario: Get a list of available metrics on a host
20
14
  When I visit the first available host
21
15
  Then the request should succeed
@@ -86,3 +80,13 @@ Feature: Export data
86
80
  | start | 1321769692 |
87
81
  | finish | 1321773292 |
88
82
 
83
+ Scenario: Retrieve data for a defined period including 95th percentile calculations
84
+ Given a list of hosts exist
85
+ When I visit "disk*/disk_ops" on the first available host with the following query parameters:
86
+ | parameter | value |
87
+ | start | 1 hour ago |
88
+ | finish | now |
89
+ | percentiles | 95 |
90
+ Then the request should succeed
91
+ Then I should receive valid JSON
92
+ And I should see a 95th percentile value for each plugin instance
@@ -5,23 +5,18 @@ Feature: Visit site
5
5
 
6
6
  Scenario: List profiles
7
7
  When I go to /profiles
8
- Then I should see a list of profiles
9
- When I follow "created"
10
- Then I should see a list of profiles
11
- When I follow "name"
12
8
  Then I should see a list of profiles sorted alphabetically
13
9
 
14
10
  Scenario: Show profile
15
11
  When I go to /profiles
16
12
  And I visit the first profile
17
- Then I should see a list of graphs
18
- And I should see a profile heading
13
+ Then I should see a profile heading
14
+ And I should see "Back to profiles"
19
15
 
20
16
  Scenario: Navigate profiles
21
17
  When I go to /profiles
22
18
  And I visit the first profile
23
- Then I should see a list of graphs
24
- And I should see a profile heading
19
+ Then I should see a profile heading
25
20
  When I follow "back to profiles"
26
- Then I should see a list of profiles
21
+ Then I should see a list of profiles sorted alphabetically
27
22
 
@@ -20,7 +20,7 @@ Then /^a visage web server should be running$/ do
20
20
  end
21
21
 
22
22
  Then /^I should see "([^"]*)" on the terminal$/ do |string|
23
- output = @pipe.read(250)
23
+ output = @pipe.read(350)
24
24
  output.should =~ /#{string}/
25
25
  end
26
26
 
@@ -42,5 +42,5 @@ Then /^I should see a file at "([^"]*)"$/ do |filename|
42
42
  end
43
43
 
44
44
  Then /^show me the output$/ do
45
- puts @pipe.read(250)
45
+ puts @pipe.read(350)
46
46
  end
@@ -103,7 +103,7 @@ end
103
103
  When /^I visit "([^"]*)" on the first available host$/ do |glob|
104
104
  host = @response["hosts"].first
105
105
  url = "/data/#{host}/#{glob}"
106
- When "I go to #{url}"
106
+ step "I go to #{url}"
107
107
  end
108
108
 
109
109
 
@@ -115,7 +115,7 @@ When /^I visit the first available host$/ do
115
115
 
116
116
  host = @response["hosts"].first
117
117
  url = "/data/#{host}"
118
- When "I go to #{url}"
118
+ step "I go to #{url}"
119
119
  end
120
120
 
121
121
  When /^I visit the first two available hosts$/ do
@@ -126,7 +126,7 @@ When /^I visit the first two available hosts$/ do
126
126
 
127
127
  host = @response["hosts"][0..1].join(',')
128
128
  url = "/data/#{host}"
129
- When "I go to #{url}"
129
+ step "I go to #{url}"
130
130
  end
131
131
 
132
132
  Then /^the JSON should have a list of plugins$/ do
@@ -150,6 +150,14 @@ When /^I visit "([^"]*)" on the first available host with the following query pa
150
150
  url = "/data/#{host}/#{glob}"
151
151
 
152
152
  params = Hash[table.hashes.map { |hash| [hash["parameter"], hash["value"]] }]
153
+ params.each do |key, value|
154
+ if value == "1 hour ago"
155
+ params[key] = (Time.now() - 3600).to_i.to_s
156
+ end
157
+ if value == "now"
158
+ params[key] = Time.now().to_i.to_s
159
+ end
160
+ end
153
161
  query = params.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}.join("&")
154
162
  url += "?#{query}"
155
163
 
@@ -175,3 +183,18 @@ Then /^I should see the following parameters for each plugin instance:$/ do |tab
175
183
 
176
184
  end
177
185
 
186
+ Then /^I should see a 95th percentile value for each plugin instance$/ do
187
+ @response.should_not be_nil
188
+
189
+ @response.each_pair do |host, plugin|
190
+ plugin.each_pair do |instance, metric|
191
+ metric.each_pair do |k, series|
192
+ series.each_pair do |k, data|
193
+ data['percentile_95'].should >= 0
194
+ end
195
+ end
196
+ end
197
+ end
198
+
199
+ end
200
+
@@ -10,7 +10,6 @@ Then /^I should see a list of graphs$/ do
10
10
  follow_redirect!
11
11
  rescue Rack::Test::Error
12
12
  end
13
-
14
13
  doc = Nokogiri::HTML(response_body)
15
14
  doc.search('div#profile div.graph').size.should > 0
16
15
  end
@@ -21,6 +21,8 @@ module Visage
21
21
  set :public_folder, @root.join('lib/visage-app/public')
22
22
  set :views, @root.join('lib/visage-app/views')
23
23
 
24
+ enable :logging
25
+
24
26
  helpers Sinatra::LinkToHelper
25
27
  helpers Sinatra::PageTitleHelper
26
28
  helpers Sinatra::RequireJSHelper
@@ -34,6 +36,7 @@ module Visage
34
36
 
35
37
  # Load up the profiles.yaml. Creates it if it doesn't already exist.
36
38
  Visage::Profile.load
39
+
37
40
  end
38
41
  end
39
42
 
@@ -107,18 +110,23 @@ module Visage
107
110
 
108
111
  # /data/:host/:plugin/:optional_plugin_instance
109
112
  get %r{/data/([^/]+)/([^/]+)((/[^/]+)*)} do
110
- host = params[:captures][0].gsub("\0", "")
111
- plugin = params[:captures][1].gsub("\0", "")
112
- instances = params[:captures][2].gsub("\0", "")
113
- start = params[:start]
114
- finish = params[:finish]
113
+ host = params[:captures][0].gsub("\0", "")
114
+ plugin = params[:captures][1].gsub("\0", "")
115
+ instances = params[:captures][2].gsub("\0", "")
116
+ start = params[:start]
117
+ finish = params[:finish]
118
+ percentiles = params[:percentiles] ||= "false"
119
+ resolution = params[:resolution]
115
120
 
116
121
  collectd = Visage::Collectd::JSON.new(:rrddir => Visage::Config.rrddir)
117
- json = collectd.json(:host => host,
118
- :plugin => plugin,
119
- :instances => instances,
120
- :start => start,
121
- :finish => finish)
122
+
123
+ json = collectd.json(:host => host,
124
+ :plugin => plugin,
125
+ :instances => instances,
126
+ :start => start,
127
+ :finish => finish,
128
+ :percentiles => percentiles,
129
+ :resolution => resolution)
122
130
 
123
131
  # If the request is cross-domain, we need to serve JSON-P.
124
132
  maybe_wrap_with_callback(json)
@@ -6,6 +6,48 @@ require 'lib/visage-app/patches'
6
6
  require 'errand'
7
7
  require 'yajl'
8
8
 
9
+ class Array
10
+ def in_groups(number, fill_with = nil)
11
+ raise "Error - in_groups of zero doesn't make sense" unless number > 0
12
+ # size / number gives minor group size;
13
+ # size % number gives how many objects need extra accomodation;
14
+ # each group hold either division or division + 1 items.
15
+ division = size / number
16
+ modulo = size % number
17
+
18
+ # create a new array avoiding dup
19
+ groups = []
20
+ start = 0
21
+
22
+ number.times do |index|
23
+ length = division + (modulo > 0 && modulo > index ? 1 : 0)
24
+ padding = fill_with != false &&
25
+ modulo > 0 && length == division ? 1 : 0
26
+ groups << slice(start, length).concat([fill_with] * padding)
27
+ start += length
28
+ end
29
+
30
+ if block_given?
31
+ groups.each{|g| yield(g) }
32
+ else
33
+ groups
34
+ end
35
+ end
36
+
37
+ def sum
38
+ inject( nil ) { |sum,x| sum ? sum+x : x }
39
+ end
40
+
41
+ def mean
42
+ if size > 0
43
+ sum / size
44
+ else
45
+ nil
46
+ end
47
+ end
48
+
49
+ end
50
+
9
51
  # Exposes RRDs as JSON.
10
52
  #
11
53
  # A loose shim onto RRDtool, with some extra logic to normalise the data.
@@ -32,14 +74,16 @@ module Visage
32
74
 
33
75
  # Entry point.
34
76
  def json(opts={})
35
- host = opts[:host]
36
- plugin = opts[:plugin]
37
- instances = opts[:instances][/\w.*/]
38
- instances = instances.blank? ? '*' : '{' + instances.split('/').join(',') + '}'
39
- rrdglob = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
40
- finish = parse_time(opts[:finish])
41
- start = parse_time(opts[:start], :default => (finish - 3600 || (Time.now - 3600).to_i))
42
- data = []
77
+ host = opts[:host]
78
+ plugin = opts[:plugin]
79
+ instances = opts[:instances][/\w.*/]
80
+ instances = instances.blank? ? '*' : '{' + instances.split('/').join(',') + '}'
81
+ percentiles = opts[:percentiles] !~ /^$|^false$/ ? true : false
82
+ resolution = opts[:resolution] || ""
83
+ rrdglob = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
84
+ finish = parse_time(opts[:finish])
85
+ start = parse_time(opts[:start], :default => (finish - 3600 || (Time.now - 3600).to_i))
86
+ data = []
43
87
 
44
88
  Dir.glob(rrdglob).map do |rrdname|
45
89
  parts = rrdname.gsub(/#{@rrddir}\//, '').split('/')
@@ -48,16 +92,45 @@ module Visage
48
92
  instance_name = File.basename(parts[2], '.rrd')
49
93
  rrd = Errand.new(:filename => rrdname)
50
94
 
51
- data << { :plugin => plugin_name, :instance => instance_name,
52
- :host => host_name,
53
- :start => start,
54
- :finish => finish,
55
- :rrd => rrd }
95
+ data << { :plugin => plugin_name, :instance => instance_name,
96
+ :host => host_name,
97
+ :start => start,
98
+ :finish => finish,
99
+ :rrd => rrd,
100
+ :percentiles => percentiles,
101
+ :resolution => resolution}
102
+
56
103
  end
57
104
 
58
105
  encode(data)
59
106
  end
60
107
 
108
+ def percentile_of_array(samples, percentage)
109
+ if samples
110
+ samples.sort[ (samples.length.to_f * ( percentage.to_f / 100.to_f ) ).to_i - 1 ]
111
+ else
112
+ raise "I can't work out percentiles on a nil sample set"
113
+ end
114
+ end
115
+
116
+ def downsample_array(samples, old_resolution, new_resolution)
117
+ return samples unless samples.length > 0
118
+ timer_start = Time.now
119
+ new_samples = []
120
+ if (new_resolution > 0) and (old_resolution > 0) and (new_resolution % old_resolution == 0)
121
+ groups_of = samples.length / (new_resolution / old_resolution)
122
+ return samples unless groups_of > 0
123
+ samples.in_groups(groups_of, false) {|group|
124
+ new_samples << group.compact.mean
125
+ }
126
+ else
127
+ raise "downsample_array: cowardly refusing to downsample as old_resolution (#{old_resolution.to_s}) doesn't go into new_resolution (#{new_resolution.to_s}) evenly, or new_resolution or old_resolution are zero."
128
+ end
129
+ timer = Time.now - timer_start
130
+
131
+ new_samples
132
+ end
133
+
61
134
  private
62
135
  # Attempt to structure the JSON reasonably sanely, so the consumer (i.e. a
63
136
  # browser) doesn't have to do a lot of computationally expensive work.
@@ -65,17 +138,25 @@ module Visage
65
138
 
66
139
  structure = {}
67
140
  datas.each do |data|
68
- fetch = data[:rrd].fetch(:function => "AVERAGE",
69
- :start => data[:start],
70
- :finish => data[:finish])
141
+
142
+ start = data[:start].to_i
143
+ finish = data[:finish].to_i
144
+ resolution = data[:resolution].to_i || 0
145
+
146
+ fetch = data[:rrd].fetch(:function => "AVERAGE",
147
+ :start => start.to_s,
148
+ :finish => finish.to_s)
149
+
71
150
  rrd_data = fetch[:data]
151
+ percentiles = data[:percentiles]
72
152
 
73
153
  # A single rrd can have multiple data sets (multiple metrics within
74
154
  # the same file). Separate the metrics.
75
155
  rrd_data.each_pair do |source, metric|
76
156
 
77
157
  # Filter out NaNs and weirdly massive values so yajl doesn't choke
78
- metric.map! do |datapoint|
158
+ # FIXME: does this actually do anything?
159
+ metric = metric.map do |datapoint|
79
160
  case
80
161
  when datapoint && datapoint.nan?
81
162
  @tripped = true
@@ -87,21 +168,40 @@ module Visage
87
168
  end
88
169
  end
89
170
 
90
- # Last value is always wack. Set to 0, so the timescale isn't off by 1.
91
- metric[-1] = 0.0
171
+ # Last value is always wack. Remove it.
172
+ metric = metric[0...metric.length-1]
92
173
  host = data[:host]
93
174
  plugin = data[:plugin]
94
175
  instance = data[:instance]
95
- start = data[:start].to_i
96
- finish = data[:finish].to_i
176
+
177
+ # only calculate percentiles if requested
178
+ if percentiles
179
+ timeperiod = finish.to_f - start.to_f
180
+ interval = (timeperiod / metric.length.to_f).round
181
+ resolution = 300
182
+ if (interval < resolution) and (resolution > 0)
183
+ metric_for_percentiles = downsample_array(metric, interval, resolution)
184
+ else
185
+ metric_for_percentiles = metric
186
+ end
187
+ metric_for_percentiles.compact!
188
+ percentiles = false unless metric_for_percentiles.length > 0
189
+ end
190
+
191
+ if metric.length > 2000
192
+ metric = downsample_array(metric, 1, metric.length / 1000)
193
+ end
97
194
 
98
195
  structure[host] ||= {}
99
196
  structure[host][plugin] ||= {}
100
197
  structure[host][plugin][instance] ||= {}
101
198
  structure[host][plugin][instance][source] ||= {}
102
- structure[host][plugin][instance][source][:start] ||= start
103
- structure[host][plugin][instance][source][:finish] ||= finish
104
- structure[host][plugin][instance][source][:data] ||= metric
199
+ structure[host][plugin][instance][source][:start] ||= start
200
+ structure[host][plugin][instance][source][:finish] ||= finish
201
+ structure[host][plugin][instance][source][:data] ||= metric
202
+ structure[host][plugin][instance][source][:percentile_95] ||= percentile_of_array(metric_for_percentiles, 95).round if percentiles
203
+ structure[host][plugin][instance][source][:percentile_50] ||= percentile_of_array(metric_for_percentiles, 50).round if percentiles
204
+ structure[host][plugin][instance][source][:percentile_5] ||= percentile_of_array(metric_for_percentiles, 5).round if percentiles
105
205
 
106
206
  end
107
207
  end