visage-app 2.0.5 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/AUTHORS +1 -0
- data/CHANGELOG.md +4 -0
- data/Gemfile.lock +1 -1
- data/README.md +5 -5
- data/Rakefile +1 -1
- data/features/json.feature +10 -6
- data/features/profiles.feature +4 -9
- data/features/step_definitions/cli_steps.rb +2 -2
- data/features/step_definitions/json_steps.rb +26 -3
- data/features/step_definitions/site_steps.rb +0 -1
- data/lib/visage-app.rb +18 -10
- data/lib/visage-app/collectd/json.rb +124 -24
- data/lib/visage-app/profile.rb +13 -7
- data/lib/visage-app/public/javascripts/builder.js +48 -5
- data/lib/visage-app/public/javascripts/graph.js +144 -9
- data/lib/visage-app/public/javascripts/highcharts-mootools-adapter.js +12 -0
- data/lib/visage-app/public/javascripts/highcharts-mootools-adapter.src.js +298 -0
- data/lib/visage-app/public/javascripts/highcharts.js +198 -131
- data/lib/visage-app/public/javascripts/highcharts.src.js +13543 -8724
- data/lib/visage-app/version.rb +1 -1
- data/lib/visage-app/views/builder.haml +2 -32
- data/lib/visage-app/views/builder_form.haml +7 -0
- data/lib/visage-app/views/layout.haml +1 -0
- data/lib/visage-app/views/profile.haml +14 -24
- data/lib/visage-app/views/profiles.haml +8 -5
- metadata +49 -47
data/.gitignore
CHANGED
data/AUTHORS
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
2.1.0 - 2012/04/20
|
2
|
+
- Add support for drawing 95th percentile lines on graphs
|
3
|
+
Thanks to Jesse Reynolds for the feature addition!
|
4
|
+
|
1
5
|
2.0.5 - 2012/03/30
|
2
6
|
- Fix a bug where metrics occasionally won't appear in the builder interface.
|
3
7
|
Thanks to Jesse Reynolds for the patch!
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -227,11 +227,11 @@ rake install
|
|
227
227
|
Releasing
|
228
228
|
---------
|
229
229
|
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
230
|
+
1. Bump the version in lib/visage-app/version.rb
|
231
|
+
2. Add an entry to CHANGELOG.md
|
232
|
+
3. `git commit` everything.
|
233
|
+
4. Build the gem with `rake build`
|
234
|
+
5. Push the gem to RubyGems.org with `rake push`
|
235
235
|
|
236
236
|
Licencing
|
237
237
|
---------
|
data/Rakefile
CHANGED
@@ -56,7 +56,7 @@ namespace :verify do
|
|
56
56
|
@root = Pathname.new(File.dirname(__FILE__)).expand_path
|
57
57
|
javascripts_path = @root.join('lib/visage-app/public/javascripts')
|
58
58
|
|
59
|
-
javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /mootools|src\.js/ }
|
59
|
+
javascripts = Dir.glob("#{javascripts_path + "*"}.js").reject {|f| f =~ /highcharts|mootools|src\.js/ }
|
60
60
|
javascripts.each do |filename|
|
61
61
|
puts "Checking #{filename}".green
|
62
62
|
count = `grep -c 'console.log' #{filename}`.strip.to_i
|
data/features/json.feature
CHANGED
@@ -10,12 +10,6 @@ Feature: Export data
|
|
10
10
|
Then I should receive valid JSON
|
11
11
|
And the JSON should have a list of hosts
|
12
12
|
|
13
|
-
Scenario: Retreive a list of hosts
|
14
|
-
When I visit the first available host
|
15
|
-
Then the request should succeed
|
16
|
-
Then I should receive valid JSON
|
17
|
-
And the JSON should have a list of plugins
|
18
|
-
|
19
13
|
Scenario: Get a list of available metrics on a host
|
20
14
|
When I visit the first available host
|
21
15
|
Then the request should succeed
|
@@ -86,3 +80,13 @@ Feature: Export data
|
|
86
80
|
| start | 1321769692 |
|
87
81
|
| finish | 1321773292 |
|
88
82
|
|
83
|
+
Scenario: Retrieve data for a defined period including 95th percentile calculations
|
84
|
+
Given a list of hosts exist
|
85
|
+
When I visit "disk*/disk_ops" on the first available host with the following query parameters:
|
86
|
+
| parameter | value |
|
87
|
+
| start | 1 hour ago |
|
88
|
+
| finish | now |
|
89
|
+
| percentiles | 95 |
|
90
|
+
Then the request should succeed
|
91
|
+
Then I should receive valid JSON
|
92
|
+
And I should see a 95th percentile value for each plugin instance
|
data/features/profiles.feature
CHANGED
@@ -5,23 +5,18 @@ Feature: Visit site
|
|
5
5
|
|
6
6
|
Scenario: List profiles
|
7
7
|
When I go to /profiles
|
8
|
-
Then I should see a list of profiles
|
9
|
-
When I follow "created"
|
10
|
-
Then I should see a list of profiles
|
11
|
-
When I follow "name"
|
12
8
|
Then I should see a list of profiles sorted alphabetically
|
13
9
|
|
14
10
|
Scenario: Show profile
|
15
11
|
When I go to /profiles
|
16
12
|
And I visit the first profile
|
17
|
-
Then I should see a
|
18
|
-
And I should see
|
13
|
+
Then I should see a profile heading
|
14
|
+
And I should see "Back to profiles"
|
19
15
|
|
20
16
|
Scenario: Navigate profiles
|
21
17
|
When I go to /profiles
|
22
18
|
And I visit the first profile
|
23
|
-
Then I should see a
|
24
|
-
And I should see a profile heading
|
19
|
+
Then I should see a profile heading
|
25
20
|
When I follow "back to profiles"
|
26
|
-
Then I should see a list of profiles
|
21
|
+
Then I should see a list of profiles sorted alphabetically
|
27
22
|
|
@@ -20,7 +20,7 @@ Then /^a visage web server should be running$/ do
|
|
20
20
|
end
|
21
21
|
|
22
22
|
Then /^I should see "([^"]*)" on the terminal$/ do |string|
|
23
|
-
output = @pipe.read(
|
23
|
+
output = @pipe.read(350)
|
24
24
|
output.should =~ /#{string}/
|
25
25
|
end
|
26
26
|
|
@@ -42,5 +42,5 @@ Then /^I should see a file at "([^"]*)"$/ do |filename|
|
|
42
42
|
end
|
43
43
|
|
44
44
|
Then /^show me the output$/ do
|
45
|
-
puts @pipe.read(
|
45
|
+
puts @pipe.read(350)
|
46
46
|
end
|
@@ -103,7 +103,7 @@ end
|
|
103
103
|
When /^I visit "([^"]*)" on the first available host$/ do |glob|
|
104
104
|
host = @response["hosts"].first
|
105
105
|
url = "/data/#{host}/#{glob}"
|
106
|
-
|
106
|
+
step "I go to #{url}"
|
107
107
|
end
|
108
108
|
|
109
109
|
|
@@ -115,7 +115,7 @@ When /^I visit the first available host$/ do
|
|
115
115
|
|
116
116
|
host = @response["hosts"].first
|
117
117
|
url = "/data/#{host}"
|
118
|
-
|
118
|
+
step "I go to #{url}"
|
119
119
|
end
|
120
120
|
|
121
121
|
When /^I visit the first two available hosts$/ do
|
@@ -126,7 +126,7 @@ When /^I visit the first two available hosts$/ do
|
|
126
126
|
|
127
127
|
host = @response["hosts"][0..1].join(',')
|
128
128
|
url = "/data/#{host}"
|
129
|
-
|
129
|
+
step "I go to #{url}"
|
130
130
|
end
|
131
131
|
|
132
132
|
Then /^the JSON should have a list of plugins$/ do
|
@@ -150,6 +150,14 @@ When /^I visit "([^"]*)" on the first available host with the following query pa
|
|
150
150
|
url = "/data/#{host}/#{glob}"
|
151
151
|
|
152
152
|
params = Hash[table.hashes.map { |hash| [hash["parameter"], hash["value"]] }]
|
153
|
+
params.each do |key, value|
|
154
|
+
if value == "1 hour ago"
|
155
|
+
params[key] = (Time.now() - 3600).to_i.to_s
|
156
|
+
end
|
157
|
+
if value == "now"
|
158
|
+
params[key] = Time.now().to_i.to_s
|
159
|
+
end
|
160
|
+
end
|
153
161
|
query = params.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}.join("&")
|
154
162
|
url += "?#{query}"
|
155
163
|
|
@@ -175,3 +183,18 @@ Then /^I should see the following parameters for each plugin instance:$/ do |tab
|
|
175
183
|
|
176
184
|
end
|
177
185
|
|
186
|
+
Then /^I should see a 95th percentile value for each plugin instance$/ do
|
187
|
+
@response.should_not be_nil
|
188
|
+
|
189
|
+
@response.each_pair do |host, plugin|
|
190
|
+
plugin.each_pair do |instance, metric|
|
191
|
+
metric.each_pair do |k, series|
|
192
|
+
series.each_pair do |k, data|
|
193
|
+
data['percentile_95'].should >= 0
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
end
|
200
|
+
|
data/lib/visage-app.rb
CHANGED
@@ -21,6 +21,8 @@ module Visage
|
|
21
21
|
set :public_folder, @root.join('lib/visage-app/public')
|
22
22
|
set :views, @root.join('lib/visage-app/views')
|
23
23
|
|
24
|
+
enable :logging
|
25
|
+
|
24
26
|
helpers Sinatra::LinkToHelper
|
25
27
|
helpers Sinatra::PageTitleHelper
|
26
28
|
helpers Sinatra::RequireJSHelper
|
@@ -34,6 +36,7 @@ module Visage
|
|
34
36
|
|
35
37
|
# Load up the profiles.yaml. Creates it if it doesn't already exist.
|
36
38
|
Visage::Profile.load
|
39
|
+
|
37
40
|
end
|
38
41
|
end
|
39
42
|
|
@@ -107,18 +110,23 @@ module Visage
|
|
107
110
|
|
108
111
|
# /data/:host/:plugin/:optional_plugin_instance
|
109
112
|
get %r{/data/([^/]+)/([^/]+)((/[^/]+)*)} do
|
110
|
-
host
|
111
|
-
plugin
|
112
|
-
instances
|
113
|
-
start
|
114
|
-
finish
|
113
|
+
host = params[:captures][0].gsub("\0", "")
|
114
|
+
plugin = params[:captures][1].gsub("\0", "")
|
115
|
+
instances = params[:captures][2].gsub("\0", "")
|
116
|
+
start = params[:start]
|
117
|
+
finish = params[:finish]
|
118
|
+
percentiles = params[:percentiles] ||= "false"
|
119
|
+
resolution = params[:resolution]
|
115
120
|
|
116
121
|
collectd = Visage::Collectd::JSON.new(:rrddir => Visage::Config.rrddir)
|
117
|
-
|
118
|
-
|
119
|
-
:
|
120
|
-
:
|
121
|
-
:
|
122
|
+
|
123
|
+
json = collectd.json(:host => host,
|
124
|
+
:plugin => plugin,
|
125
|
+
:instances => instances,
|
126
|
+
:start => start,
|
127
|
+
:finish => finish,
|
128
|
+
:percentiles => percentiles,
|
129
|
+
:resolution => resolution)
|
122
130
|
|
123
131
|
# If the request is cross-domain, we need to serve JSON-P.
|
124
132
|
maybe_wrap_with_callback(json)
|
@@ -6,6 +6,48 @@ require 'lib/visage-app/patches'
|
|
6
6
|
require 'errand'
|
7
7
|
require 'yajl'
|
8
8
|
|
9
|
+
class Array
|
10
|
+
def in_groups(number, fill_with = nil)
|
11
|
+
raise "Error - in_groups of zero doesn't make sense" unless number > 0
|
12
|
+
# size / number gives minor group size;
|
13
|
+
# size % number gives how many objects need extra accomodation;
|
14
|
+
# each group hold either division or division + 1 items.
|
15
|
+
division = size / number
|
16
|
+
modulo = size % number
|
17
|
+
|
18
|
+
# create a new array avoiding dup
|
19
|
+
groups = []
|
20
|
+
start = 0
|
21
|
+
|
22
|
+
number.times do |index|
|
23
|
+
length = division + (modulo > 0 && modulo > index ? 1 : 0)
|
24
|
+
padding = fill_with != false &&
|
25
|
+
modulo > 0 && length == division ? 1 : 0
|
26
|
+
groups << slice(start, length).concat([fill_with] * padding)
|
27
|
+
start += length
|
28
|
+
end
|
29
|
+
|
30
|
+
if block_given?
|
31
|
+
groups.each{|g| yield(g) }
|
32
|
+
else
|
33
|
+
groups
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def sum
|
38
|
+
inject( nil ) { |sum,x| sum ? sum+x : x }
|
39
|
+
end
|
40
|
+
|
41
|
+
def mean
|
42
|
+
if size > 0
|
43
|
+
sum / size
|
44
|
+
else
|
45
|
+
nil
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
|
9
51
|
# Exposes RRDs as JSON.
|
10
52
|
#
|
11
53
|
# A loose shim onto RRDtool, with some extra logic to normalise the data.
|
@@ -32,14 +74,16 @@ module Visage
|
|
32
74
|
|
33
75
|
# Entry point.
|
34
76
|
def json(opts={})
|
35
|
-
host
|
36
|
-
plugin
|
37
|
-
instances
|
38
|
-
instances
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
77
|
+
host = opts[:host]
|
78
|
+
plugin = opts[:plugin]
|
79
|
+
instances = opts[:instances][/\w.*/]
|
80
|
+
instances = instances.blank? ? '*' : '{' + instances.split('/').join(',') + '}'
|
81
|
+
percentiles = opts[:percentiles] !~ /^$|^false$/ ? true : false
|
82
|
+
resolution = opts[:resolution] || ""
|
83
|
+
rrdglob = "#{@rrddir}/#{host}/#{plugin}/#{instances}.rrd"
|
84
|
+
finish = parse_time(opts[:finish])
|
85
|
+
start = parse_time(opts[:start], :default => (finish - 3600 || (Time.now - 3600).to_i))
|
86
|
+
data = []
|
43
87
|
|
44
88
|
Dir.glob(rrdglob).map do |rrdname|
|
45
89
|
parts = rrdname.gsub(/#{@rrddir}\//, '').split('/')
|
@@ -48,16 +92,45 @@ module Visage
|
|
48
92
|
instance_name = File.basename(parts[2], '.rrd')
|
49
93
|
rrd = Errand.new(:filename => rrdname)
|
50
94
|
|
51
|
-
data << { :plugin
|
52
|
-
:host
|
53
|
-
:start
|
54
|
-
:finish
|
55
|
-
:rrd
|
95
|
+
data << { :plugin => plugin_name, :instance => instance_name,
|
96
|
+
:host => host_name,
|
97
|
+
:start => start,
|
98
|
+
:finish => finish,
|
99
|
+
:rrd => rrd,
|
100
|
+
:percentiles => percentiles,
|
101
|
+
:resolution => resolution}
|
102
|
+
|
56
103
|
end
|
57
104
|
|
58
105
|
encode(data)
|
59
106
|
end
|
60
107
|
|
108
|
+
def percentile_of_array(samples, percentage)
|
109
|
+
if samples
|
110
|
+
samples.sort[ (samples.length.to_f * ( percentage.to_f / 100.to_f ) ).to_i - 1 ]
|
111
|
+
else
|
112
|
+
raise "I can't work out percentiles on a nil sample set"
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def downsample_array(samples, old_resolution, new_resolution)
|
117
|
+
return samples unless samples.length > 0
|
118
|
+
timer_start = Time.now
|
119
|
+
new_samples = []
|
120
|
+
if (new_resolution > 0) and (old_resolution > 0) and (new_resolution % old_resolution == 0)
|
121
|
+
groups_of = samples.length / (new_resolution / old_resolution)
|
122
|
+
return samples unless groups_of > 0
|
123
|
+
samples.in_groups(groups_of, false) {|group|
|
124
|
+
new_samples << group.compact.mean
|
125
|
+
}
|
126
|
+
else
|
127
|
+
raise "downsample_array: cowardly refusing to downsample as old_resolution (#{old_resolution.to_s}) doesn't go into new_resolution (#{new_resolution.to_s}) evenly, or new_resolution or old_resolution are zero."
|
128
|
+
end
|
129
|
+
timer = Time.now - timer_start
|
130
|
+
|
131
|
+
new_samples
|
132
|
+
end
|
133
|
+
|
61
134
|
private
|
62
135
|
# Attempt to structure the JSON reasonably sanely, so the consumer (i.e. a
|
63
136
|
# browser) doesn't have to do a lot of computationally expensive work.
|
@@ -65,17 +138,25 @@ module Visage
|
|
65
138
|
|
66
139
|
structure = {}
|
67
140
|
datas.each do |data|
|
68
|
-
|
69
|
-
|
70
|
-
|
141
|
+
|
142
|
+
start = data[:start].to_i
|
143
|
+
finish = data[:finish].to_i
|
144
|
+
resolution = data[:resolution].to_i || 0
|
145
|
+
|
146
|
+
fetch = data[:rrd].fetch(:function => "AVERAGE",
|
147
|
+
:start => start.to_s,
|
148
|
+
:finish => finish.to_s)
|
149
|
+
|
71
150
|
rrd_data = fetch[:data]
|
151
|
+
percentiles = data[:percentiles]
|
72
152
|
|
73
153
|
# A single rrd can have multiple data sets (multiple metrics within
|
74
154
|
# the same file). Separate the metrics.
|
75
155
|
rrd_data.each_pair do |source, metric|
|
76
156
|
|
77
157
|
# Filter out NaNs and weirdly massive values so yajl doesn't choke
|
78
|
-
|
158
|
+
# FIXME: does this actually do anything?
|
159
|
+
metric = metric.map do |datapoint|
|
79
160
|
case
|
80
161
|
when datapoint && datapoint.nan?
|
81
162
|
@tripped = true
|
@@ -87,21 +168,40 @@ module Visage
|
|
87
168
|
end
|
88
169
|
end
|
89
170
|
|
90
|
-
# Last value is always wack.
|
91
|
-
metric[-1]
|
171
|
+
# Last value is always wack. Remove it.
|
172
|
+
metric = metric[0...metric.length-1]
|
92
173
|
host = data[:host]
|
93
174
|
plugin = data[:plugin]
|
94
175
|
instance = data[:instance]
|
95
|
-
|
96
|
-
|
176
|
+
|
177
|
+
# only calculate percentiles if requested
|
178
|
+
if percentiles
|
179
|
+
timeperiod = finish.to_f - start.to_f
|
180
|
+
interval = (timeperiod / metric.length.to_f).round
|
181
|
+
resolution = 300
|
182
|
+
if (interval < resolution) and (resolution > 0)
|
183
|
+
metric_for_percentiles = downsample_array(metric, interval, resolution)
|
184
|
+
else
|
185
|
+
metric_for_percentiles = metric
|
186
|
+
end
|
187
|
+
metric_for_percentiles.compact!
|
188
|
+
percentiles = false unless metric_for_percentiles.length > 0
|
189
|
+
end
|
190
|
+
|
191
|
+
if metric.length > 2000
|
192
|
+
metric = downsample_array(metric, 1, metric.length / 1000)
|
193
|
+
end
|
97
194
|
|
98
195
|
structure[host] ||= {}
|
99
196
|
structure[host][plugin] ||= {}
|
100
197
|
structure[host][plugin][instance] ||= {}
|
101
198
|
structure[host][plugin][instance][source] ||= {}
|
102
|
-
structure[host][plugin][instance][source][:start]
|
103
|
-
structure[host][plugin][instance][source][:finish]
|
104
|
-
structure[host][plugin][instance][source][:data]
|
199
|
+
structure[host][plugin][instance][source][:start] ||= start
|
200
|
+
structure[host][plugin][instance][source][:finish] ||= finish
|
201
|
+
structure[host][plugin][instance][source][:data] ||= metric
|
202
|
+
structure[host][plugin][instance][source][:percentile_95] ||= percentile_of_array(metric_for_percentiles, 95).round if percentiles
|
203
|
+
structure[host][plugin][instance][source][:percentile_50] ||= percentile_of_array(metric_for_percentiles, 50).round if percentiles
|
204
|
+
structure[host][plugin][instance][source][:percentile_5] ||= percentile_of_array(metric_for_percentiles, 5).round if percentiles
|
105
205
|
|
106
206
|
end
|
107
207
|
end
|