vpsb_client 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +22 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +29 -0
- data/Rakefile +2 -0
- data/bin/close_trial.rb +20 -0
- data/bin/create_trial.rb +13 -0
- data/bin/get_current_trial.rb +8 -0
- data/bin/get_item_ids.rb +9 -0
- data/bin/get_trial_last_metric.rb +8 -0
- data/bin/signed_in.rb +7 -0
- data/bin/sync_metrics.rb +19 -0
- data/bin/test_timing_interval.rb +18 -0
- data/bin/upload_metric.rb +37 -0
- data/doc/design.rb +109 -0
- data/lib/vpsb_client/api/close_trial_request.rb +23 -0
- data/lib/vpsb_client/api/create_trial_request.rb +38 -0
- data/lib/vpsb_client/api/get_current_trial_request.rb +30 -0
- data/lib/vpsb_client/api/get_item_id_request.rb +33 -0
- data/lib/vpsb_client/api/get_trial_last_metric_request.rb +25 -0
- data/lib/vpsb_client/api/post_metric_request.rb +28 -0
- data/lib/vpsb_client/api/request.rb +57 -0
- data/lib/vpsb_client/api/response.rb +30 -0
- data/lib/vpsb_client/builders/system_info_parser.rb +127 -0
- data/lib/vpsb_client/builders/trial.rb +54 -0
- data/lib/vpsb_client/client/upload_metrics.rb +39 -0
- data/lib/vpsb_client/config.rb +18 -0
- data/lib/vpsb_client/curl_wrapper.rb +41 -0
- data/lib/vpsb_client/datafiles/formatted_sar_log_parser.rb +38 -0
- data/lib/vpsb_client/datafiles/logfile_decompressor.rb +48 -0
- data/lib/vpsb_client/datafiles/pxx_aggregator.rb +94 -0
- data/lib/vpsb_client/datafiles/sar_manager.rb +71 -0
- data/lib/vpsb_client/datafiles/timing_log_parser.rb +32 -0
- data/lib/vpsb_client/http_client.rb +60 -0
- data/lib/vpsb_client/manager.rb +149 -0
- data/lib/vpsb_client/metrics/interval_builder.rb +85 -0
- data/lib/vpsb_client/metrics/interval_config.rb +45 -0
- data/lib/vpsb_client/metrics/manager.rb +34 -0
- data/lib/vpsb_client/metrics/uploader.rb +16 -0
- data/lib/vpsb_client/version.rb +3 -0
- data/lib/vpsb_client.rb +33 -0
- data/spec/lib/client/upload_metrics_spec.rb +23 -0
- data/spec/lib/config_spec.rb +26 -0
- data/spec/lib/logfile_decompressor_spec.rb +74 -0
- data/spec/lib/metrics/interval_builder_spec.rb +88 -0
- data/spec/lib/metrics/interval_config_spec.rb +99 -0
- data/spec/lib/metrics/manager_spec.rb +60 -0
- data/spec/lib/metrics/uploader_spec.rb +45 -0
- data/spec/lib/pxx_spec.rb +120 -0
- data/spec/lib/request_spec.rb +186 -0
- data/spec/lib/sar_manager_spec.rb +57 -0
- data/spec/lib/system_info_parser_spec.rb +51 -0
- data/spec/spec_helper.rb +9 -0
- data/spec/support/sarfiles/formatted/formatted_sa20 +147 -0
- data/spec/support/sarfiles/formatted/formatted_sa21 +85 -0
- data/spec/support/sarfiles/orig/sa18 +0 -0
- data/spec/support/sarfiles/orig/sa19 +0 -0
- data/spec/support/sarfiles/orig/sa20 +0 -0
- data/spec/support/sarfiles/orig/sa21 +0 -0
- data/spec/support/timingfiles/other.log.2.gz +0 -0
- data/spec/support/timingfiles/timings.log +11 -0
- data/spec/support/timingfiles/timings.log.1 +11 -0
- data/spec/support/timingfiles/timings.log.2.gz +0 -0
- data/spec/support/timingfiles/timings.log.3.gz +0 -0
- data/spec/support/vpsb.yml +12 -0
- data/vpsb_client.gemspec +29 -0
- metadata +241 -0
@@ -0,0 +1,149 @@
|
|
1
|
+
require 'io/console'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/manager"
|
5
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/client/upload_metrics"
|
6
|
+
|
7
|
+
module VpsbClient
|
8
|
+
class Manager
|
9
|
+
attr_reader :http_client, :logger, :config
|
10
|
+
|
11
|
+
class LastMetricNotFoundError < StandardError; end
|
12
|
+
|
13
|
+
def initialize(config_path, logger=Logger.new(STDOUT))
|
14
|
+
@config_path = config_path
|
15
|
+
@logger = logger
|
16
|
+
end
|
17
|
+
|
18
|
+
def setup
|
19
|
+
VpsbClient.logger = @logger
|
20
|
+
@config = Config.new(@config_path)
|
21
|
+
@curl_wrapper = CurlWrapper.new(@config['auth_token'])
|
22
|
+
@http_client = HttpClient.new(@curl_wrapper, @config['vpsb_protocol'], @config['vpsb_hostname'])
|
23
|
+
end
|
24
|
+
|
25
|
+
def enabled?
|
26
|
+
@config.fetch(:enabled, false)
|
27
|
+
end
|
28
|
+
|
29
|
+
def signed_in?
|
30
|
+
return @signed_in if @signed_in
|
31
|
+
id_request = Api::GetItemIdRequest.new(@http_client, 'hosters', 'linode')
|
32
|
+
curl_response = id_request.run
|
33
|
+
|
34
|
+
begin
|
35
|
+
http_response = Api::Response.new(curl_response)
|
36
|
+
rescue Api::Response::NotAuthenticated => e
|
37
|
+
return @signed_in = false
|
38
|
+
end
|
39
|
+
@signed_in = true
|
40
|
+
end
|
41
|
+
|
42
|
+
def create_trial
|
43
|
+
unless enabled?
|
44
|
+
VpsbClient.logger.debug "not running because vpsb_client is disabled"
|
45
|
+
return
|
46
|
+
end
|
47
|
+
|
48
|
+
builder = Builders::Trial.new(@config, hoster_id, application_id, plan_id)
|
49
|
+
create_trial_request = Api::CreateTrialRequest.new(@http_client, builder.create_params)
|
50
|
+
curl_response = create_trial_request.run
|
51
|
+
http_response = Api::Response.new(curl_response)
|
52
|
+
Api::CreateTrialRequest.trial(http_response)
|
53
|
+
end
|
54
|
+
|
55
|
+
def current_trial
|
56
|
+
builder = Builders::Trial.new(@config)
|
57
|
+
current_trial_request = Api::GetCurrentTrialRequest.new(@http_client, builder.lookup_params)
|
58
|
+
curl_response = current_trial_request.run
|
59
|
+
http_response = Api::Response.new(curl_response)
|
60
|
+
Api::GetCurrentTrialRequest.trial(http_response)
|
61
|
+
end
|
62
|
+
|
63
|
+
def trial_last_metric_started_at(trial_id, length)
|
64
|
+
current_trial_request = Api::GetTrialLastMetricRequest.new(@http_client, { trial_id: trial_id, length: length})
|
65
|
+
curl_response = current_trial_request.run
|
66
|
+
http_response = Api::Response.new(curl_response)
|
67
|
+
Api::GetTrialLastMetricRequest.started_at(http_response)
|
68
|
+
end
|
69
|
+
|
70
|
+
def hoster_id
|
71
|
+
return @hoster_id if @hoster_id
|
72
|
+
id_request = Api::GetItemIdRequest.new(@http_client, 'hosters', @config['hoster_name'])
|
73
|
+
http_response = Api::Response.new(id_request.run)
|
74
|
+
id = Api::GetItemIdRequest.item_id(http_response)
|
75
|
+
raise NameError, "#{@config['hoster_name']} hoster not found" unless id
|
76
|
+
@hoster_id = id
|
77
|
+
end
|
78
|
+
|
79
|
+
def application_id
|
80
|
+
return @application_id if @application_id
|
81
|
+
id_request = Api::GetItemIdRequest.new(@http_client, 'applications', @config['application_name'])
|
82
|
+
http_response = Api::Response.new(id_request.run)
|
83
|
+
id = Api::GetItemIdRequest.item_id(http_response)
|
84
|
+
raise NameError, "#{@config['application_name']} application not found" unless id
|
85
|
+
@application_id = id
|
86
|
+
end
|
87
|
+
|
88
|
+
def plan_id
|
89
|
+
return @plan_id if @plan_id
|
90
|
+
id_request = Api::GetPlanIdRequest.new(@http_client, hoster_id, @config['plan_name'])
|
91
|
+
http_response = Api::Response.new(id_request.run)
|
92
|
+
id = Api::GetPlanIdRequest.item_id(http_response)
|
93
|
+
raise NameError, "#{@config['plan_name']} plan not found" unless id
|
94
|
+
@plan_id = id
|
95
|
+
end
|
96
|
+
|
97
|
+
include Client::UploadMetrics
|
98
|
+
|
99
|
+
def close_trial(trial)
|
100
|
+
unless enabled?
|
101
|
+
logger.debug "[vpsb] not running because vpsb_client is disabled"
|
102
|
+
return
|
103
|
+
end
|
104
|
+
|
105
|
+
prepare_logfiles
|
106
|
+
|
107
|
+
metric_ids = []
|
108
|
+
interval_length = 604800
|
109
|
+
last_started_at = trial_last_metric_started_at(trial['id'], interval_length)
|
110
|
+
if last_started_at
|
111
|
+
start_time = last_started_at + interval_length
|
112
|
+
else
|
113
|
+
logger.debug "[vpsb] close_trial - no last metric found"
|
114
|
+
start_time = Time.now - interval_length
|
115
|
+
end
|
116
|
+
logger.debug "[vpsb] close_trial - length=#{interval_length} start_time=#{start_time} force=false"
|
117
|
+
interval_config = Metrics::IntervalConfig.new(start_time, interval_length, force: true)
|
118
|
+
metrics_manager = metrics_manager(trial['id'], interval_config)
|
119
|
+
metrics_manager.run
|
120
|
+
metric_ids += metrics_manager.created_metric_ids
|
121
|
+
logger.debug "[vpsb] Created metric ids: #{metric_ids.inspect}"
|
122
|
+
|
123
|
+
close_request = Api::CloseTrialRequest.new(@http_client, trial['id'])
|
124
|
+
http_response = Api::Response.new(close_request.run)
|
125
|
+
logger.debug "[vpsb] close request response code = #{http_response.code}"
|
126
|
+
http_response
|
127
|
+
end
|
128
|
+
|
129
|
+
private
|
130
|
+
|
131
|
+
def prepare_logfiles
|
132
|
+
sar_manager = Datafiles::SarManager.new(@config['sar_path'], @config['formatted_sar_path'])
|
133
|
+
sar_manager.run
|
134
|
+
|
135
|
+
logfile_decompressor = Datafiles::LogfileDecompressor.new(@config['timing_path'], @config['timing_path'], :filename_prefix => 'timings')
|
136
|
+
logfile_decompressor.run
|
137
|
+
end
|
138
|
+
|
139
|
+
|
140
|
+
def metrics_manager(trial_id, interval_config)
|
141
|
+
min_start_time = interval_config.min_start_time
|
142
|
+
builder = Metrics::IntervalBuilder.new(@config['formatted_sar_path'], @config['timing_path'], min_start_time, interval_config.length)
|
143
|
+
uploader = Metrics::Uploader.new(@http_client, trial_id)
|
144
|
+
|
145
|
+
Metrics::Manager.new(builder, uploader, interval_config)
|
146
|
+
end
|
147
|
+
|
148
|
+
end
|
149
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require "#{File.expand_path('../..', __FILE__)}/datafiles/formatted_sar_log_parser"
|
2
|
+
require "#{File.expand_path('../..', __FILE__)}/datafiles/timing_log_parser"
|
3
|
+
|
4
|
+
module VpsbClient
|
5
|
+
module Metrics
|
6
|
+
class IntervalBuilder
|
7
|
+
class FileNotFound < StandardError; end
|
8
|
+
|
9
|
+
VALID_METRIC_KEYS = [ :trial_id, :started_at, :duration_seconds, :num_requests, :resptime_total_ms, :resptime_db_ms, :resptime_view_ms, :cpu_idle, :cpu_steal, :iowait, :p50_total_ms, :p75_total_ms, :p95_total_ms, :p99_total_ms, :p75_iowait_pct, :p95_iowait_pct, :p99_iowait_pct,:p75_cpusteal_pct, :p95_cpusteal_pct, :p99_cpusteal_pct, :p75_cpuidle_pct, :p95_cpuidle_pct, :p99_cpuidle_pct ]
|
10
|
+
|
11
|
+
def initialize(sar_path, timing_path, pole_time, interval_length)
|
12
|
+
@sar_path = sar_path
|
13
|
+
@timing_path = timing_path
|
14
|
+
@pole_time = pole_time
|
15
|
+
@interval_length = interval_length
|
16
|
+
end
|
17
|
+
|
18
|
+
def each(&block)
|
19
|
+
return enum_for(:each) unless block_given?
|
20
|
+
|
21
|
+
sar_filenames = Dir.glob("#{@sar_path}/formatted_sa*")
|
22
|
+
timing_filenames = Dir.glob("#{@timing_path}/timings.log*")
|
23
|
+
|
24
|
+
sar_filenames.reject! { |f| f =~ /\.gz$/ }
|
25
|
+
timing_filenames.reject! { |f| f =~ /\.gz$/ }
|
26
|
+
|
27
|
+
raise FileNotFound, "No file matching #{@sar_path}/formatted_sa*" unless sar_filenames.any?
|
28
|
+
raise FileNotFound, "No file matching #{@timing_path}/timings.log*" unless timing_filenames.any?
|
29
|
+
|
30
|
+
builder_options = { offset_by_start_time: @pole_time }
|
31
|
+
|
32
|
+
sar_files = LogfileInterval::LogfileSet.new(sar_filenames, Datafiles::FormattedSarLogParser, :desc)
|
33
|
+
timing_files = LogfileInterval::LogfileSet.new(timing_filenames, Datafiles::TimingLogParser, :desc)
|
34
|
+
|
35
|
+
begin
|
36
|
+
sar_builder = LogfileInterval::IntervalBuilder.new(sar_files, Datafiles::FormattedSarLogParser, @interval_length, builder_options)
|
37
|
+
timing_builder = LogfileInterval::IntervalBuilder.new(timing_files, Datafiles::TimingLogParser, @interval_length, builder_options)
|
38
|
+
|
39
|
+
sar_enum = sar_builder.each_interval
|
40
|
+
timing_enum = timing_builder.each_interval
|
41
|
+
while(timing_interval = timing_enum.next) do
|
42
|
+
while(sar_interval = sar_enum.next) do
|
43
|
+
break if sar_interval.start_time == timing_interval.start_time
|
44
|
+
raise "sar_interval older than timing_interval: #{sar_interval.start_time} > #{timing_interval.start_time}" if sar_interval.start_time > timing_interval.start_time
|
45
|
+
end
|
46
|
+
|
47
|
+
yield convert_to_metric(timing_interval, sar_interval)
|
48
|
+
end
|
49
|
+
rescue StopIteration
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def convert_to_metric(timing_interval, sar_interval)
|
56
|
+
interval = timing_interval.to_hash.merge(sar_interval)
|
57
|
+
interval[:duration_seconds] = timing_interval.length
|
58
|
+
interval[:started_at] = timing_interval.start_time
|
59
|
+
|
60
|
+
pxx_total_ms = timing_interval[:pxx_total_ms]
|
61
|
+
interval[:p50_total_ms] = pxx_total_ms[50]
|
62
|
+
interval[:p75_total_ms] = pxx_total_ms[75]
|
63
|
+
interval[:p95_total_ms] = pxx_total_ms[95]
|
64
|
+
interval[:p99_total_ms] = pxx_total_ms[99]
|
65
|
+
|
66
|
+
pxx_iowait = sar_interval[:pxx_iowait]
|
67
|
+
interval[:p75_iowait_pct] = pxx_iowait[75]
|
68
|
+
interval[:p95_iowait_pct] = pxx_iowait[95]
|
69
|
+
interval[:p99_iowait_pct] = pxx_iowait[99]
|
70
|
+
|
71
|
+
pxx_cpusteal = sar_interval[:pxx_cpusteal]
|
72
|
+
interval[:p75_cpusteal_pct] = pxx_cpusteal[75]
|
73
|
+
interval[:p95_cpusteal_pct] = pxx_cpusteal[95]
|
74
|
+
interval[:p99_cpusteal_pct] = pxx_cpusteal[99]
|
75
|
+
|
76
|
+
pxx_cpuidle = sar_interval[:pxx_cpuidle]
|
77
|
+
interval[:p75_cpuidle_pct] = 100.0 - pxx_cpuidle[75]
|
78
|
+
interval[:p95_cpuidle_pct] = 100.0 - pxx_cpuidle[95]
|
79
|
+
interval[:p99_cpuidle_pct] = 100.0 - pxx_cpuidle[99]
|
80
|
+
|
81
|
+
interval.select! { |k| VALID_METRIC_KEYS.include?(k) }
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module VpsbClient
|
2
|
+
module Metrics
|
3
|
+
class IntervalConfig
|
4
|
+
def initialize(start_time, interval_length, options={})
|
5
|
+
@start_time = start_time
|
6
|
+
@interval_length = interval_length
|
7
|
+
@force = options.fetch(:force, false)
|
8
|
+
end
|
9
|
+
|
10
|
+
def aligned?
|
11
|
+
if @force
|
12
|
+
false
|
13
|
+
elsif @interval_length < 86400
|
14
|
+
true
|
15
|
+
else
|
16
|
+
false
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def min_start_time
|
21
|
+
if @force
|
22
|
+
@start_time
|
23
|
+
elsif aligned?
|
24
|
+
lower_boundary_time(@start_time)
|
25
|
+
else
|
26
|
+
@start_time
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def min_end_time
|
31
|
+
min_start_time + @interval_length
|
32
|
+
end
|
33
|
+
|
34
|
+
def length
|
35
|
+
@interval_length
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def lower_boundary_time(t)
|
41
|
+
Time.at((t.to_i / @interval_length.to_i) * @interval_length.to_i)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/interval_builder"
|
2
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/interval_config"
|
3
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/uploader"
|
4
|
+
|
5
|
+
module VpsbClient
|
6
|
+
module Metrics
|
7
|
+
class Manager
|
8
|
+
attr_reader :created_metric_ids
|
9
|
+
|
10
|
+
def initialize(builder, uploader, interval_config)
|
11
|
+
@interval_config = interval_config
|
12
|
+
@builder = builder
|
13
|
+
@uploader = uploader
|
14
|
+
@created_metric_ids = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
if @interval_config.min_end_time > Time.now
|
19
|
+
VpsbClient.logger.info "Skipping #{@interval_config.length} because too early (min_end_time=#{@interval_config.min_end_time})"
|
20
|
+
return
|
21
|
+
end
|
22
|
+
|
23
|
+
@builder.each do |metric|
|
24
|
+
VpsbClient.logger.debug "metric[:started_at]=#{metric[:started_at]} @interval_config.min_start_time=#{@interval_config.min_start_time}"
|
25
|
+
if metric[:started_at] < @interval_config.min_start_time - 1
|
26
|
+
VpsbClient.logger.debug "[vpsb] stop builder loop as #{metric[:started_at]} < #{@interval_config.min_start_time}"
|
27
|
+
break
|
28
|
+
end
|
29
|
+
@created_metric_ids << @uploader.upload(metric)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module VpsbClient
|
2
|
+
module Metrics
|
3
|
+
class Uploader
|
4
|
+
def initialize(http_client, trial_id)
|
5
|
+
@http_client = http_client
|
6
|
+
@trial_id = trial_id
|
7
|
+
end
|
8
|
+
|
9
|
+
def upload(metric)
|
10
|
+
upload_request = Api::PostMetricRequest.new(@http_client, @trial_id, metric)
|
11
|
+
http_response = Api::Response.new(upload_request.run)
|
12
|
+
Api::PostMetricRequest.metric_id(http_response)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
data/lib/vpsb_client.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
VPSB_BASE_PATH = File.expand_path('..', __FILE__)
|
2
|
+
|
3
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/version"
|
4
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/config"
|
5
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/http_client"
|
6
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/curl_wrapper"
|
7
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/manager"
|
8
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/request"
|
9
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/response"
|
10
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/get_current_trial_request"
|
11
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/get_trial_last_metric_request"
|
12
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/get_item_id_request"
|
13
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/create_trial_request"
|
14
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/post_metric_request"
|
15
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/close_trial_request"
|
16
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/builders/system_info_parser"
|
17
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/builders/trial"
|
18
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/datafiles/sar_manager"
|
19
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/datafiles/logfile_decompressor"
|
20
|
+
|
21
|
+
module VpsbClient
|
22
|
+
def self.logger=(logger)
|
23
|
+
@@logger = logger
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.logger
|
27
|
+
if defined?(@@logger)
|
28
|
+
@@logger
|
29
|
+
else
|
30
|
+
@@logger = Logger.new('/dev/null')
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module VpsbClient
|
4
|
+
module Client
|
5
|
+
describe UploadMetrics do
|
6
|
+
before :each do
|
7
|
+
config = { }
|
8
|
+
@manager = Manager.new('fake_path', Logger.new(STDOUT))
|
9
|
+
allow(@manager).to receive(:prepare_logfiles)
|
10
|
+
allow(@manager).to receive(:enabled?).and_return(true)
|
11
|
+
allow(@manager).to receive(:config).and_return(config)
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'upload metrics for all lengths' do
|
15
|
+
trial = double('trial')
|
16
|
+
expect(@manager).to receive(:upload_for_interval_length).with(trial, 600).and_return([])
|
17
|
+
expect(@manager).to receive(:upload_for_interval_length).with(trial, 3600).and_return([])
|
18
|
+
expect(@manager).to receive(:upload_for_interval_length).with(trial, 86400).and_return([])
|
19
|
+
@manager.upload_metrics(trial)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
#require File.join(File.dirname(__FILE__), '..', 'support/lib/timing_log')
|
3
|
+
|
4
|
+
module VpsbClient
|
5
|
+
data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
|
6
|
+
|
7
|
+
describe Config do
|
8
|
+
before :each do
|
9
|
+
support_dir = File.join(File.dirname(__FILE__), '..', 'support/')
|
10
|
+
@config = Config.new("#{support_dir}/vpsb.yml")
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'parses the config file' do
|
14
|
+
expect(@config.fetch('vpsb_hostname')).to eq('localhost:3000')
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'raises an exception for non existent keys' do
|
18
|
+
expect{@config.fetch('foobar')}.to raise_error
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'understands []' do
|
22
|
+
expect{@config.fetch('vpsb_hostname')}.to_not raise_error
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require "#{File.expand_path('../../..', __FILE__)}/lib/vpsb_client/datafiles/logfile_decompressor"
|
3
|
+
|
4
|
+
module VpsbClient
|
5
|
+
|
6
|
+
module Datafiles
|
7
|
+
data_dir = File.join(File.dirname(__FILE__), '..', 'support/timingfiles')
|
8
|
+
|
9
|
+
describe LogfileDecompressor do
|
10
|
+
before :all do
|
11
|
+
@orig_dir = data_dir
|
12
|
+
@target_dir = data_dir
|
13
|
+
end
|
14
|
+
|
15
|
+
context 'without rotation id limit' do
|
16
|
+
before :each do
|
17
|
+
@manager = LogfileDecompressor.new(@orig_dir, @target_dir, :filename_prefix => 'timings')
|
18
|
+
end
|
19
|
+
|
20
|
+
after :each do
|
21
|
+
File.unlink("#{@target_dir}/timings.log.2")
|
22
|
+
File.unlink("#{@target_dir}/timings.log.3")
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'unzips files with .gz extension' do
|
26
|
+
@manager.run
|
27
|
+
expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
|
28
|
+
expect(File.exist?("#{@target_dir}/timings.log.3")).to be true
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'only unzips files with specified prefix' do
|
32
|
+
@manager.run
|
33
|
+
expect(File.exist?("#{@target_dir}/other.log.2")).to be false
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'with rotation id limit' do
|
38
|
+
before :each do
|
39
|
+
@manager = LogfileDecompressor.new(@orig_dir, @target_dir, :filename_prefix => 'timings',
|
40
|
+
:max_rotation_id => 2)
|
41
|
+
end
|
42
|
+
|
43
|
+
after :each do
|
44
|
+
File.unlink("#{@target_dir}/timings.log.2")
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'unzips only files below limit' do
|
48
|
+
@manager.run
|
49
|
+
expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
|
50
|
+
expect(File.exist?("#{@target_dir}/timings.log.3")).to be false
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'without filename prefix' do
|
55
|
+
before :each do
|
56
|
+
@manager = LogfileDecompressor.new(@orig_dir, @target_dir)
|
57
|
+
end
|
58
|
+
|
59
|
+
after :each do
|
60
|
+
File.unlink("#{@target_dir}/timings.log.2")
|
61
|
+
File.unlink("#{@target_dir}/timings.log.3")
|
62
|
+
File.unlink("#{@target_dir}/other.log.2")
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'unzips all files' do
|
66
|
+
@manager.run
|
67
|
+
expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
|
68
|
+
expect(File.exist?("#{@target_dir}/timings.log.3")).to be true
|
69
|
+
expect(File.exist?("#{@target_dir}/other.log.2")).to be true
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module VpsbClient
|
4
|
+
module Metrics
|
5
|
+
support_dir = File.join(File.dirname(__FILE__), '../..', 'support/')
|
6
|
+
|
7
|
+
OLDEST_TIME_IN_LOGFILES = Time.new(2014,9,20,0,0,0)
|
8
|
+
NEWEST_TIME_IN_LOGFILES = Time.new(2014,9,21,13,56,0)
|
9
|
+
|
10
|
+
describe IntervalBuilder do
|
11
|
+
before :each do
|
12
|
+
@sar_dir = "#{support_dir}/sarfiles/formatted"
|
13
|
+
@timing_dir = "#{support_dir}/timingfiles"
|
14
|
+
@length = 3600
|
15
|
+
allow(Time).to receive(:now).and_return(NEWEST_TIME_IN_LOGFILES + @length)
|
16
|
+
end
|
17
|
+
|
18
|
+
context 'with aligned pole time' do
|
19
|
+
before :each do
|
20
|
+
aligned_pole_time = OLDEST_TIME_IN_LOGFILES - 12*3600
|
21
|
+
@builder = IntervalBuilder.new(@sar_dir, @timing_dir, aligned_pole_time, @length)
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'should yield once per full hour' do
|
25
|
+
expected_num_intervals = (NEWEST_TIME_IN_LOGFILES - OLDEST_TIME_IN_LOGFILES) / @length
|
26
|
+
expect { |b| @builder.each(&b) }.to yield_control.at_least(expected_num_intervals)
|
27
|
+
end
|
28
|
+
|
29
|
+
describe 'first interval' do
|
30
|
+
it 'is the most recent' do
|
31
|
+
interval_enum = @builder.each
|
32
|
+
expected_start_time = lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length)
|
33
|
+
expect(interval_enum.next[:started_at]).to eq(expected_start_time)
|
34
|
+
expect(interval_enum.next[:started_at]).to eq(expected_start_time - @length)
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'has the requested length' do
|
38
|
+
i = @builder.each.first
|
39
|
+
expect(i[:duration_seconds]).to eq(@length)
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'has the right averaged timing' do
|
43
|
+
i = @builder.each.first
|
44
|
+
expect(i[:started_at]).to eq(lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length))
|
45
|
+
expect(i[:resptime_total_ms]).to eq(29.5)
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'as the right p95 cpu idle' do
|
49
|
+
i = @builder.each.first
|
50
|
+
expect(i[:p99_cpuidle_pct]).to eq(88.0)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe 'last interval' do
|
55
|
+
it 'starts at the oldest data point' do
|
56
|
+
last_interval = nil
|
57
|
+
@builder.each do |i|
|
58
|
+
last_interval = i
|
59
|
+
end
|
60
|
+
expect(last_interval[:started_at]).to eq(OLDEST_TIME_IN_LOGFILES)
|
61
|
+
expect(last_interval[:duration_seconds]).to eq(@length)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'with unaligned pole time' do
|
67
|
+
before :each do
|
68
|
+
@offset = 60
|
69
|
+
unaligned_pole_time = OLDEST_TIME_IN_LOGFILES - 12*3600 + @offset
|
70
|
+
@builder = IntervalBuilder.new(@sar_dir, @timing_dir, unaligned_pole_time, @length)
|
71
|
+
end
|
72
|
+
|
73
|
+
describe 'first interval' do
|
74
|
+
it 'start with on offset start_time' do
|
75
|
+
i = @builder.each.first
|
76
|
+
expected_start_time = lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length) + @offset
|
77
|
+
expect(i[:started_at]).to eq(expected_start_time)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def lower_time_boundary(t, len)
|
83
|
+
Time.at((t.to_i / len) * len)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module VpsbClient
|
4
|
+
module Metrics
|
5
|
+
describe IntervalConfig do
|
6
|
+
describe :length do
|
7
|
+
it 'returns the provided length' do
|
8
|
+
ic = IntervalConfig.new(nil, 3600)
|
9
|
+
expect(ic.length).to eq(3600)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe :min_end_time do
|
14
|
+
it 'is min_start_time + length' do
|
15
|
+
t = Time.new(2014, 11, 15, 20, 0, 0)
|
16
|
+
ic = IntervalConfig.new(t, 3600)
|
17
|
+
expect(ic.min_end_time).to eq(ic.min_start_time + 3600)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context 'with force' do
|
22
|
+
before :each do
|
23
|
+
@forced_start_time = Time.new(2014, 11, 15, 20, 0, 0)
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'with short interval length' do
|
27
|
+
before :each do
|
28
|
+
@length = 60
|
29
|
+
@ic = IntervalConfig.new(@forced_start_time, @length, force: true)
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'aligned? is false' do
|
33
|
+
expect(@ic.aligned?).to eq(false)
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'min_start_time is forced_start_time' do
|
37
|
+
expect(@ic.min_start_time).to eq(@forced_start_time)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'with long interval length' do
|
42
|
+
before :each do
|
43
|
+
@length = 604800
|
44
|
+
@ic = IntervalConfig.new(@forced_start_time, @length, force: true)
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'aligned? is false' do
|
48
|
+
expect(@ic.aligned?).to eq(false)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'min_start_time is forced_start_time' do
|
52
|
+
expect(@ic.min_start_time).to eq(@forced_start_time)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
context 'without force' do
|
58
|
+
before :each do
|
59
|
+
@forced_start_time = nil
|
60
|
+
@fallback_start_time = Time.new(2014, 11, 15, 20, 0, 0)
|
61
|
+
end
|
62
|
+
|
63
|
+
context 'with short interval length' do
|
64
|
+
before :each do
|
65
|
+
@length = 60
|
66
|
+
@ic = IntervalConfig.new(@fallback_start_time, @length)
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'aligned? is true' do
|
70
|
+
expect(@ic.aligned?).to eq(true)
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'min_start_time is lower aligned boundary of fallback_start_time' do
|
74
|
+
expect(@ic.min_start_time).to eq(lower_time_boundary(@fallback_start_time, @length))
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
context 'with long interval length' do
|
79
|
+
before :each do
|
80
|
+
@length = 604800
|
81
|
+
@ic = IntervalConfig.new(@fallback_start_time, @length)
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'aligned? is false' do
|
85
|
+
expect(@ic.aligned?).to eq(false)
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'min_start_time is fallback_start_time' do
|
89
|
+
expect(@ic.min_start_time).to eq(@fallback_start_time)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def lower_time_boundary(t, len)
|
95
|
+
Time.at((t.to_i / len) * len)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|