vpsb_client 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +22 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +29 -0
- data/Rakefile +2 -0
- data/bin/close_trial.rb +20 -0
- data/bin/create_trial.rb +13 -0
- data/bin/get_current_trial.rb +8 -0
- data/bin/get_item_ids.rb +9 -0
- data/bin/get_trial_last_metric.rb +8 -0
- data/bin/signed_in.rb +7 -0
- data/bin/sync_metrics.rb +19 -0
- data/bin/test_timing_interval.rb +18 -0
- data/bin/upload_metric.rb +37 -0
- data/doc/design.rb +109 -0
- data/lib/vpsb_client/api/close_trial_request.rb +23 -0
- data/lib/vpsb_client/api/create_trial_request.rb +38 -0
- data/lib/vpsb_client/api/get_current_trial_request.rb +30 -0
- data/lib/vpsb_client/api/get_item_id_request.rb +33 -0
- data/lib/vpsb_client/api/get_trial_last_metric_request.rb +25 -0
- data/lib/vpsb_client/api/post_metric_request.rb +28 -0
- data/lib/vpsb_client/api/request.rb +57 -0
- data/lib/vpsb_client/api/response.rb +30 -0
- data/lib/vpsb_client/builders/system_info_parser.rb +127 -0
- data/lib/vpsb_client/builders/trial.rb +54 -0
- data/lib/vpsb_client/client/upload_metrics.rb +39 -0
- data/lib/vpsb_client/config.rb +18 -0
- data/lib/vpsb_client/curl_wrapper.rb +41 -0
- data/lib/vpsb_client/datafiles/formatted_sar_log_parser.rb +38 -0
- data/lib/vpsb_client/datafiles/logfile_decompressor.rb +48 -0
- data/lib/vpsb_client/datafiles/pxx_aggregator.rb +94 -0
- data/lib/vpsb_client/datafiles/sar_manager.rb +71 -0
- data/lib/vpsb_client/datafiles/timing_log_parser.rb +32 -0
- data/lib/vpsb_client/http_client.rb +60 -0
- data/lib/vpsb_client/manager.rb +149 -0
- data/lib/vpsb_client/metrics/interval_builder.rb +85 -0
- data/lib/vpsb_client/metrics/interval_config.rb +45 -0
- data/lib/vpsb_client/metrics/manager.rb +34 -0
- data/lib/vpsb_client/metrics/uploader.rb +16 -0
- data/lib/vpsb_client/version.rb +3 -0
- data/lib/vpsb_client.rb +33 -0
- data/spec/lib/client/upload_metrics_spec.rb +23 -0
- data/spec/lib/config_spec.rb +26 -0
- data/spec/lib/logfile_decompressor_spec.rb +74 -0
- data/spec/lib/metrics/interval_builder_spec.rb +88 -0
- data/spec/lib/metrics/interval_config_spec.rb +99 -0
- data/spec/lib/metrics/manager_spec.rb +60 -0
- data/spec/lib/metrics/uploader_spec.rb +45 -0
- data/spec/lib/pxx_spec.rb +120 -0
- data/spec/lib/request_spec.rb +186 -0
- data/spec/lib/sar_manager_spec.rb +57 -0
- data/spec/lib/system_info_parser_spec.rb +51 -0
- data/spec/spec_helper.rb +9 -0
- data/spec/support/sarfiles/formatted/formatted_sa20 +147 -0
- data/spec/support/sarfiles/formatted/formatted_sa21 +85 -0
- data/spec/support/sarfiles/orig/sa18 +0 -0
- data/spec/support/sarfiles/orig/sa19 +0 -0
- data/spec/support/sarfiles/orig/sa20 +0 -0
- data/spec/support/sarfiles/orig/sa21 +0 -0
- data/spec/support/timingfiles/other.log.2.gz +0 -0
- data/spec/support/timingfiles/timings.log +11 -0
- data/spec/support/timingfiles/timings.log.1 +11 -0
- data/spec/support/timingfiles/timings.log.2.gz +0 -0
- data/spec/support/timingfiles/timings.log.3.gz +0 -0
- data/spec/support/vpsb.yml +12 -0
- data/vpsb_client.gemspec +29 -0
- metadata +241 -0
@@ -0,0 +1,149 @@
|
|
1
|
+
require 'io/console'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/manager"
|
5
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/client/upload_metrics"
|
6
|
+
|
7
|
+
module VpsbClient
|
8
|
+
class Manager
|
9
|
+
attr_reader :http_client, :logger, :config
|
10
|
+
|
11
|
+
class LastMetricNotFoundError < StandardError; end
|
12
|
+
|
13
|
+
def initialize(config_path, logger=Logger.new(STDOUT))
|
14
|
+
@config_path = config_path
|
15
|
+
@logger = logger
|
16
|
+
end
|
17
|
+
|
18
|
+
def setup
|
19
|
+
VpsbClient.logger = @logger
|
20
|
+
@config = Config.new(@config_path)
|
21
|
+
@curl_wrapper = CurlWrapper.new(@config['auth_token'])
|
22
|
+
@http_client = HttpClient.new(@curl_wrapper, @config['vpsb_protocol'], @config['vpsb_hostname'])
|
23
|
+
end
|
24
|
+
|
25
|
+
def enabled?
|
26
|
+
@config.fetch(:enabled, false)
|
27
|
+
end
|
28
|
+
|
29
|
+
def signed_in?
|
30
|
+
return @signed_in if @signed_in
|
31
|
+
id_request = Api::GetItemIdRequest.new(@http_client, 'hosters', 'linode')
|
32
|
+
curl_response = id_request.run
|
33
|
+
|
34
|
+
begin
|
35
|
+
http_response = Api::Response.new(curl_response)
|
36
|
+
rescue Api::Response::NotAuthenticated => e
|
37
|
+
return @signed_in = false
|
38
|
+
end
|
39
|
+
@signed_in = true
|
40
|
+
end
|
41
|
+
|
42
|
+
def create_trial
|
43
|
+
unless enabled?
|
44
|
+
VpsbClient.logger.debug "not running because vpsb_client is disabled"
|
45
|
+
return
|
46
|
+
end
|
47
|
+
|
48
|
+
builder = Builders::Trial.new(@config, hoster_id, application_id, plan_id)
|
49
|
+
create_trial_request = Api::CreateTrialRequest.new(@http_client, builder.create_params)
|
50
|
+
curl_response = create_trial_request.run
|
51
|
+
http_response = Api::Response.new(curl_response)
|
52
|
+
Api::CreateTrialRequest.trial(http_response)
|
53
|
+
end
|
54
|
+
|
55
|
+
def current_trial
|
56
|
+
builder = Builders::Trial.new(@config)
|
57
|
+
current_trial_request = Api::GetCurrentTrialRequest.new(@http_client, builder.lookup_params)
|
58
|
+
curl_response = current_trial_request.run
|
59
|
+
http_response = Api::Response.new(curl_response)
|
60
|
+
Api::GetCurrentTrialRequest.trial(http_response)
|
61
|
+
end
|
62
|
+
|
63
|
+
def trial_last_metric_started_at(trial_id, length)
|
64
|
+
current_trial_request = Api::GetTrialLastMetricRequest.new(@http_client, { trial_id: trial_id, length: length})
|
65
|
+
curl_response = current_trial_request.run
|
66
|
+
http_response = Api::Response.new(curl_response)
|
67
|
+
Api::GetTrialLastMetricRequest.started_at(http_response)
|
68
|
+
end
|
69
|
+
|
70
|
+
def hoster_id
|
71
|
+
return @hoster_id if @hoster_id
|
72
|
+
id_request = Api::GetItemIdRequest.new(@http_client, 'hosters', @config['hoster_name'])
|
73
|
+
http_response = Api::Response.new(id_request.run)
|
74
|
+
id = Api::GetItemIdRequest.item_id(http_response)
|
75
|
+
raise NameError, "#{@config['hoster_name']} hoster not found" unless id
|
76
|
+
@hoster_id = id
|
77
|
+
end
|
78
|
+
|
79
|
+
def application_id
|
80
|
+
return @application_id if @application_id
|
81
|
+
id_request = Api::GetItemIdRequest.new(@http_client, 'applications', @config['application_name'])
|
82
|
+
http_response = Api::Response.new(id_request.run)
|
83
|
+
id = Api::GetItemIdRequest.item_id(http_response)
|
84
|
+
raise NameError, "#{@config['application_name']} application not found" unless id
|
85
|
+
@application_id = id
|
86
|
+
end
|
87
|
+
|
88
|
+
def plan_id
|
89
|
+
return @plan_id if @plan_id
|
90
|
+
id_request = Api::GetPlanIdRequest.new(@http_client, hoster_id, @config['plan_name'])
|
91
|
+
http_response = Api::Response.new(id_request.run)
|
92
|
+
id = Api::GetPlanIdRequest.item_id(http_response)
|
93
|
+
raise NameError, "#{@config['plan_name']} plan not found" unless id
|
94
|
+
@plan_id = id
|
95
|
+
end
|
96
|
+
|
97
|
+
include Client::UploadMetrics
|
98
|
+
|
99
|
+
def close_trial(trial)
|
100
|
+
unless enabled?
|
101
|
+
logger.debug "[vpsb] not running because vpsb_client is disabled"
|
102
|
+
return
|
103
|
+
end
|
104
|
+
|
105
|
+
prepare_logfiles
|
106
|
+
|
107
|
+
metric_ids = []
|
108
|
+
interval_length = 604800
|
109
|
+
last_started_at = trial_last_metric_started_at(trial['id'], interval_length)
|
110
|
+
if last_started_at
|
111
|
+
start_time = last_started_at + interval_length
|
112
|
+
else
|
113
|
+
logger.debug "[vpsb] close_trial - no last metric found"
|
114
|
+
start_time = Time.now - interval_length
|
115
|
+
end
|
116
|
+
logger.debug "[vpsb] close_trial - length=#{interval_length} start_time=#{start_time} force=false"
|
117
|
+
interval_config = Metrics::IntervalConfig.new(start_time, interval_length, force: true)
|
118
|
+
metrics_manager = metrics_manager(trial['id'], interval_config)
|
119
|
+
metrics_manager.run
|
120
|
+
metric_ids += metrics_manager.created_metric_ids
|
121
|
+
logger.debug "[vpsb] Created metric ids: #{metric_ids.inspect}"
|
122
|
+
|
123
|
+
close_request = Api::CloseTrialRequest.new(@http_client, trial['id'])
|
124
|
+
http_response = Api::Response.new(close_request.run)
|
125
|
+
logger.debug "[vpsb] close request response code = #{http_response.code}"
|
126
|
+
http_response
|
127
|
+
end
|
128
|
+
|
129
|
+
private
|
130
|
+
|
131
|
+
def prepare_logfiles
|
132
|
+
sar_manager = Datafiles::SarManager.new(@config['sar_path'], @config['formatted_sar_path'])
|
133
|
+
sar_manager.run
|
134
|
+
|
135
|
+
logfile_decompressor = Datafiles::LogfileDecompressor.new(@config['timing_path'], @config['timing_path'], :filename_prefix => 'timings')
|
136
|
+
logfile_decompressor.run
|
137
|
+
end
|
138
|
+
|
139
|
+
|
140
|
+
def metrics_manager(trial_id, interval_config)
|
141
|
+
min_start_time = interval_config.min_start_time
|
142
|
+
builder = Metrics::IntervalBuilder.new(@config['formatted_sar_path'], @config['timing_path'], min_start_time, interval_config.length)
|
143
|
+
uploader = Metrics::Uploader.new(@http_client, trial_id)
|
144
|
+
|
145
|
+
Metrics::Manager.new(builder, uploader, interval_config)
|
146
|
+
end
|
147
|
+
|
148
|
+
end
|
149
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require "#{File.expand_path('../..', __FILE__)}/datafiles/formatted_sar_log_parser"
|
2
|
+
require "#{File.expand_path('../..', __FILE__)}/datafiles/timing_log_parser"
|
3
|
+
|
4
|
+
module VpsbClient
|
5
|
+
module Metrics
|
6
|
+
class IntervalBuilder
|
7
|
+
class FileNotFound < StandardError; end
|
8
|
+
|
9
|
+
VALID_METRIC_KEYS = [ :trial_id, :started_at, :duration_seconds, :num_requests, :resptime_total_ms, :resptime_db_ms, :resptime_view_ms, :cpu_idle, :cpu_steal, :iowait, :p50_total_ms, :p75_total_ms, :p95_total_ms, :p99_total_ms, :p75_iowait_pct, :p95_iowait_pct, :p99_iowait_pct,:p75_cpusteal_pct, :p95_cpusteal_pct, :p99_cpusteal_pct, :p75_cpuidle_pct, :p95_cpuidle_pct, :p99_cpuidle_pct ]
|
10
|
+
|
11
|
+
def initialize(sar_path, timing_path, pole_time, interval_length)
|
12
|
+
@sar_path = sar_path
|
13
|
+
@timing_path = timing_path
|
14
|
+
@pole_time = pole_time
|
15
|
+
@interval_length = interval_length
|
16
|
+
end
|
17
|
+
|
18
|
+
def each(&block)
|
19
|
+
return enum_for(:each) unless block_given?
|
20
|
+
|
21
|
+
sar_filenames = Dir.glob("#{@sar_path}/formatted_sa*")
|
22
|
+
timing_filenames = Dir.glob("#{@timing_path}/timings.log*")
|
23
|
+
|
24
|
+
sar_filenames.reject! { |f| f =~ /\.gz$/ }
|
25
|
+
timing_filenames.reject! { |f| f =~ /\.gz$/ }
|
26
|
+
|
27
|
+
raise FileNotFound, "No file matching #{@sar_path}/formatted_sa*" unless sar_filenames.any?
|
28
|
+
raise FileNotFound, "No file matching #{@timing_path}/timings.log*" unless timing_filenames.any?
|
29
|
+
|
30
|
+
builder_options = { offset_by_start_time: @pole_time }
|
31
|
+
|
32
|
+
sar_files = LogfileInterval::LogfileSet.new(sar_filenames, Datafiles::FormattedSarLogParser, :desc)
|
33
|
+
timing_files = LogfileInterval::LogfileSet.new(timing_filenames, Datafiles::TimingLogParser, :desc)
|
34
|
+
|
35
|
+
begin
|
36
|
+
sar_builder = LogfileInterval::IntervalBuilder.new(sar_files, Datafiles::FormattedSarLogParser, @interval_length, builder_options)
|
37
|
+
timing_builder = LogfileInterval::IntervalBuilder.new(timing_files, Datafiles::TimingLogParser, @interval_length, builder_options)
|
38
|
+
|
39
|
+
sar_enum = sar_builder.each_interval
|
40
|
+
timing_enum = timing_builder.each_interval
|
41
|
+
while(timing_interval = timing_enum.next) do
|
42
|
+
while(sar_interval = sar_enum.next) do
|
43
|
+
break if sar_interval.start_time == timing_interval.start_time
|
44
|
+
raise "sar_interval older than timing_interval: #{sar_interval.start_time} > #{timing_interval.start_time}" if sar_interval.start_time > timing_interval.start_time
|
45
|
+
end
|
46
|
+
|
47
|
+
yield convert_to_metric(timing_interval, sar_interval)
|
48
|
+
end
|
49
|
+
rescue StopIteration
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def convert_to_metric(timing_interval, sar_interval)
|
56
|
+
interval = timing_interval.to_hash.merge(sar_interval)
|
57
|
+
interval[:duration_seconds] = timing_interval.length
|
58
|
+
interval[:started_at] = timing_interval.start_time
|
59
|
+
|
60
|
+
pxx_total_ms = timing_interval[:pxx_total_ms]
|
61
|
+
interval[:p50_total_ms] = pxx_total_ms[50]
|
62
|
+
interval[:p75_total_ms] = pxx_total_ms[75]
|
63
|
+
interval[:p95_total_ms] = pxx_total_ms[95]
|
64
|
+
interval[:p99_total_ms] = pxx_total_ms[99]
|
65
|
+
|
66
|
+
pxx_iowait = sar_interval[:pxx_iowait]
|
67
|
+
interval[:p75_iowait_pct] = pxx_iowait[75]
|
68
|
+
interval[:p95_iowait_pct] = pxx_iowait[95]
|
69
|
+
interval[:p99_iowait_pct] = pxx_iowait[99]
|
70
|
+
|
71
|
+
pxx_cpusteal = sar_interval[:pxx_cpusteal]
|
72
|
+
interval[:p75_cpusteal_pct] = pxx_cpusteal[75]
|
73
|
+
interval[:p95_cpusteal_pct] = pxx_cpusteal[95]
|
74
|
+
interval[:p99_cpusteal_pct] = pxx_cpusteal[99]
|
75
|
+
|
76
|
+
pxx_cpuidle = sar_interval[:pxx_cpuidle]
|
77
|
+
interval[:p75_cpuidle_pct] = 100.0 - pxx_cpuidle[75]
|
78
|
+
interval[:p95_cpuidle_pct] = 100.0 - pxx_cpuidle[95]
|
79
|
+
interval[:p99_cpuidle_pct] = 100.0 - pxx_cpuidle[99]
|
80
|
+
|
81
|
+
interval.select! { |k| VALID_METRIC_KEYS.include?(k) }
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module VpsbClient
|
2
|
+
module Metrics
|
3
|
+
class IntervalConfig
|
4
|
+
def initialize(start_time, interval_length, options={})
|
5
|
+
@start_time = start_time
|
6
|
+
@interval_length = interval_length
|
7
|
+
@force = options.fetch(:force, false)
|
8
|
+
end
|
9
|
+
|
10
|
+
def aligned?
|
11
|
+
if @force
|
12
|
+
false
|
13
|
+
elsif @interval_length < 86400
|
14
|
+
true
|
15
|
+
else
|
16
|
+
false
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def min_start_time
|
21
|
+
if @force
|
22
|
+
@start_time
|
23
|
+
elsif aligned?
|
24
|
+
lower_boundary_time(@start_time)
|
25
|
+
else
|
26
|
+
@start_time
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def min_end_time
|
31
|
+
min_start_time + @interval_length
|
32
|
+
end
|
33
|
+
|
34
|
+
def length
|
35
|
+
@interval_length
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def lower_boundary_time(t)
|
41
|
+
Time.at((t.to_i / @interval_length.to_i) * @interval_length.to_i)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/interval_builder"
|
2
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/interval_config"
|
3
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/metrics/uploader"
|
4
|
+
|
5
|
+
module VpsbClient
|
6
|
+
module Metrics
|
7
|
+
class Manager
|
8
|
+
attr_reader :created_metric_ids
|
9
|
+
|
10
|
+
def initialize(builder, uploader, interval_config)
|
11
|
+
@interval_config = interval_config
|
12
|
+
@builder = builder
|
13
|
+
@uploader = uploader
|
14
|
+
@created_metric_ids = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
if @interval_config.min_end_time > Time.now
|
19
|
+
VpsbClient.logger.info "Skipping #{@interval_config.length} because too early (min_end_time=#{@interval_config.min_end_time})"
|
20
|
+
return
|
21
|
+
end
|
22
|
+
|
23
|
+
@builder.each do |metric|
|
24
|
+
VpsbClient.logger.debug "metric[:started_at]=#{metric[:started_at]} @interval_config.min_start_time=#{@interval_config.min_start_time}"
|
25
|
+
if metric[:started_at] < @interval_config.min_start_time - 1
|
26
|
+
VpsbClient.logger.debug "[vpsb] stop builder loop as #{metric[:started_at]} < #{@interval_config.min_start_time}"
|
27
|
+
break
|
28
|
+
end
|
29
|
+
@created_metric_ids << @uploader.upload(metric)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module VpsbClient
|
2
|
+
module Metrics
|
3
|
+
class Uploader
|
4
|
+
def initialize(http_client, trial_id)
|
5
|
+
@http_client = http_client
|
6
|
+
@trial_id = trial_id
|
7
|
+
end
|
8
|
+
|
9
|
+
def upload(metric)
|
10
|
+
upload_request = Api::PostMetricRequest.new(@http_client, @trial_id, metric)
|
11
|
+
http_response = Api::Response.new(upload_request.run)
|
12
|
+
Api::PostMetricRequest.metric_id(http_response)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
data/lib/vpsb_client.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
VPSB_BASE_PATH = File.expand_path('..', __FILE__)
|
2
|
+
|
3
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/version"
|
4
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/config"
|
5
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/http_client"
|
6
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/curl_wrapper"
|
7
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/manager"
|
8
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/request"
|
9
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/response"
|
10
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/get_current_trial_request"
|
11
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/get_trial_last_metric_request"
|
12
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/get_item_id_request"
|
13
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/create_trial_request"
|
14
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/post_metric_request"
|
15
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/api/close_trial_request"
|
16
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/builders/system_info_parser"
|
17
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/builders/trial"
|
18
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/datafiles/sar_manager"
|
19
|
+
require "#{VPSB_BASE_PATH}/vpsb_client/datafiles/logfile_decompressor"
|
20
|
+
|
21
|
+
module VpsbClient
|
22
|
+
def self.logger=(logger)
|
23
|
+
@@logger = logger
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.logger
|
27
|
+
if defined?(@@logger)
|
28
|
+
@@logger
|
29
|
+
else
|
30
|
+
@@logger = Logger.new('/dev/null')
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module VpsbClient
|
4
|
+
module Client
|
5
|
+
describe UploadMetrics do
|
6
|
+
before :each do
|
7
|
+
config = { }
|
8
|
+
@manager = Manager.new('fake_path', Logger.new(STDOUT))
|
9
|
+
allow(@manager).to receive(:prepare_logfiles)
|
10
|
+
allow(@manager).to receive(:enabled?).and_return(true)
|
11
|
+
allow(@manager).to receive(:config).and_return(config)
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'upload metrics for all lengths' do
|
15
|
+
trial = double('trial')
|
16
|
+
expect(@manager).to receive(:upload_for_interval_length).with(trial, 600).and_return([])
|
17
|
+
expect(@manager).to receive(:upload_for_interval_length).with(trial, 3600).and_return([])
|
18
|
+
expect(@manager).to receive(:upload_for_interval_length).with(trial, 86400).and_return([])
|
19
|
+
@manager.upload_metrics(trial)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
#require File.join(File.dirname(__FILE__), '..', 'support/lib/timing_log')
|
3
|
+
|
4
|
+
module VpsbClient
|
5
|
+
data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
|
6
|
+
|
7
|
+
describe Config do
|
8
|
+
before :each do
|
9
|
+
support_dir = File.join(File.dirname(__FILE__), '..', 'support/')
|
10
|
+
@config = Config.new("#{support_dir}/vpsb.yml")
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'parses the config file' do
|
14
|
+
expect(@config.fetch('vpsb_hostname')).to eq('localhost:3000')
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'raises an exception for non existent keys' do
|
18
|
+
expect{@config.fetch('foobar')}.to raise_error
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'understands []' do
|
22
|
+
expect{@config.fetch('vpsb_hostname')}.to_not raise_error
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require "#{File.expand_path('../../..', __FILE__)}/lib/vpsb_client/datafiles/logfile_decompressor"
|
3
|
+
|
4
|
+
module VpsbClient
|
5
|
+
|
6
|
+
module Datafiles
|
7
|
+
data_dir = File.join(File.dirname(__FILE__), '..', 'support/timingfiles')
|
8
|
+
|
9
|
+
describe LogfileDecompressor do
|
10
|
+
before :all do
|
11
|
+
@orig_dir = data_dir
|
12
|
+
@target_dir = data_dir
|
13
|
+
end
|
14
|
+
|
15
|
+
context 'without rotation id limit' do
|
16
|
+
before :each do
|
17
|
+
@manager = LogfileDecompressor.new(@orig_dir, @target_dir, :filename_prefix => 'timings')
|
18
|
+
end
|
19
|
+
|
20
|
+
after :each do
|
21
|
+
File.unlink("#{@target_dir}/timings.log.2")
|
22
|
+
File.unlink("#{@target_dir}/timings.log.3")
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'unzips files with .gz extension' do
|
26
|
+
@manager.run
|
27
|
+
expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
|
28
|
+
expect(File.exist?("#{@target_dir}/timings.log.3")).to be true
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'only unzips files with specified prefix' do
|
32
|
+
@manager.run
|
33
|
+
expect(File.exist?("#{@target_dir}/other.log.2")).to be false
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'with rotation id limit' do
|
38
|
+
before :each do
|
39
|
+
@manager = LogfileDecompressor.new(@orig_dir, @target_dir, :filename_prefix => 'timings',
|
40
|
+
:max_rotation_id => 2)
|
41
|
+
end
|
42
|
+
|
43
|
+
after :each do
|
44
|
+
File.unlink("#{@target_dir}/timings.log.2")
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'unzips only files below limit' do
|
48
|
+
@manager.run
|
49
|
+
expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
|
50
|
+
expect(File.exist?("#{@target_dir}/timings.log.3")).to be false
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'without filename prefix' do
|
55
|
+
before :each do
|
56
|
+
@manager = LogfileDecompressor.new(@orig_dir, @target_dir)
|
57
|
+
end
|
58
|
+
|
59
|
+
after :each do
|
60
|
+
File.unlink("#{@target_dir}/timings.log.2")
|
61
|
+
File.unlink("#{@target_dir}/timings.log.3")
|
62
|
+
File.unlink("#{@target_dir}/other.log.2")
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'unzips all files' do
|
66
|
+
@manager.run
|
67
|
+
expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
|
68
|
+
expect(File.exist?("#{@target_dir}/timings.log.3")).to be true
|
69
|
+
expect(File.exist?("#{@target_dir}/other.log.2")).to be true
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module VpsbClient
|
4
|
+
module Metrics
|
5
|
+
support_dir = File.join(File.dirname(__FILE__), '../..', 'support/')
|
6
|
+
|
7
|
+
OLDEST_TIME_IN_LOGFILES = Time.new(2014,9,20,0,0,0)
|
8
|
+
NEWEST_TIME_IN_LOGFILES = Time.new(2014,9,21,13,56,0)
|
9
|
+
|
10
|
+
describe IntervalBuilder do
|
11
|
+
before :each do
|
12
|
+
@sar_dir = "#{support_dir}/sarfiles/formatted"
|
13
|
+
@timing_dir = "#{support_dir}/timingfiles"
|
14
|
+
@length = 3600
|
15
|
+
allow(Time).to receive(:now).and_return(NEWEST_TIME_IN_LOGFILES + @length)
|
16
|
+
end
|
17
|
+
|
18
|
+
context 'with aligned pole time' do
|
19
|
+
before :each do
|
20
|
+
aligned_pole_time = OLDEST_TIME_IN_LOGFILES - 12*3600
|
21
|
+
@builder = IntervalBuilder.new(@sar_dir, @timing_dir, aligned_pole_time, @length)
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'should yield once per full hour' do
|
25
|
+
expected_num_intervals = (NEWEST_TIME_IN_LOGFILES - OLDEST_TIME_IN_LOGFILES) / @length
|
26
|
+
expect { |b| @builder.each(&b) }.to yield_control.at_least(expected_num_intervals)
|
27
|
+
end
|
28
|
+
|
29
|
+
describe 'first interval' do
|
30
|
+
it 'is the most recent' do
|
31
|
+
interval_enum = @builder.each
|
32
|
+
expected_start_time = lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length)
|
33
|
+
expect(interval_enum.next[:started_at]).to eq(expected_start_time)
|
34
|
+
expect(interval_enum.next[:started_at]).to eq(expected_start_time - @length)
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'has the requested length' do
|
38
|
+
i = @builder.each.first
|
39
|
+
expect(i[:duration_seconds]).to eq(@length)
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'has the right averaged timing' do
|
43
|
+
i = @builder.each.first
|
44
|
+
expect(i[:started_at]).to eq(lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length))
|
45
|
+
expect(i[:resptime_total_ms]).to eq(29.5)
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'as the right p95 cpu idle' do
|
49
|
+
i = @builder.each.first
|
50
|
+
expect(i[:p99_cpuidle_pct]).to eq(88.0)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe 'last interval' do
|
55
|
+
it 'starts at the oldest data point' do
|
56
|
+
last_interval = nil
|
57
|
+
@builder.each do |i|
|
58
|
+
last_interval = i
|
59
|
+
end
|
60
|
+
expect(last_interval[:started_at]).to eq(OLDEST_TIME_IN_LOGFILES)
|
61
|
+
expect(last_interval[:duration_seconds]).to eq(@length)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'with unaligned pole time' do
|
67
|
+
before :each do
|
68
|
+
@offset = 60
|
69
|
+
unaligned_pole_time = OLDEST_TIME_IN_LOGFILES - 12*3600 + @offset
|
70
|
+
@builder = IntervalBuilder.new(@sar_dir, @timing_dir, unaligned_pole_time, @length)
|
71
|
+
end
|
72
|
+
|
73
|
+
describe 'first interval' do
|
74
|
+
it 'start with on offset start_time' do
|
75
|
+
i = @builder.each.first
|
76
|
+
expected_start_time = lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length) + @offset
|
77
|
+
expect(i[:started_at]).to eq(expected_start_time)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def lower_time_boundary(t, len)
|
83
|
+
Time.at((t.to_i / len) * len)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module VpsbClient
|
4
|
+
module Metrics
|
5
|
+
describe IntervalConfig do
|
6
|
+
describe :length do
|
7
|
+
it 'returns the provided length' do
|
8
|
+
ic = IntervalConfig.new(nil, 3600)
|
9
|
+
expect(ic.length).to eq(3600)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe :min_end_time do
|
14
|
+
it 'is min_start_time + length' do
|
15
|
+
t = Time.new(2014, 11, 15, 20, 0, 0)
|
16
|
+
ic = IntervalConfig.new(t, 3600)
|
17
|
+
expect(ic.min_end_time).to eq(ic.min_start_time + 3600)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context 'with force' do
|
22
|
+
before :each do
|
23
|
+
@forced_start_time = Time.new(2014, 11, 15, 20, 0, 0)
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'with short interval length' do
|
27
|
+
before :each do
|
28
|
+
@length = 60
|
29
|
+
@ic = IntervalConfig.new(@forced_start_time, @length, force: true)
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'aligned? is false' do
|
33
|
+
expect(@ic.aligned?).to eq(false)
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'min_start_time is forced_start_time' do
|
37
|
+
expect(@ic.min_start_time).to eq(@forced_start_time)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'with long interval length' do
|
42
|
+
before :each do
|
43
|
+
@length = 604800
|
44
|
+
@ic = IntervalConfig.new(@forced_start_time, @length, force: true)
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'aligned? is false' do
|
48
|
+
expect(@ic.aligned?).to eq(false)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'min_start_time is forced_start_time' do
|
52
|
+
expect(@ic.min_start_time).to eq(@forced_start_time)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
context 'without force' do
|
58
|
+
before :each do
|
59
|
+
@forced_start_time = nil
|
60
|
+
@fallback_start_time = Time.new(2014, 11, 15, 20, 0, 0)
|
61
|
+
end
|
62
|
+
|
63
|
+
context 'with short interval length' do
|
64
|
+
before :each do
|
65
|
+
@length = 60
|
66
|
+
@ic = IntervalConfig.new(@fallback_start_time, @length)
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'aligned? is true' do
|
70
|
+
expect(@ic.aligned?).to eq(true)
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'min_start_time is lower aligned boundary of fallback_start_time' do
|
74
|
+
expect(@ic.min_start_time).to eq(lower_time_boundary(@fallback_start_time, @length))
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
context 'with long interval length' do
|
79
|
+
before :each do
|
80
|
+
@length = 604800
|
81
|
+
@ic = IntervalConfig.new(@fallback_start_time, @length)
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'aligned? is false' do
|
85
|
+
expect(@ic.aligned?).to eq(false)
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'min_start_time is fallback_start_time' do
|
89
|
+
expect(@ic.min_start_time).to eq(@fallback_start_time)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def lower_time_boundary(t, len)
|
95
|
+
Time.at((t.to_i / len) * len)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|