vpsb_client 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +22 -0
  3. data/Gemfile +4 -0
  4. data/LICENSE.txt +22 -0
  5. data/README.md +29 -0
  6. data/Rakefile +2 -0
  7. data/bin/close_trial.rb +20 -0
  8. data/bin/create_trial.rb +13 -0
  9. data/bin/get_current_trial.rb +8 -0
  10. data/bin/get_item_ids.rb +9 -0
  11. data/bin/get_trial_last_metric.rb +8 -0
  12. data/bin/signed_in.rb +7 -0
  13. data/bin/sync_metrics.rb +19 -0
  14. data/bin/test_timing_interval.rb +18 -0
  15. data/bin/upload_metric.rb +37 -0
  16. data/doc/design.rb +109 -0
  17. data/lib/vpsb_client/api/close_trial_request.rb +23 -0
  18. data/lib/vpsb_client/api/create_trial_request.rb +38 -0
  19. data/lib/vpsb_client/api/get_current_trial_request.rb +30 -0
  20. data/lib/vpsb_client/api/get_item_id_request.rb +33 -0
  21. data/lib/vpsb_client/api/get_trial_last_metric_request.rb +25 -0
  22. data/lib/vpsb_client/api/post_metric_request.rb +28 -0
  23. data/lib/vpsb_client/api/request.rb +57 -0
  24. data/lib/vpsb_client/api/response.rb +30 -0
  25. data/lib/vpsb_client/builders/system_info_parser.rb +127 -0
  26. data/lib/vpsb_client/builders/trial.rb +54 -0
  27. data/lib/vpsb_client/client/upload_metrics.rb +39 -0
  28. data/lib/vpsb_client/config.rb +18 -0
  29. data/lib/vpsb_client/curl_wrapper.rb +41 -0
  30. data/lib/vpsb_client/datafiles/formatted_sar_log_parser.rb +38 -0
  31. data/lib/vpsb_client/datafiles/logfile_decompressor.rb +48 -0
  32. data/lib/vpsb_client/datafiles/pxx_aggregator.rb +94 -0
  33. data/lib/vpsb_client/datafiles/sar_manager.rb +71 -0
  34. data/lib/vpsb_client/datafiles/timing_log_parser.rb +32 -0
  35. data/lib/vpsb_client/http_client.rb +60 -0
  36. data/lib/vpsb_client/manager.rb +149 -0
  37. data/lib/vpsb_client/metrics/interval_builder.rb +85 -0
  38. data/lib/vpsb_client/metrics/interval_config.rb +45 -0
  39. data/lib/vpsb_client/metrics/manager.rb +34 -0
  40. data/lib/vpsb_client/metrics/uploader.rb +16 -0
  41. data/lib/vpsb_client/version.rb +3 -0
  42. data/lib/vpsb_client.rb +33 -0
  43. data/spec/lib/client/upload_metrics_spec.rb +23 -0
  44. data/spec/lib/config_spec.rb +26 -0
  45. data/spec/lib/logfile_decompressor_spec.rb +74 -0
  46. data/spec/lib/metrics/interval_builder_spec.rb +88 -0
  47. data/spec/lib/metrics/interval_config_spec.rb +99 -0
  48. data/spec/lib/metrics/manager_spec.rb +60 -0
  49. data/spec/lib/metrics/uploader_spec.rb +45 -0
  50. data/spec/lib/pxx_spec.rb +120 -0
  51. data/spec/lib/request_spec.rb +186 -0
  52. data/spec/lib/sar_manager_spec.rb +57 -0
  53. data/spec/lib/system_info_parser_spec.rb +51 -0
  54. data/spec/spec_helper.rb +9 -0
  55. data/spec/support/sarfiles/formatted/formatted_sa20 +147 -0
  56. data/spec/support/sarfiles/formatted/formatted_sa21 +85 -0
  57. data/spec/support/sarfiles/orig/sa18 +0 -0
  58. data/spec/support/sarfiles/orig/sa19 +0 -0
  59. data/spec/support/sarfiles/orig/sa20 +0 -0
  60. data/spec/support/sarfiles/orig/sa21 +0 -0
  61. data/spec/support/timingfiles/other.log.2.gz +0 -0
  62. data/spec/support/timingfiles/timings.log +11 -0
  63. data/spec/support/timingfiles/timings.log.1 +11 -0
  64. data/spec/support/timingfiles/timings.log.2.gz +0 -0
  65. data/spec/support/timingfiles/timings.log.3.gz +0 -0
  66. data/spec/support/vpsb.yml +12 -0
  67. data/vpsb_client.gemspec +29 -0
  68. metadata +241 -0
@@ -0,0 +1,149 @@
1
+ require 'io/console'
2
+ require 'logger'
3
+
4
+ require "#{VPSB_BASE_PATH}/vpsb_client/metrics/manager"
5
+ require "#{VPSB_BASE_PATH}/vpsb_client/client/upload_metrics"
6
+
7
+ module VpsbClient
8
+ class Manager
9
+ attr_reader :http_client, :logger, :config
10
+
11
+ class LastMetricNotFoundError < StandardError; end
12
+
13
+ def initialize(config_path, logger=Logger.new(STDOUT))
14
+ @config_path = config_path
15
+ @logger = logger
16
+ end
17
+
18
+ def setup
19
+ VpsbClient.logger = @logger
20
+ @config = Config.new(@config_path)
21
+ @curl_wrapper = CurlWrapper.new(@config['auth_token'])
22
+ @http_client = HttpClient.new(@curl_wrapper, @config['vpsb_protocol'], @config['vpsb_hostname'])
23
+ end
24
+
25
+ def enabled?
26
+ @config.fetch(:enabled, false)
27
+ end
28
+
29
+ def signed_in?
30
+ return @signed_in if @signed_in
31
+ id_request = Api::GetItemIdRequest.new(@http_client, 'hosters', 'linode')
32
+ curl_response = id_request.run
33
+
34
+ begin
35
+ http_response = Api::Response.new(curl_response)
36
+ rescue Api::Response::NotAuthenticated => e
37
+ return @signed_in = false
38
+ end
39
+ @signed_in = true
40
+ end
41
+
42
+ def create_trial
43
+ unless enabled?
44
+ VpsbClient.logger.debug "not running because vpsb_client is disabled"
45
+ return
46
+ end
47
+
48
+ builder = Builders::Trial.new(@config, hoster_id, application_id, plan_id)
49
+ create_trial_request = Api::CreateTrialRequest.new(@http_client, builder.create_params)
50
+ curl_response = create_trial_request.run
51
+ http_response = Api::Response.new(curl_response)
52
+ Api::CreateTrialRequest.trial(http_response)
53
+ end
54
+
55
+ def current_trial
56
+ builder = Builders::Trial.new(@config)
57
+ current_trial_request = Api::GetCurrentTrialRequest.new(@http_client, builder.lookup_params)
58
+ curl_response = current_trial_request.run
59
+ http_response = Api::Response.new(curl_response)
60
+ Api::GetCurrentTrialRequest.trial(http_response)
61
+ end
62
+
63
+ def trial_last_metric_started_at(trial_id, length)
64
+ current_trial_request = Api::GetTrialLastMetricRequest.new(@http_client, { trial_id: trial_id, length: length})
65
+ curl_response = current_trial_request.run
66
+ http_response = Api::Response.new(curl_response)
67
+ Api::GetTrialLastMetricRequest.started_at(http_response)
68
+ end
69
+
70
+ def hoster_id
71
+ return @hoster_id if @hoster_id
72
+ id_request = Api::GetItemIdRequest.new(@http_client, 'hosters', @config['hoster_name'])
73
+ http_response = Api::Response.new(id_request.run)
74
+ id = Api::GetItemIdRequest.item_id(http_response)
75
+ raise NameError, "#{@config['hoster_name']} hoster not found" unless id
76
+ @hoster_id = id
77
+ end
78
+
79
+ def application_id
80
+ return @application_id if @application_id
81
+ id_request = Api::GetItemIdRequest.new(@http_client, 'applications', @config['application_name'])
82
+ http_response = Api::Response.new(id_request.run)
83
+ id = Api::GetItemIdRequest.item_id(http_response)
84
+ raise NameError, "#{@config['application_name']} application not found" unless id
85
+ @application_id = id
86
+ end
87
+
88
+ def plan_id
89
+ return @plan_id if @plan_id
90
+ id_request = Api::GetPlanIdRequest.new(@http_client, hoster_id, @config['plan_name'])
91
+ http_response = Api::Response.new(id_request.run)
92
+ id = Api::GetPlanIdRequest.item_id(http_response)
93
+ raise NameError, "#{@config['plan_name']} plan not found" unless id
94
+ @plan_id = id
95
+ end
96
+
97
+ include Client::UploadMetrics
98
+
99
+ def close_trial(trial)
100
+ unless enabled?
101
+ logger.debug "[vpsb] not running because vpsb_client is disabled"
102
+ return
103
+ end
104
+
105
+ prepare_logfiles
106
+
107
+ metric_ids = []
108
+ interval_length = 604800
109
+ last_started_at = trial_last_metric_started_at(trial['id'], interval_length)
110
+ if last_started_at
111
+ start_time = last_started_at + interval_length
112
+ else
113
+ logger.debug "[vpsb] close_trial - no last metric found"
114
+ start_time = Time.now - interval_length
115
+ end
116
+ logger.debug "[vpsb] close_trial - length=#{interval_length} start_time=#{start_time} force=false"
117
+ interval_config = Metrics::IntervalConfig.new(start_time, interval_length, force: true)
118
+ metrics_manager = metrics_manager(trial['id'], interval_config)
119
+ metrics_manager.run
120
+ metric_ids += metrics_manager.created_metric_ids
121
+ logger.debug "[vpsb] Created metric ids: #{metric_ids.inspect}"
122
+
123
+ close_request = Api::CloseTrialRequest.new(@http_client, trial['id'])
124
+ http_response = Api::Response.new(close_request.run)
125
+ logger.debug "[vpsb] close request response code = #{http_response.code}"
126
+ http_response
127
+ end
128
+
129
+ private
130
+
131
+ def prepare_logfiles
132
+ sar_manager = Datafiles::SarManager.new(@config['sar_path'], @config['formatted_sar_path'])
133
+ sar_manager.run
134
+
135
+ logfile_decompressor = Datafiles::LogfileDecompressor.new(@config['timing_path'], @config['timing_path'], :filename_prefix => 'timings')
136
+ logfile_decompressor.run
137
+ end
138
+
139
+
140
+ def metrics_manager(trial_id, interval_config)
141
+ min_start_time = interval_config.min_start_time
142
+ builder = Metrics::IntervalBuilder.new(@config['formatted_sar_path'], @config['timing_path'], min_start_time, interval_config.length)
143
+ uploader = Metrics::Uploader.new(@http_client, trial_id)
144
+
145
+ Metrics::Manager.new(builder, uploader, interval_config)
146
+ end
147
+
148
+ end
149
+ end
@@ -0,0 +1,85 @@
1
+ require "#{File.expand_path('../..', __FILE__)}/datafiles/formatted_sar_log_parser"
2
+ require "#{File.expand_path('../..', __FILE__)}/datafiles/timing_log_parser"
3
+
4
+ module VpsbClient
5
+ module Metrics
6
+ class IntervalBuilder
7
+ class FileNotFound < StandardError; end
8
+
9
+ VALID_METRIC_KEYS = [ :trial_id, :started_at, :duration_seconds, :num_requests, :resptime_total_ms, :resptime_db_ms, :resptime_view_ms, :cpu_idle, :cpu_steal, :iowait, :p50_total_ms, :p75_total_ms, :p95_total_ms, :p99_total_ms, :p75_iowait_pct, :p95_iowait_pct, :p99_iowait_pct,:p75_cpusteal_pct, :p95_cpusteal_pct, :p99_cpusteal_pct, :p75_cpuidle_pct, :p95_cpuidle_pct, :p99_cpuidle_pct ]
10
+
11
+ def initialize(sar_path, timing_path, pole_time, interval_length)
12
+ @sar_path = sar_path
13
+ @timing_path = timing_path
14
+ @pole_time = pole_time
15
+ @interval_length = interval_length
16
+ end
17
+
18
+ def each(&block)
19
+ return enum_for(:each) unless block_given?
20
+
21
+ sar_filenames = Dir.glob("#{@sar_path}/formatted_sa*")
22
+ timing_filenames = Dir.glob("#{@timing_path}/timings.log*")
23
+
24
+ sar_filenames.reject! { |f| f =~ /\.gz$/ }
25
+ timing_filenames.reject! { |f| f =~ /\.gz$/ }
26
+
27
+ raise FileNotFound, "No file matching #{@sar_path}/formatted_sa*" unless sar_filenames.any?
28
+ raise FileNotFound, "No file matching #{@timing_path}/timings.log*" unless timing_filenames.any?
29
+
30
+ builder_options = { offset_by_start_time: @pole_time }
31
+
32
+ sar_files = LogfileInterval::LogfileSet.new(sar_filenames, Datafiles::FormattedSarLogParser, :desc)
33
+ timing_files = LogfileInterval::LogfileSet.new(timing_filenames, Datafiles::TimingLogParser, :desc)
34
+
35
+ begin
36
+ sar_builder = LogfileInterval::IntervalBuilder.new(sar_files, Datafiles::FormattedSarLogParser, @interval_length, builder_options)
37
+ timing_builder = LogfileInterval::IntervalBuilder.new(timing_files, Datafiles::TimingLogParser, @interval_length, builder_options)
38
+
39
+ sar_enum = sar_builder.each_interval
40
+ timing_enum = timing_builder.each_interval
41
+ while(timing_interval = timing_enum.next) do
42
+ while(sar_interval = sar_enum.next) do
43
+ break if sar_interval.start_time == timing_interval.start_time
44
+ raise "sar_interval older than timing_interval: #{sar_interval.start_time} > #{timing_interval.start_time}" if sar_interval.start_time > timing_interval.start_time
45
+ end
46
+
47
+ yield convert_to_metric(timing_interval, sar_interval)
48
+ end
49
+ rescue StopIteration
50
+ end
51
+ end
52
+
53
+ private
54
+
55
+ def convert_to_metric(timing_interval, sar_interval)
56
+ interval = timing_interval.to_hash.merge(sar_interval)
57
+ interval[:duration_seconds] = timing_interval.length
58
+ interval[:started_at] = timing_interval.start_time
59
+
60
+ pxx_total_ms = timing_interval[:pxx_total_ms]
61
+ interval[:p50_total_ms] = pxx_total_ms[50]
62
+ interval[:p75_total_ms] = pxx_total_ms[75]
63
+ interval[:p95_total_ms] = pxx_total_ms[95]
64
+ interval[:p99_total_ms] = pxx_total_ms[99]
65
+
66
+ pxx_iowait = sar_interval[:pxx_iowait]
67
+ interval[:p75_iowait_pct] = pxx_iowait[75]
68
+ interval[:p95_iowait_pct] = pxx_iowait[95]
69
+ interval[:p99_iowait_pct] = pxx_iowait[99]
70
+
71
+ pxx_cpusteal = sar_interval[:pxx_cpusteal]
72
+ interval[:p75_cpusteal_pct] = pxx_cpusteal[75]
73
+ interval[:p95_cpusteal_pct] = pxx_cpusteal[95]
74
+ interval[:p99_cpusteal_pct] = pxx_cpusteal[99]
75
+
76
+ pxx_cpuidle = sar_interval[:pxx_cpuidle]
77
+ interval[:p75_cpuidle_pct] = 100.0 - pxx_cpuidle[75]
78
+ interval[:p95_cpuidle_pct] = 100.0 - pxx_cpuidle[95]
79
+ interval[:p99_cpuidle_pct] = 100.0 - pxx_cpuidle[99]
80
+
81
+ interval.select! { |k| VALID_METRIC_KEYS.include?(k) }
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,45 @@
1
+ module VpsbClient
2
+ module Metrics
3
+ class IntervalConfig
4
+ def initialize(start_time, interval_length, options={})
5
+ @start_time = start_time
6
+ @interval_length = interval_length
7
+ @force = options.fetch(:force, false)
8
+ end
9
+
10
+ def aligned?
11
+ if @force
12
+ false
13
+ elsif @interval_length < 86400
14
+ true
15
+ else
16
+ false
17
+ end
18
+ end
19
+
20
+ def min_start_time
21
+ if @force
22
+ @start_time
23
+ elsif aligned?
24
+ lower_boundary_time(@start_time)
25
+ else
26
+ @start_time
27
+ end
28
+ end
29
+
30
+ def min_end_time
31
+ min_start_time + @interval_length
32
+ end
33
+
34
+ def length
35
+ @interval_length
36
+ end
37
+
38
+ private
39
+
40
+ def lower_boundary_time(t)
41
+ Time.at((t.to_i / @interval_length.to_i) * @interval_length.to_i)
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,34 @@
1
+ require "#{VPSB_BASE_PATH}/vpsb_client/metrics/interval_builder"
2
+ require "#{VPSB_BASE_PATH}/vpsb_client/metrics/interval_config"
3
+ require "#{VPSB_BASE_PATH}/vpsb_client/metrics/uploader"
4
+
5
+ module VpsbClient
6
+ module Metrics
7
+ class Manager
8
+ attr_reader :created_metric_ids
9
+
10
+ def initialize(builder, uploader, interval_config)
11
+ @interval_config = interval_config
12
+ @builder = builder
13
+ @uploader = uploader
14
+ @created_metric_ids = []
15
+ end
16
+
17
+ def run
18
+ if @interval_config.min_end_time > Time.now
19
+ VpsbClient.logger.info "Skipping #{@interval_config.length} because too early (min_end_time=#{@interval_config.min_end_time})"
20
+ return
21
+ end
22
+
23
+ @builder.each do |metric|
24
+ VpsbClient.logger.debug "metric[:started_at]=#{metric[:started_at]} @interval_config.min_start_time=#{@interval_config.min_start_time}"
25
+ if metric[:started_at] < @interval_config.min_start_time - 1
26
+ VpsbClient.logger.debug "[vpsb] stop builder loop as #{metric[:started_at]} < #{@interval_config.min_start_time}"
27
+ break
28
+ end
29
+ @created_metric_ids << @uploader.upload(metric)
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,16 @@
1
+ module VpsbClient
2
+ module Metrics
3
+ class Uploader
4
+ def initialize(http_client, trial_id)
5
+ @http_client = http_client
6
+ @trial_id = trial_id
7
+ end
8
+
9
+ def upload(metric)
10
+ upload_request = Api::PostMetricRequest.new(@http_client, @trial_id, metric)
11
+ http_response = Api::Response.new(upload_request.run)
12
+ Api::PostMetricRequest.metric_id(http_response)
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,3 @@
1
+ module VpsbClient
2
+ VERSION = "0.0.2"
3
+ end
@@ -0,0 +1,33 @@
1
+ VPSB_BASE_PATH = File.expand_path('..', __FILE__)
2
+
3
+ require "#{VPSB_BASE_PATH}/vpsb_client/version"
4
+ require "#{VPSB_BASE_PATH}/vpsb_client/config"
5
+ require "#{VPSB_BASE_PATH}/vpsb_client/http_client"
6
+ require "#{VPSB_BASE_PATH}/vpsb_client/curl_wrapper"
7
+ require "#{VPSB_BASE_PATH}/vpsb_client/manager"
8
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/request"
9
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/response"
10
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/get_current_trial_request"
11
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/get_trial_last_metric_request"
12
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/get_item_id_request"
13
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/create_trial_request"
14
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/post_metric_request"
15
+ require "#{VPSB_BASE_PATH}/vpsb_client/api/close_trial_request"
16
+ require "#{VPSB_BASE_PATH}/vpsb_client/builders/system_info_parser"
17
+ require "#{VPSB_BASE_PATH}/vpsb_client/builders/trial"
18
+ require "#{VPSB_BASE_PATH}/vpsb_client/datafiles/sar_manager"
19
+ require "#{VPSB_BASE_PATH}/vpsb_client/datafiles/logfile_decompressor"
20
+
21
+ module VpsbClient
22
+ def self.logger=(logger)
23
+ @@logger = logger
24
+ end
25
+
26
+ def self.logger
27
+ if defined?(@@logger)
28
+ @@logger
29
+ else
30
+ @@logger = Logger.new('/dev/null')
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,23 @@
1
+ require 'spec_helper'
2
+
3
+ module VpsbClient
4
+ module Client
5
+ describe UploadMetrics do
6
+ before :each do
7
+ config = { }
8
+ @manager = Manager.new('fake_path', Logger.new(STDOUT))
9
+ allow(@manager).to receive(:prepare_logfiles)
10
+ allow(@manager).to receive(:enabled?).and_return(true)
11
+ allow(@manager).to receive(:config).and_return(config)
12
+ end
13
+
14
+ it 'upload metrics for all lengths' do
15
+ trial = double('trial')
16
+ expect(@manager).to receive(:upload_for_interval_length).with(trial, 600).and_return([])
17
+ expect(@manager).to receive(:upload_for_interval_length).with(trial, 3600).and_return([])
18
+ expect(@manager).to receive(:upload_for_interval_length).with(trial, 86400).and_return([])
19
+ @manager.upload_metrics(trial)
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+ #require File.join(File.dirname(__FILE__), '..', 'support/lib/timing_log')
3
+
4
+ module VpsbClient
5
+ data_dir = File.join(File.dirname(__FILE__), '..', 'support/logfiles')
6
+
7
+ describe Config do
8
+ before :each do
9
+ support_dir = File.join(File.dirname(__FILE__), '..', 'support/')
10
+ @config = Config.new("#{support_dir}/vpsb.yml")
11
+ end
12
+
13
+ it 'parses the config file' do
14
+ expect(@config.fetch('vpsb_hostname')).to eq('localhost:3000')
15
+ end
16
+
17
+ it 'raises an exception for non existent keys' do
18
+ expect{@config.fetch('foobar')}.to raise_error
19
+ end
20
+
21
+ it 'understands []' do
22
+ expect{@config.fetch('vpsb_hostname')}.to_not raise_error
23
+ end
24
+
25
+ end
26
+ end
@@ -0,0 +1,74 @@
1
+ require 'spec_helper'
2
+ require "#{File.expand_path('../../..', __FILE__)}/lib/vpsb_client/datafiles/logfile_decompressor"
3
+
4
+ module VpsbClient
5
+
6
+ module Datafiles
7
+ data_dir = File.join(File.dirname(__FILE__), '..', 'support/timingfiles')
8
+
9
+ describe LogfileDecompressor do
10
+ before :all do
11
+ @orig_dir = data_dir
12
+ @target_dir = data_dir
13
+ end
14
+
15
+ context 'without rotation id limit' do
16
+ before :each do
17
+ @manager = LogfileDecompressor.new(@orig_dir, @target_dir, :filename_prefix => 'timings')
18
+ end
19
+
20
+ after :each do
21
+ File.unlink("#{@target_dir}/timings.log.2")
22
+ File.unlink("#{@target_dir}/timings.log.3")
23
+ end
24
+
25
+ it 'unzips files with .gz extension' do
26
+ @manager.run
27
+ expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
28
+ expect(File.exist?("#{@target_dir}/timings.log.3")).to be true
29
+ end
30
+
31
+ it 'only unzips files with specified prefix' do
32
+ @manager.run
33
+ expect(File.exist?("#{@target_dir}/other.log.2")).to be false
34
+ end
35
+ end
36
+
37
+ context 'with rotation id limit' do
38
+ before :each do
39
+ @manager = LogfileDecompressor.new(@orig_dir, @target_dir, :filename_prefix => 'timings',
40
+ :max_rotation_id => 2)
41
+ end
42
+
43
+ after :each do
44
+ File.unlink("#{@target_dir}/timings.log.2")
45
+ end
46
+
47
+ it 'unzips only files below limit' do
48
+ @manager.run
49
+ expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
50
+ expect(File.exist?("#{@target_dir}/timings.log.3")).to be false
51
+ end
52
+ end
53
+
54
+ context 'without filename prefix' do
55
+ before :each do
56
+ @manager = LogfileDecompressor.new(@orig_dir, @target_dir)
57
+ end
58
+
59
+ after :each do
60
+ File.unlink("#{@target_dir}/timings.log.2")
61
+ File.unlink("#{@target_dir}/timings.log.3")
62
+ File.unlink("#{@target_dir}/other.log.2")
63
+ end
64
+
65
+ it 'unzips all files' do
66
+ @manager.run
67
+ expect(File.exist?("#{@target_dir}/timings.log.2")).to be true
68
+ expect(File.exist?("#{@target_dir}/timings.log.3")).to be true
69
+ expect(File.exist?("#{@target_dir}/other.log.2")).to be true
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,88 @@
1
+ require 'spec_helper'
2
+
3
+ module VpsbClient
4
+ module Metrics
5
+ support_dir = File.join(File.dirname(__FILE__), '../..', 'support/')
6
+
7
+ OLDEST_TIME_IN_LOGFILES = Time.new(2014,9,20,0,0,0)
8
+ NEWEST_TIME_IN_LOGFILES = Time.new(2014,9,21,13,56,0)
9
+
10
+ describe IntervalBuilder do
11
+ before :each do
12
+ @sar_dir = "#{support_dir}/sarfiles/formatted"
13
+ @timing_dir = "#{support_dir}/timingfiles"
14
+ @length = 3600
15
+ allow(Time).to receive(:now).and_return(NEWEST_TIME_IN_LOGFILES + @length)
16
+ end
17
+
18
+ context 'with aligned pole time' do
19
+ before :each do
20
+ aligned_pole_time = OLDEST_TIME_IN_LOGFILES - 12*3600
21
+ @builder = IntervalBuilder.new(@sar_dir, @timing_dir, aligned_pole_time, @length)
22
+ end
23
+
24
+ it 'should yield once per full hour' do
25
+ expected_num_intervals = (NEWEST_TIME_IN_LOGFILES - OLDEST_TIME_IN_LOGFILES) / @length
26
+ expect { |b| @builder.each(&b) }.to yield_control.at_least(expected_num_intervals)
27
+ end
28
+
29
+ describe 'first interval' do
30
+ it 'is the most recent' do
31
+ interval_enum = @builder.each
32
+ expected_start_time = lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length)
33
+ expect(interval_enum.next[:started_at]).to eq(expected_start_time)
34
+ expect(interval_enum.next[:started_at]).to eq(expected_start_time - @length)
35
+ end
36
+
37
+ it 'has the requested length' do
38
+ i = @builder.each.first
39
+ expect(i[:duration_seconds]).to eq(@length)
40
+ end
41
+
42
+ it 'has the right averaged timing' do
43
+ i = @builder.each.first
44
+ expect(i[:started_at]).to eq(lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length))
45
+ expect(i[:resptime_total_ms]).to eq(29.5)
46
+ end
47
+
48
+ it 'as the right p95 cpu idle' do
49
+ i = @builder.each.first
50
+ expect(i[:p99_cpuidle_pct]).to eq(88.0)
51
+ end
52
+ end
53
+
54
+ describe 'last interval' do
55
+ it 'starts at the oldest data point' do
56
+ last_interval = nil
57
+ @builder.each do |i|
58
+ last_interval = i
59
+ end
60
+ expect(last_interval[:started_at]).to eq(OLDEST_TIME_IN_LOGFILES)
61
+ expect(last_interval[:duration_seconds]).to eq(@length)
62
+ end
63
+ end
64
+ end
65
+
66
+ context 'with unaligned pole time' do
67
+ before :each do
68
+ @offset = 60
69
+ unaligned_pole_time = OLDEST_TIME_IN_LOGFILES - 12*3600 + @offset
70
+ @builder = IntervalBuilder.new(@sar_dir, @timing_dir, unaligned_pole_time, @length)
71
+ end
72
+
73
+ describe 'first interval' do
74
+ it 'start with on offset start_time' do
75
+ i = @builder.each.first
76
+ expected_start_time = lower_time_boundary(NEWEST_TIME_IN_LOGFILES, @length) + @offset
77
+ expect(i[:started_at]).to eq(expected_start_time)
78
+ end
79
+ end
80
+ end
81
+
82
+ def lower_time_boundary(t, len)
83
+ Time.at((t.to_i / len) * len)
84
+ end
85
+ end
86
+ end
87
+ end
88
+
@@ -0,0 +1,99 @@
1
+ require 'spec_helper'
2
+
3
+ module VpsbClient
4
+ module Metrics
5
+ describe IntervalConfig do
6
+ describe :length do
7
+ it 'returns the provided length' do
8
+ ic = IntervalConfig.new(nil, 3600)
9
+ expect(ic.length).to eq(3600)
10
+ end
11
+ end
12
+
13
+ describe :min_end_time do
14
+ it 'is min_start_time + length' do
15
+ t = Time.new(2014, 11, 15, 20, 0, 0)
16
+ ic = IntervalConfig.new(t, 3600)
17
+ expect(ic.min_end_time).to eq(ic.min_start_time + 3600)
18
+ end
19
+ end
20
+
21
+ context 'with force' do
22
+ before :each do
23
+ @forced_start_time = Time.new(2014, 11, 15, 20, 0, 0)
24
+ end
25
+
26
+ context 'with short interval length' do
27
+ before :each do
28
+ @length = 60
29
+ @ic = IntervalConfig.new(@forced_start_time, @length, force: true)
30
+ end
31
+
32
+ it 'aligned? is false' do
33
+ expect(@ic.aligned?).to eq(false)
34
+ end
35
+
36
+ it 'min_start_time is forced_start_time' do
37
+ expect(@ic.min_start_time).to eq(@forced_start_time)
38
+ end
39
+ end
40
+
41
+ context 'with long interval length' do
42
+ before :each do
43
+ @length = 604800
44
+ @ic = IntervalConfig.new(@forced_start_time, @length, force: true)
45
+ end
46
+
47
+ it 'aligned? is false' do
48
+ expect(@ic.aligned?).to eq(false)
49
+ end
50
+
51
+ it 'min_start_time is forced_start_time' do
52
+ expect(@ic.min_start_time).to eq(@forced_start_time)
53
+ end
54
+ end
55
+ end
56
+
57
+ context 'without force' do
58
+ before :each do
59
+ @forced_start_time = nil
60
+ @fallback_start_time = Time.new(2014, 11, 15, 20, 0, 0)
61
+ end
62
+
63
+ context 'with short interval length' do
64
+ before :each do
65
+ @length = 60
66
+ @ic = IntervalConfig.new(@fallback_start_time, @length)
67
+ end
68
+
69
+ it 'aligned? is true' do
70
+ expect(@ic.aligned?).to eq(true)
71
+ end
72
+
73
+ it 'min_start_time is lower aligned boundary of fallback_start_time' do
74
+ expect(@ic.min_start_time).to eq(lower_time_boundary(@fallback_start_time, @length))
75
+ end
76
+ end
77
+
78
+ context 'with long interval length' do
79
+ before :each do
80
+ @length = 604800
81
+ @ic = IntervalConfig.new(@fallback_start_time, @length)
82
+ end
83
+
84
+ it 'aligned? is false' do
85
+ expect(@ic.aligned?).to eq(false)
86
+ end
87
+
88
+ it 'min_start_time is fallback_start_time' do
89
+ expect(@ic.min_start_time).to eq(@fallback_start_time)
90
+ end
91
+ end
92
+ end
93
+
94
+ def lower_time_boundary(t, len)
95
+ Time.at((t.to_i / len) * len)
96
+ end
97
+ end
98
+ end
99
+ end