oneacct-export 0.2.7 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rspec +1 -0
- data/.travis.yml +1 -21
- data/bin/oneacct-export +7 -6
- data/config/conf.yml +25 -7
- data/lib/data_validators/apel_data_validator.rb +99 -0
- data/lib/data_validators/data_compute.rb +57 -0
- data/lib/data_validators/data_validator.rb +12 -0
- data/lib/data_validators/data_validator_helper.rb +15 -0
- data/lib/data_validators/logstash_data_validator.rb +82 -0
- data/lib/data_validators/pbs_data_validator.rb +86 -0
- data/lib/errors/not_implemented_error.rb +3 -0
- data/lib/errors/validation_error.rb +3 -0
- data/lib/errors.rb +2 -0
- data/lib/input_validator.rb +12 -2
- data/lib/one_data_accessor.rb +11 -10
- data/lib/one_worker.rb +109 -137
- data/lib/oneacct_exporter/version.rb +1 -1
- data/lib/oneacct_exporter.rb +9 -7
- data/lib/oneacct_opts.rb +36 -13
- data/lib/output_types.rb +5 -0
- data/lib/redis_conf.rb +2 -2
- data/lib/settings.rb +3 -3
- data/lib/templates/apel-0.2.erb +6 -6
- data/lib/templates/logstash-0.1.erb +3 -0
- data/lib/templates/pbs-0.1.erb +6 -0
- data/mock/{one_worker_vm8.xml → one_worker_vm_dn01.xml} +76 -74
- data/mock/one_worker_vm_dn02.xml +174 -0
- data/mock/{one_worker_DISK_missing.xml → one_worker_vm_empty_disk_records.xml} +10 -6
- data/mock/one_worker_vm_empty_history_records.xml +131 -0
- data/mock/one_worker_vm_image_name01.xml +175 -0
- data/mock/{one_worker_valid_machine.xml → one_worker_vm_image_name02.xml} +35 -7
- data/mock/one_worker_vm_image_name03.xml +167 -0
- data/mock/{one_worker_vm2.xml → one_worker_vm_image_name04.xml} +38 -9
- data/mock/{one_worker_vm1.xml → one_worker_vm_image_name05.xml} +36 -8
- data/mock/{one_worker_vm9.xml → one_worker_vm_image_name06.xml} +8 -5
- data/oneacct-export.gemspec +1 -0
- data/spec/data_validators/apel_data_validator_spec.rb +497 -0
- data/spec/data_validators/data_compute_spec.rb +193 -0
- data/spec/data_validators/data_validator_helper_spec.rb +66 -0
- data/spec/data_validators/data_validator_spec.rb +14 -0
- data/spec/data_validators/logstash_data_validator_spec.rb +469 -0
- data/spec/data_validators/pbs_data_validator_spec.rb +353 -0
- data/spec/one_worker_spec.rb +234 -542
- data/spec/oneacct_exporter_spec.rb +1 -41
- data/spec/oneacct_opts_spec.rb +135 -32
- data/spec/spec_helper.rb +18 -1
- metadata +51 -52
- data/mock/one_worker_DEPLOY_ID_missing.xml +0 -136
- data/mock/one_worker_DISK_SIZE_nan.xml +0 -147
- data/mock/one_worker_ETIME_0.xml +0 -137
- data/mock/one_worker_ETIME_missing.xml +0 -136
- data/mock/one_worker_ETIME_nan.xml +0 -137
- data/mock/one_worker_GID_missing.xml +0 -136
- data/mock/one_worker_GNAME_missing.xml +0 -136
- data/mock/one_worker_HISTORY_RECORDS_missing.xml +0 -91
- data/mock/one_worker_HISTORY_many.xml +0 -137
- data/mock/one_worker_HISTORY_missing.xml +0 -93
- data/mock/one_worker_HISTORY_one.xml +0 -115
- data/mock/one_worker_IMAGE_ID_missing.xml +0 -136
- data/mock/one_worker_MEMORY_0.xml +0 -137
- data/mock/one_worker_MEMORY_missing.xml +0 -135
- data/mock/one_worker_MEMORY_nan.xml +0 -137
- data/mock/one_worker_NET_RX_0.xml +0 -137
- data/mock/one_worker_NET_RX_missing.xml +0 -136
- data/mock/one_worker_NET_RX_nan.xml +0 -137
- data/mock/one_worker_NET_TX_0.xml +0 -137
- data/mock/one_worker_NET_TX_missing.xml +0 -136
- data/mock/one_worker_NET_TX_nan.xml +0 -137
- data/mock/one_worker_RETIME_0_RUNNING.xml +0 -115
- data/mock/one_worker_RETIME_0_STOPPED.xml +0 -115
- data/mock/one_worker_RETIME_missing.xml +0 -114
- data/mock/one_worker_RSTIME_0.xml +0 -115
- data/mock/one_worker_RSTIME_>_RETIME.xml +0 -115
- data/mock/one_worker_RSTIME_missing.xml +0 -114
- data/mock/one_worker_STATE_missing.xml +0 -136
- data/mock/one_worker_STATE_out_of_range.xml +0 -137
- data/mock/one_worker_STIME_>_ETIME.xml +0 -137
- data/mock/one_worker_STIME_missing.xml +0 -136
- data/mock/one_worker_STIME_nan.xml +0 -137
- data/mock/one_worker_TEMPLATE_missing.xml +0 -79
- data/mock/one_worker_UID_missing.xml +0 -136
- data/mock/one_worker_VCPU_0.xml +0 -137
- data/mock/one_worker_VCPU_missing.xml +0 -136
- data/mock/one_worker_VCPU_nan.xml +0 -137
- data/mock/one_worker_malformed_vm.xml +0 -136
- data/mock/one_worker_vm3.xml +0 -137
- data/mock/one_worker_vm4.xml +0 -106
- data/mock/one_worker_vm5.xml +0 -106
- data/mock/one_worker_vm6.xml +0 -107
- data/mock/one_worker_vm7.xml +0 -147
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e33fa6806cd2ea1a65ac02313c0a3aa3cf549095
|
4
|
+
data.tar.gz: 3691a9b9a6eea6c22987c3f78934c199dc7e6596
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0b049a0279a7cbdb37b002faa9c554596201aee6965a13789f0b234657421f4e32ce03add5708458540cfc37d80307a922837acf10028ecb60e311bdb98da60a
|
7
|
+
data.tar.gz: bf8ad5f4cce2931091cfcd5851780b2cbf6c42eb0f5d7e949135147387bb657653fec11cc7b8e0bfcb76fe5af69773aef608b18ece6ab873d438d9ae2978d685
|
data/.gitignore
CHANGED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper --color --format Fuubar
|
data/.travis.yml
CHANGED
@@ -3,32 +3,12 @@ language: ruby
|
|
3
3
|
rvm:
|
4
4
|
- 2.0.0
|
5
5
|
- 2.1
|
6
|
+
- 2.2
|
6
7
|
- ruby-head
|
7
|
-
- jruby-19mode
|
8
|
-
- jruby-head
|
9
|
-
|
10
|
-
jdk:
|
11
|
-
- openjdk7
|
12
|
-
- oraclejdk7
|
13
|
-
- openjdk6
|
14
8
|
|
15
9
|
matrix:
|
16
10
|
allow_failures:
|
17
11
|
- rvm: ruby-head
|
18
|
-
- rvm: jruby-head
|
19
|
-
exclude:
|
20
|
-
- rvm: 2.0.0
|
21
|
-
jdk: openjdk7
|
22
|
-
- rvm: 2.0.0
|
23
|
-
jdk: oraclejdk7
|
24
|
-
- rvm: 2.1
|
25
|
-
jdk: openjdk7
|
26
|
-
- rvm: 2.1
|
27
|
-
jdk: oraclejdk7
|
28
|
-
- rvm: ruby-head
|
29
|
-
jdk: openjdk7
|
30
|
-
- rvm: ruby-head
|
31
|
-
jdk: oraclejdk7
|
32
12
|
fast_finish: true
|
33
13
|
|
34
14
|
branches:
|
data/bin/oneacct-export
CHANGED
@@ -6,15 +6,16 @@ require 'oneacct_exporter'
|
|
6
6
|
require 'oneacct_exporter/log'
|
7
7
|
require 'settings'
|
8
8
|
require 'fileutils'
|
9
|
+
require 'json'
|
9
10
|
require 'oneacct_opts'
|
10
11
|
|
11
|
-
#parse options from command line
|
12
|
+
# parse options from command line
|
12
13
|
options = OneacctOpts.parse(ARGV)
|
13
14
|
|
14
|
-
#initialize default logger
|
15
|
+
# initialize default logger
|
15
16
|
log = Logger.new(STDOUT)
|
16
17
|
|
17
|
-
#initialize specific logger according to the configuration
|
18
|
+
# initialize specific logger according to the configuration
|
18
19
|
if Settings['logging'] && Settings['logging']['log_type'] == 'file'
|
19
20
|
begin
|
20
21
|
log_file = File.open(Settings['logging']['log_file'], File::WRONLY | File::CREAT | File::APPEND)
|
@@ -38,7 +39,7 @@ groups = {}
|
|
38
39
|
groups[:include] = options.include_groups if options.include_groups
|
39
40
|
groups[:exclude] = options.exclude_groups if options.exclude_groups
|
40
41
|
|
41
|
-
#read groups restriction from file if chosen
|
42
|
+
# read groups restriction from file if chosen
|
42
43
|
if options.groups_file
|
43
44
|
log.debug('Reading groups from file...')
|
44
45
|
if File.exist?(options.groups_file) && File.readable?(options.groups_file)
|
@@ -54,7 +55,7 @@ if options.groups_file
|
|
54
55
|
end
|
55
56
|
end
|
56
57
|
|
57
|
-
#create output directory
|
58
|
+
# create output directory
|
58
59
|
begin
|
59
60
|
FileUtils.mkdir_p Settings.output['output_dir']
|
60
61
|
rescue SystemCallError => e
|
@@ -73,6 +74,6 @@ opts[:compatibility] = options.compatibility
|
|
73
74
|
|
74
75
|
log.debug(opts)
|
75
76
|
|
76
|
-
#run the export
|
77
|
+
# run the export
|
77
78
|
oneacct_exporter = OneacctExporter.new(opts, log)
|
78
79
|
oneacct_exporter.export
|
data/config/conf.yml
CHANGED
@@ -1,12 +1,21 @@
|
|
1
1
|
---
|
2
2
|
defaults: &defaults
|
3
|
-
site_name: Undefined # Usually a short provider name, e.g. CESNET
|
4
|
-
cloud_type: OpenNebula # CMF type, only OpenNebula is supported
|
5
|
-
endpoint: https://occi.localhost.com:11443/ # URL of your OCCI endpoint, e.g. https://fqdn.example.com:11443/
|
6
3
|
output:
|
7
4
|
output_dir: /var/spool/apel/outgoing/00000000 # Directory for outgoing messages
|
8
|
-
output_type: apel-0.2 # Format of outgoing messages. apel-0.2
|
5
|
+
output_type: apel-0.2 # Format of outgoing messages. Choices are: apel-0.2, pbs-0.1, logstash-0.1
|
9
6
|
num_of_vms_per_file: 500 # Maximum number of virtual machine records per one output file
|
7
|
+
apel: # Options for apel output format
|
8
|
+
site_name: Undefined # Usually a short provider name, e.g. CESNET
|
9
|
+
cloud_type: OpenNebula # CMF type, only OpenNebula is supported
|
10
|
+
endpoint: https://occi.localhost.com:11443/ # URL of your OCCI endpoint, e.g. https://fqdn.example.com:11443/
|
11
|
+
pbs: # Options for pbs output format
|
12
|
+
realm: REALM # Owner's realm, e.g. META
|
13
|
+
queue: cloud # Queue name
|
14
|
+
scratch_type: local # Data store type
|
15
|
+
host_identifier: on_localhost # Identifier for host OpenNebula is running on
|
16
|
+
logstash: # Options for logstash output format
|
17
|
+
host: localhost # Host OpenNebula is running on
|
18
|
+
port: 11443 # Port OpenNebula's RPC is listening on
|
10
19
|
logging:
|
11
20
|
log_type: file # Two options: file, syslog. Defaults to stdout
|
12
21
|
log_file: /var/log/oneacct-export/oneacct-export.log # Used when type file selected
|
@@ -33,13 +42,22 @@ development:
|
|
33
42
|
|
34
43
|
|
35
44
|
test:
|
36
|
-
site_name: <placeholder>
|
37
|
-
cloud_type: <placeholder>
|
38
|
-
endpoint: <placeholder>
|
39
45
|
output:
|
40
46
|
output_dir: <placeholder>
|
41
47
|
output_type: <placeholder>
|
42
48
|
num_of_vms_per_file: <placeholder>
|
49
|
+
apel:
|
50
|
+
site_name: <placeholder>
|
51
|
+
cloud_type: <placeholder>
|
52
|
+
endpoint: <placeholder>
|
53
|
+
pbs:
|
54
|
+
realm: <placeholder>
|
55
|
+
queue: <placeholder>
|
56
|
+
scratch_type: <placeholder>
|
57
|
+
host_identifier: <placeholder>
|
58
|
+
logstash:
|
59
|
+
host: <placeholder>
|
60
|
+
port: <placeholder>
|
43
61
|
logging:
|
44
62
|
log_type: <placeholder>
|
45
63
|
log_file: <placeholder>
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'data_validators/data_validator'
|
2
|
+
require 'data_validators/data_compute'
|
3
|
+
require 'data_validators/data_validator_helper'
|
4
|
+
require 'errors'
|
5
|
+
|
6
|
+
module DataValidators
|
7
|
+
# Data validator class for apel output type
|
8
|
+
class ApelDataValidator < DataValidator
|
9
|
+
include InputValidator
|
10
|
+
include Errors
|
11
|
+
include DataCompute
|
12
|
+
include DataValidatorHelper
|
13
|
+
|
14
|
+
B_IN_GB = 1_073_741_824
|
15
|
+
STATES = %w(started started suspended started suspended suspended completed completed suspended)
|
16
|
+
DEFAULT_VALUE = 'NULL'
|
17
|
+
|
18
|
+
attr_reader :log
|
19
|
+
|
20
|
+
def initialize(log = Logger.new(STDOUT))
|
21
|
+
@log = log
|
22
|
+
end
|
23
|
+
|
24
|
+
# All possible output fields and their default values:
|
25
|
+
#
|
26
|
+
# valid_data['endpoint'] - required
|
27
|
+
# valid_data['site_name'] - required
|
28
|
+
# valid_data['cloud_type'] - required
|
29
|
+
# valid_data['vm_uuid'] - required
|
30
|
+
# valid_data['start_time'] - required
|
31
|
+
# valid_data['end_time'] - defaults to NULL, has to be bigger than valid_data['start_time'] if number
|
32
|
+
# valid_data['machine_name'] - defaults to "one-#{valid_data['vm_uuid']}"
|
33
|
+
# valid_data['user_id'] - defaults to NULL
|
34
|
+
# valid_data['group_id'] - defaults to NULL
|
35
|
+
# valid_data['user_dn'] - defaults to NULL
|
36
|
+
# valid_data['group_name'] - defaults to nil
|
37
|
+
# valid_data['status'] - defaults to NULL
|
38
|
+
# valid_data['duration'] - required
|
39
|
+
# valid_data['suspend'] - defaults to NULL
|
40
|
+
# valid_data['cpu_count'] - defaults to 1
|
41
|
+
# valid_data['network_inbound'] - defaults to 0
|
42
|
+
# valid_data['network_outbound'] - defaults to 0
|
43
|
+
# valid_data['memory'] - defaults to 0
|
44
|
+
# valid_data['image_name'] - defaults to NULL
|
45
|
+
# valid_data['disk_size'] -defaults to NULL
|
46
|
+
def validate_data(data = nil)
|
47
|
+
unless data
|
48
|
+
fail Errors::ValidationError, 'Skipping a malformed record. '\
|
49
|
+
'No data available to validate'
|
50
|
+
end
|
51
|
+
|
52
|
+
valid_data = data.clone
|
53
|
+
|
54
|
+
fail_validation 'Endpoint' unless string?(data['endpoint'])
|
55
|
+
fail_validation 'SiteName' unless string?(data['site_name'])
|
56
|
+
fail_validation 'CloudType' unless string?(data['cloud_type'])
|
57
|
+
fail_validation 'VMUUID' unless string?(data['vm_uuid'])
|
58
|
+
|
59
|
+
fail_validation 'StartTime' unless non_zero_number?(data['start_time'])
|
60
|
+
start_time = data['start_time'].to_i
|
61
|
+
valid_data['start_time'] = Time.at(start_time)
|
62
|
+
fail_validation 'EndTime' unless number?(data['end_time'])
|
63
|
+
end_time = data['end_time'].to_i
|
64
|
+
valid_data['end_time'] = end_time == 0 ? 'NULL' : Time.at(end_time)
|
65
|
+
fail_validation 'EndTime' if end_time != 0 && valid_data['start_time'] > valid_data['end_time']
|
66
|
+
|
67
|
+
valid_data['machine_name'] = default(data['machine_name'], :string, "one-#{valid_data['vm_uuid']}")
|
68
|
+
valid_data['user_id'] = default(data['user_id'], :string, DEFAULT_VALUE)
|
69
|
+
valid_data['group_id'] = default(data['group_id'], :string, DEFAULT_VALUE)
|
70
|
+
valid_data['user_dn'] = default(data['user_dn'], :string, DEFAULT_VALUE)
|
71
|
+
valid_data['user_name'] = default(data['user_name'], :string, DEFAULT_VALUE)
|
72
|
+
valid_data['group_name'] = default(data['group_name'], :string, nil)
|
73
|
+
|
74
|
+
status = default(data['status_code'], :number, nil)
|
75
|
+
if status
|
76
|
+
status = status.to_i
|
77
|
+
fail_validation 'Status' unless status.to_s == data['status_code'] && status < STATES.size && status >= 0
|
78
|
+
end
|
79
|
+
valid_data['status'] = status ? STATES[status] : 'NULL'
|
80
|
+
|
81
|
+
fail_validation 'HISTORY_RECORDS' if (!data['history']) || data['history'].empty?
|
82
|
+
|
83
|
+
duration = sum_rstime(data['history'], valid_data['status'] == 'completed', valid_data['vm_uuid'])
|
84
|
+
valid_data['duration'] = Time.at(duration)
|
85
|
+
valid_data['suspend'] = end_time == 0 ? 'NULL' : (end_time - start_time) - duration
|
86
|
+
valid_data['cpu_count'] = default(data['cpu_count'], :nzn, '1')
|
87
|
+
|
88
|
+
valid_data['network_inbound'] = (default(data['network_inbound'], :number, 0).to_i / B_IN_GB).round
|
89
|
+
valid_data['network_outbound'] = (default(data['network_outbound'], :number, 0).to_i / B_IN_GB).round
|
90
|
+
|
91
|
+
valid_data['memory'] = default(data['memory'], :number, '0')
|
92
|
+
valid_data['image_name'] = default(data['image_name'], :string, DEFAULT_VALUE)
|
93
|
+
disk_size_sum = sum_disk_size(data['disks'], valid_data['vm_uuid'])
|
94
|
+
valid_data['disk_size'] = disk_size_sum ? disk_size_sum : 'NULL'
|
95
|
+
|
96
|
+
valid_data
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# This module expects module DataValidatorHelper to be included with him
|
2
|
+
module DataValidators
|
3
|
+
module DataCompute
|
4
|
+
# Sums RSTIME (time when virtual machine was actually running)
|
5
|
+
#
|
6
|
+
# @param [Array] history records
|
7
|
+
# @param [Boolean] completed whether vm was completed or not
|
8
|
+
# @param [Fixnum] vm_id vm's id
|
9
|
+
#
|
10
|
+
# @return [Integer] sum of time when virtual machine was actually running
|
11
|
+
def sum_rstime(history_records, completed, vm_id)
|
12
|
+
return nil unless history_records
|
13
|
+
|
14
|
+
rstime = 0
|
15
|
+
|
16
|
+
history_records.each do |record|
|
17
|
+
next unless default(record['rstart_time'], :nzn, nil) && default(record['rend_time'], :number, nil)
|
18
|
+
rstart_time = record['rstart_time'].to_i
|
19
|
+
rend_time = record['rend_time'].to_i
|
20
|
+
|
21
|
+
if (rend_time > 0 && rstart_time > rend_time) || (rend_time == 0 && completed)
|
22
|
+
fail Errors::ValidationError, 'Skipping a malformed record. '\
|
23
|
+
"History records' times are invalid for vm with id #{vm_id}."
|
24
|
+
end
|
25
|
+
|
26
|
+
rend_time = rend_time == 0 ? Time.now.to_i : rend_time
|
27
|
+
|
28
|
+
rstime += rend_time - rstart_time
|
29
|
+
end
|
30
|
+
|
31
|
+
rstime
|
32
|
+
end
|
33
|
+
|
34
|
+
# Sums disk size of all disks within the virtual machine
|
35
|
+
#
|
36
|
+
# @param [Array] disk records
|
37
|
+
#
|
38
|
+
# @return [Integer] sum of disk sizes in GB rounded up
|
39
|
+
def sum_disk_size(disks, vm_id)
|
40
|
+
return nil unless disks
|
41
|
+
|
42
|
+
disk_size = 0
|
43
|
+
|
44
|
+
disks.each do |disk|
|
45
|
+
size = default(disk['size'], :number, nil)
|
46
|
+
unless size
|
47
|
+
log.warn("Disk size invalid for vm with id #{vm_id}")
|
48
|
+
return nil
|
49
|
+
end
|
50
|
+
|
51
|
+
disk_size += size.to_i
|
52
|
+
end
|
53
|
+
|
54
|
+
disk_size
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module DataValidators
|
2
|
+
# Interface class for data validator implementations
|
3
|
+
class DataValidator
|
4
|
+
# Validates data for specific output formate and sets default values if necessary.
|
5
|
+
#
|
6
|
+
# @param data [Hash] data to be validated
|
7
|
+
# @return [Hash] data with default values set if necessary
|
8
|
+
def validate_data(data = nil)
|
9
|
+
fail Errors::NotImplementedError, "#{__method__} is just a stub!"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# This module expects modules Errors and InputValidator to be included with him
|
2
|
+
module DataValidators
|
3
|
+
module DataValidatorHelper
|
4
|
+
def fail_validation(field)
|
5
|
+
fail Errors::ValidationError, 'Skipping a malformed record. '\
|
6
|
+
"Field '#{field}' is invalid."
|
7
|
+
end
|
8
|
+
|
9
|
+
def default(value, condition_method, default_value)
|
10
|
+
return string?(value) ? value : default_value if condition_method == :string
|
11
|
+
return number?(value) ? value : default_value if condition_method == :number
|
12
|
+
return non_zero_number?(value) ? value : default_value if condition_method == :nzn
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'data_validators/data_validator'
|
2
|
+
require 'data_validators/data_compute'
|
3
|
+
require 'errors'
|
4
|
+
|
5
|
+
module DataValidators
|
6
|
+
class LogstashDataValidator
|
7
|
+
include InputValidator
|
8
|
+
include Errors
|
9
|
+
include DataCompute
|
10
|
+
include DataValidatorHelper
|
11
|
+
|
12
|
+
attr_reader :log
|
13
|
+
|
14
|
+
def initialize(log = Logger.new(STDOUT))
|
15
|
+
@log = log
|
16
|
+
end
|
17
|
+
|
18
|
+
def validate_data(data = nil)
|
19
|
+
unless data
|
20
|
+
fail Errors::ValidationError, 'Skipping a malformed record. '\
|
21
|
+
'No data available to validate'
|
22
|
+
end
|
23
|
+
|
24
|
+
valid_data = data.clone
|
25
|
+
|
26
|
+
fail_validation 'start_time' unless non_zero_number?(data['start_time'])
|
27
|
+
valid_data['start_time'] = data['start_time'].to_i
|
28
|
+
fail_validation 'end_time' unless number?(data['end_time'])
|
29
|
+
valid_data['end_time'] = data['end_time'].to_i
|
30
|
+
fail_validation 'end_time' if valid_data['end_time'] != 0 && valid_data['start_time'] > valid_data['end_time']
|
31
|
+
|
32
|
+
fail_validation 'user_id' unless number?(data['user_id'])
|
33
|
+
valid_data['user_id'] = data['user_id'].to_i
|
34
|
+
fail_validation 'group_id' unless number?(data['group_id'])
|
35
|
+
valid_data['group_id'] = data['group_id'].to_i
|
36
|
+
|
37
|
+
fail_validation 'status_code' unless number?(data['status_code'])
|
38
|
+
valid_data['status_code'] = data['status_code'].to_i
|
39
|
+
|
40
|
+
fail_validation 'cpu_count' unless number?(data['cpu_count'])
|
41
|
+
valid_data['cpu_count'] = data['cpu_count'].to_i
|
42
|
+
fail_validation 'network_inbound' unless number?(data['network_inbound'])
|
43
|
+
valid_data['network_inbound'] = data['network_inbound'].to_i
|
44
|
+
fail_validation 'network_outbound' unless number?(data['network_outbound'])
|
45
|
+
valid_data['network_outbound'] = data['network_outbound'].to_i
|
46
|
+
fail_validation 'memory' unless number?(data['memory'])
|
47
|
+
valid_data['memory'] = data['memory'].to_i
|
48
|
+
|
49
|
+
fail_validation 'history' unless data['history']
|
50
|
+
history = []
|
51
|
+
data['history'].each do |h|
|
52
|
+
history_record = h.clone
|
53
|
+
fail_validation 'history record start_time' unless non_zero_number?(h['start_time'])
|
54
|
+
history_record['start_time'] = h['start_time'].to_i
|
55
|
+
fail_validation 'history record end_time' unless number?(h['end_time'])
|
56
|
+
history_record['end_time'] = h['end_time'].to_i
|
57
|
+
fail_validation 'history record rstart_time' unless non_zero_number?(h['rstart_time'])
|
58
|
+
history_record['rstart_time'] = h['rstart_time'].to_i
|
59
|
+
fail_validation 'history record rend_time' unless number?(h['rend_time'])
|
60
|
+
history_record['rend_time'] = h['rend_time'].to_i
|
61
|
+
fail_validation 'history record seq' unless number?(h['seq'])
|
62
|
+
history_record['seq'] = h['seq'].to_i
|
63
|
+
|
64
|
+
history << history_record
|
65
|
+
end
|
66
|
+
valid_data['history'] = history
|
67
|
+
|
68
|
+
fail_validation 'disks' unless data['disks']
|
69
|
+
disks = []
|
70
|
+
data['disks'].each do |d|
|
71
|
+
disk = d.clone
|
72
|
+
disk['size'] = d['size']
|
73
|
+
disk['size'] = d['size'].to_i if number?(d['size'])
|
74
|
+
|
75
|
+
disks << disk
|
76
|
+
end
|
77
|
+
valid_data['disks'] = disks
|
78
|
+
|
79
|
+
valid_data
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'data_validators/data_validator'
|
2
|
+
require 'data_validators/data_compute'
|
3
|
+
require 'errors'
|
4
|
+
|
5
|
+
module DataValidators
|
6
|
+
# Data validator class for pbs output type
|
7
|
+
class PbsDataValidator < DataValidator
|
8
|
+
include InputValidator
|
9
|
+
include Errors
|
10
|
+
include DataCompute
|
11
|
+
include DataValidatorHelper
|
12
|
+
|
13
|
+
COMPLETED = '6'
|
14
|
+
|
15
|
+
attr_reader :log
|
16
|
+
|
17
|
+
def initialize(log = Logger.new(STDOUT))
|
18
|
+
@log = log
|
19
|
+
end
|
20
|
+
|
21
|
+
# All possible output fields and their default values:
|
22
|
+
#
|
23
|
+
# valid_data['host'] - required
|
24
|
+
# valid_data['pbs_queue'] - required
|
25
|
+
# valid_data['realm'] - required
|
26
|
+
# valid_data['scratch_type'] - optional, defaults to nil
|
27
|
+
# valid_data['vm_uuid'] - required
|
28
|
+
# valid_data['machine_name'] - required, defaults to "one-#{valid_data['vm_uuid']}"
|
29
|
+
# valid_data['user_name'] - required
|
30
|
+
# valid_data['group_name'] - required
|
31
|
+
# valid_data['duration'] - required, defaults to 00:00:00
|
32
|
+
# valid_data['cpu_count'] - required
|
33
|
+
# valid_data['memory'] - required
|
34
|
+
# valid_data['disk_size'] - optional, defaults to nil
|
35
|
+
# valid_data['history'] - set of history records
|
36
|
+
# history_record['start_time'] - required
|
37
|
+
# history_record['end_time'] - required
|
38
|
+
# history_record['state'] - required, either all history records 'U' or last history record with 'E' if vm finished
|
39
|
+
# history_record['seq'] - required
|
40
|
+
# history_record['hostname'] - required
|
41
|
+
def validate_data(data = nil)
|
42
|
+
unless data
|
43
|
+
fail Errors::ValidationError, 'Skipping a malformed record. '\
|
44
|
+
'No data available to validate'
|
45
|
+
end
|
46
|
+
|
47
|
+
valid_data = data.clone
|
48
|
+
|
49
|
+
fail_validation 'host' unless string?(data['host'])
|
50
|
+
fail_validation 'queue' unless string?(data['pbs_queue'])
|
51
|
+
fail_validation 'owner' unless string?(data['realm'])
|
52
|
+
fail_validation 'VMUUID' unless string?(data['vm_uuid'])
|
53
|
+
fail_validation 'owner' unless string?(data['user_name'])
|
54
|
+
fail_validation 'group' unless string?(data['group_name'])
|
55
|
+
fail_validation 'ppn' unless number?(data['cpu_count'])
|
56
|
+
fail_validation 'mem' unless number?(data['memory'])
|
57
|
+
fail_validation 'HISTORY_RECORDS' if (!data['history']) || data['history'].empty?
|
58
|
+
|
59
|
+
history = []
|
60
|
+
data['history'].each do |h|
|
61
|
+
history_record = h.clone
|
62
|
+
fail_validation 'start' unless non_zero_number?(h['start_time'])
|
63
|
+
history_record['start_time'] = Time.at(h['start_time'].to_i)
|
64
|
+
fail_validation 'end' unless number?(h['end_time'])
|
65
|
+
history_record['end_time'] = Time.at(h['end_time'].to_i)
|
66
|
+
fail_validation 'seq' unless number?(h['seq'])
|
67
|
+
fail_validation 'hostname' unless string?(h['hostname'])
|
68
|
+
|
69
|
+
history_record['state'] = 'U'
|
70
|
+
history << history_record
|
71
|
+
end
|
72
|
+
|
73
|
+
history.last['state'] = 'E' if data['status_code'] == COMPLETED
|
74
|
+
valid_data['history'] = history
|
75
|
+
|
76
|
+
valid_data['machine_name'] = default(data['machine_name'], :string, "one-#{valid_data['vm_uuid']}")
|
77
|
+
|
78
|
+
duration = sum_rstime(data['history'], data['status_code'] == COMPLETED, valid_data['vm_uuid'])
|
79
|
+
valid_data['duration'] = Time.at(duration)
|
80
|
+
|
81
|
+
valid_data['disk_size'] = sum_disk_size(data['disks'], valid_data['vm_uuid'])
|
82
|
+
|
83
|
+
valid_data
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
data/lib/errors.rb
CHANGED
@@ -3,5 +3,7 @@ require 'errors/resource_not_found_error'
|
|
3
3
|
require 'errors/resource_retrieval_error'
|
4
4
|
require 'errors/resource_state_error'
|
5
5
|
require 'errors/user_not_authorized_error'
|
6
|
+
require 'errors/validation_error'
|
7
|
+
require 'errors/not_implemented_error'
|
6
8
|
|
7
9
|
module Errors; end
|
data/lib/input_validator.rb
CHANGED
@@ -4,16 +4,26 @@ require 'uri'
|
|
4
4
|
module InputValidator
|
5
5
|
URI_RE = /\A#{URI.regexp}\z/
|
6
6
|
NUMBER_RE = /\A[[:digit:]]+\z/
|
7
|
+
STRING_RE = /\A[[:print:]]+\z/
|
8
|
+
NON_ZERO_NUMBER_RE = /\A[1-9][[:digit:]]*\z/
|
7
9
|
|
8
10
|
def is?(object, regexp)
|
9
11
|
object.to_s =~ regexp
|
10
12
|
end
|
11
13
|
|
12
|
-
def
|
14
|
+
def number?(object)
|
13
15
|
is?(object, NUMBER_RE)
|
14
16
|
end
|
15
17
|
|
16
|
-
def
|
18
|
+
def uri?(object)
|
17
19
|
is?(object, URI_RE)
|
18
20
|
end
|
21
|
+
|
22
|
+
def string?(object)
|
23
|
+
is?(object, STRING_RE)
|
24
|
+
end
|
25
|
+
|
26
|
+
def non_zero_number?(object)
|
27
|
+
is?(object, NON_ZERO_NUMBER_RE)
|
28
|
+
end
|
19
29
|
end
|
data/lib/one_data_accessor.rb
CHANGED
@@ -9,7 +9,8 @@ require 'input_validator'
|
|
9
9
|
# @attr_reader [any logger] logger
|
10
10
|
# @attr_reader [Integer] batch_size number of vm records to request
|
11
11
|
# @attr_reader [OpenNebula::Client] client client for communicaton with OpenNebula
|
12
|
-
# @attr_reader [TrueClass, FalseClass] compatibility whether or not communicate in
|
12
|
+
# @attr_reader [TrueClass, FalseClass] compatibility whether or not communicate in
|
13
|
+
# compatibility mode (omit some newer API functions)
|
13
14
|
class OneDataAccessor
|
14
15
|
include Errors
|
15
16
|
include InputValidator
|
@@ -23,7 +24,7 @@ class OneDataAccessor
|
|
23
24
|
@compatibility = compatibility
|
24
25
|
|
25
26
|
@batch_size = Settings.output['num_of_vms_per_file'] ? Settings.output['num_of_vms_per_file'] : 500
|
26
|
-
fail ArgumentError, 'Wrong number of vms per file.' unless
|
27
|
+
fail ArgumentError, 'Wrong number of vms per file.' unless number?(@batch_size)
|
27
28
|
|
28
29
|
@compatibility_vm_pool = nil
|
29
30
|
|
@@ -34,7 +35,7 @@ class OneDataAccessor
|
|
34
35
|
def initialize_client
|
35
36
|
secret = Settings['xml_rpc'] ? Settings.xml_rpc['secret'] : nil
|
36
37
|
endpoint = Settings['xml_rpc'] ? Settings.xml_rpc['endpoint'] : nil
|
37
|
-
fail ArgumentError, "#{endpoint} is not a valid URL." if endpoint && !
|
38
|
+
fail ArgumentError, "#{endpoint} is not a valid URL." if endpoint && !uri?(endpoint)
|
38
39
|
|
39
40
|
@client = OpenNebula::Client.new(secret, endpoint)
|
40
41
|
end
|
@@ -48,7 +49,7 @@ class OneDataAccessor
|
|
48
49
|
def mapping(pool_class, xpath)
|
49
50
|
@log.debug("Generating mapping for class: #{pool_class} and xpath: '#{xpath}'.")
|
50
51
|
pool = pool_class.new(@client)
|
51
|
-
#call info_all method instead of info on pools that support it
|
52
|
+
# call info_all method instead of info on pools that support it
|
52
53
|
if pool.respond_to? 'info_all'
|
53
54
|
rc = pool.info_all
|
54
55
|
check_retval(rc, Errors::ResourceRetrievalError)
|
@@ -57,7 +58,7 @@ class OneDataAccessor
|
|
57
58
|
check_retval(rc, Errors::ResourceRetrievalError)
|
58
59
|
end
|
59
60
|
|
60
|
-
#generate mapping
|
61
|
+
# generate mapping
|
61
62
|
map = {}
|
62
63
|
pool.each do |item|
|
63
64
|
unless item['ID']
|
@@ -76,7 +77,7 @@ class OneDataAccessor
|
|
76
77
|
#
|
77
78
|
# @return [OpenNebula::VirtualMachine] virtual machine
|
78
79
|
def vm(vm_id)
|
79
|
-
fail ArgumentError, "#{vm_id} is not a valid id." unless
|
80
|
+
fail ArgumentError, "#{vm_id} is not a valid id." unless number?(vm_id)
|
80
81
|
@log.debug("Retrieving virtual machine with id: #{vm_id}.")
|
81
82
|
vm = OpenNebula::VirtualMachine.new(OpenNebula::VirtualMachine.build_xml(vm_id), @client)
|
82
83
|
rc = vm.info
|
@@ -93,7 +94,7 @@ class OneDataAccessor
|
|
93
94
|
# @return [Array] array with virtual machines' IDs
|
94
95
|
def vms(batch_number, range, groups)
|
95
96
|
vms = []
|
96
|
-
#load specific batch
|
97
|
+
# load specific batch
|
97
98
|
vm_pool = load_vm_pool(batch_number)
|
98
99
|
return nil if vm_pool.count == 0
|
99
100
|
|
@@ -104,7 +105,7 @@ class OneDataAccessor
|
|
104
105
|
next
|
105
106
|
end
|
106
107
|
|
107
|
-
#skip unsuitable virtual machines
|
108
|
+
# skip unsuitable virtual machines
|
108
109
|
next unless want?(vm, range, groups)
|
109
110
|
|
110
111
|
vms << vm['ID'].to_i
|
@@ -145,12 +146,12 @@ class OneDataAccessor
|
|
145
146
|
#
|
146
147
|
# @param [Integer] batch_number
|
147
148
|
def load_vm_pool(batch_number)
|
148
|
-
fail ArgumentError, "#{batch_number} is not a valid number" unless
|
149
|
+
fail ArgumentError, "#{batch_number} is not a valid number" unless number?(batch_number)
|
149
150
|
@log.debug("Loading vm pool with batch number: #{batch_number}.")
|
150
151
|
from = batch_number * @batch_size
|
151
152
|
to = (batch_number + 1) * @batch_size - 1
|
152
153
|
|
153
|
-
#if in compatibility mode, whole virtual machine pool has to be loaded for the first time
|
154
|
+
# if in compatibility mode, whole virtual machine pool has to be loaded for the first time
|
154
155
|
if @compatibility
|
155
156
|
unless @compatibility_vm_pool
|
156
157
|
vm_pool = OpenNebula::VirtualMachinePool.new(@client)
|