oneacct-export 0.2.7 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rspec +1 -0
- data/.travis.yml +1 -21
- data/bin/oneacct-export +7 -6
- data/config/conf.yml +25 -7
- data/lib/data_validators/apel_data_validator.rb +99 -0
- data/lib/data_validators/data_compute.rb +57 -0
- data/lib/data_validators/data_validator.rb +12 -0
- data/lib/data_validators/data_validator_helper.rb +15 -0
- data/lib/data_validators/logstash_data_validator.rb +82 -0
- data/lib/data_validators/pbs_data_validator.rb +86 -0
- data/lib/errors/not_implemented_error.rb +3 -0
- data/lib/errors/validation_error.rb +3 -0
- data/lib/errors.rb +2 -0
- data/lib/input_validator.rb +12 -2
- data/lib/one_data_accessor.rb +11 -10
- data/lib/one_worker.rb +109 -137
- data/lib/oneacct_exporter/version.rb +1 -1
- data/lib/oneacct_exporter.rb +9 -7
- data/lib/oneacct_opts.rb +36 -13
- data/lib/output_types.rb +5 -0
- data/lib/redis_conf.rb +2 -2
- data/lib/settings.rb +3 -3
- data/lib/templates/apel-0.2.erb +6 -6
- data/lib/templates/logstash-0.1.erb +3 -0
- data/lib/templates/pbs-0.1.erb +6 -0
- data/mock/{one_worker_vm8.xml → one_worker_vm_dn01.xml} +76 -74
- data/mock/one_worker_vm_dn02.xml +174 -0
- data/mock/{one_worker_DISK_missing.xml → one_worker_vm_empty_disk_records.xml} +10 -6
- data/mock/one_worker_vm_empty_history_records.xml +131 -0
- data/mock/one_worker_vm_image_name01.xml +175 -0
- data/mock/{one_worker_valid_machine.xml → one_worker_vm_image_name02.xml} +35 -7
- data/mock/one_worker_vm_image_name03.xml +167 -0
- data/mock/{one_worker_vm2.xml → one_worker_vm_image_name04.xml} +38 -9
- data/mock/{one_worker_vm1.xml → one_worker_vm_image_name05.xml} +36 -8
- data/mock/{one_worker_vm9.xml → one_worker_vm_image_name06.xml} +8 -5
- data/oneacct-export.gemspec +1 -0
- data/spec/data_validators/apel_data_validator_spec.rb +497 -0
- data/spec/data_validators/data_compute_spec.rb +193 -0
- data/spec/data_validators/data_validator_helper_spec.rb +66 -0
- data/spec/data_validators/data_validator_spec.rb +14 -0
- data/spec/data_validators/logstash_data_validator_spec.rb +469 -0
- data/spec/data_validators/pbs_data_validator_spec.rb +353 -0
- data/spec/one_worker_spec.rb +234 -542
- data/spec/oneacct_exporter_spec.rb +1 -41
- data/spec/oneacct_opts_spec.rb +135 -32
- data/spec/spec_helper.rb +18 -1
- metadata +51 -52
- data/mock/one_worker_DEPLOY_ID_missing.xml +0 -136
- data/mock/one_worker_DISK_SIZE_nan.xml +0 -147
- data/mock/one_worker_ETIME_0.xml +0 -137
- data/mock/one_worker_ETIME_missing.xml +0 -136
- data/mock/one_worker_ETIME_nan.xml +0 -137
- data/mock/one_worker_GID_missing.xml +0 -136
- data/mock/one_worker_GNAME_missing.xml +0 -136
- data/mock/one_worker_HISTORY_RECORDS_missing.xml +0 -91
- data/mock/one_worker_HISTORY_many.xml +0 -137
- data/mock/one_worker_HISTORY_missing.xml +0 -93
- data/mock/one_worker_HISTORY_one.xml +0 -115
- data/mock/one_worker_IMAGE_ID_missing.xml +0 -136
- data/mock/one_worker_MEMORY_0.xml +0 -137
- data/mock/one_worker_MEMORY_missing.xml +0 -135
- data/mock/one_worker_MEMORY_nan.xml +0 -137
- data/mock/one_worker_NET_RX_0.xml +0 -137
- data/mock/one_worker_NET_RX_missing.xml +0 -136
- data/mock/one_worker_NET_RX_nan.xml +0 -137
- data/mock/one_worker_NET_TX_0.xml +0 -137
- data/mock/one_worker_NET_TX_missing.xml +0 -136
- data/mock/one_worker_NET_TX_nan.xml +0 -137
- data/mock/one_worker_RETIME_0_RUNNING.xml +0 -115
- data/mock/one_worker_RETIME_0_STOPPED.xml +0 -115
- data/mock/one_worker_RETIME_missing.xml +0 -114
- data/mock/one_worker_RSTIME_0.xml +0 -115
- data/mock/one_worker_RSTIME_>_RETIME.xml +0 -115
- data/mock/one_worker_RSTIME_missing.xml +0 -114
- data/mock/one_worker_STATE_missing.xml +0 -136
- data/mock/one_worker_STATE_out_of_range.xml +0 -137
- data/mock/one_worker_STIME_>_ETIME.xml +0 -137
- data/mock/one_worker_STIME_missing.xml +0 -136
- data/mock/one_worker_STIME_nan.xml +0 -137
- data/mock/one_worker_TEMPLATE_missing.xml +0 -79
- data/mock/one_worker_UID_missing.xml +0 -136
- data/mock/one_worker_VCPU_0.xml +0 -137
- data/mock/one_worker_VCPU_missing.xml +0 -136
- data/mock/one_worker_VCPU_nan.xml +0 -137
- data/mock/one_worker_malformed_vm.xml +0 -136
- data/mock/one_worker_vm3.xml +0 -137
- data/mock/one_worker_vm4.xml +0 -106
- data/mock/one_worker_vm5.xml +0 -106
- data/mock/one_worker_vm6.xml +0 -107
- data/mock/one_worker_vm7.xml +0 -147
data/lib/one_worker.rb
CHANGED
@@ -7,29 +7,43 @@ require 'one_writer'
|
|
7
7
|
require 'sidekiq_conf'
|
8
8
|
require 'oneacct_exporter/log'
|
9
9
|
require 'settings'
|
10
|
+
require 'data_validators/apel_data_validator'
|
11
|
+
require 'data_validators/pbs_data_validator'
|
12
|
+
require 'data_validators/logstash_data_validator'
|
13
|
+
require 'output_types'
|
14
|
+
require 'errors'
|
10
15
|
|
11
16
|
# Sidekiq worker class
|
12
17
|
class OneWorker
|
13
18
|
include Sidekiq::Worker
|
19
|
+
include OutputTypes
|
20
|
+
include Errors
|
21
|
+
|
22
|
+
sidekiq_options retry: 5, dead: false, \
|
23
|
+
queue: (Settings['sidekiq'] && Settings.sidekiq['queue']) ? Settings.sidekiq['queue'].to_sym : :default
|
24
|
+
|
25
|
+
# Prepare data that are specific for output type and common for every virtual machine
|
26
|
+
def output_type_specific_data
|
27
|
+
data = {}
|
28
|
+
if Settings.output['output_type'] == PBS_OT && Settings.output['pbs']
|
29
|
+
data['realm'] = Settings.output.pbs['realm']
|
30
|
+
data['pbs_queue'] = Settings.output.pbs['queue']
|
31
|
+
data['scratch_type'] = Settings.output.pbs['scratch_type']
|
32
|
+
data['host'] = Settings.output.pbs['host_identifier']
|
33
|
+
end
|
14
34
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
STRING = /[[:print:]]+/
|
21
|
-
NUMBER = /[[:digit:]]+/
|
22
|
-
NON_ZERO = /[1-9][[:digit:]]*/
|
23
|
-
STATES = %w(started started suspended started suspended suspended completed completed suspended)
|
35
|
+
if Settings.output['output_type'] == APEL_OT
|
36
|
+
data['endpoint'] = Settings.output.apel['endpoint'].chomp('/')
|
37
|
+
data['site_name'] = Settings.output.apel['site_name']
|
38
|
+
data['cloud_type'] = Settings.output.apel['cloud_type']
|
39
|
+
end
|
24
40
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
common_data['site_name'] = Settings['site_name']
|
30
|
-
common_data['cloud_type'] = Settings['cloud_type']
|
41
|
+
if Settings.output['output_type'] == LOGSTASH_OT
|
42
|
+
data['host'] = Settings.output.logstash['host']
|
43
|
+
data['port'] = Settings.output.logstash['port']
|
44
|
+
end
|
31
45
|
|
32
|
-
|
46
|
+
data
|
33
47
|
end
|
34
48
|
|
35
49
|
# Create mapping of user ID and specified element
|
@@ -53,7 +67,7 @@ class OneWorker
|
|
53
67
|
oda.mapping(pool_type, mapping)
|
54
68
|
rescue => e
|
55
69
|
msg = "Couldn't create map: #{e.message}. "\
|
56
|
-
|
70
|
+
'Stopping to avoid malformed records.'
|
57
71
|
logger.error(msg)
|
58
72
|
raise msg
|
59
73
|
end
|
@@ -68,80 +82,75 @@ class OneWorker
|
|
68
82
|
return nil
|
69
83
|
end
|
70
84
|
|
71
|
-
# Obtain and parse required data from vm
|
85
|
+
# Obtain and parse required data from vm
|
72
86
|
#
|
73
87
|
# @return [Hash] required data from virtual machine
|
74
88
|
def process_vm(vm, user_map, image_map)
|
75
|
-
data =
|
89
|
+
data = output_type_specific_data
|
90
|
+
|
91
|
+
data['vm_uuid'] = vm['ID']
|
92
|
+
data['start_time'] = vm['STIME']
|
93
|
+
data['end_time'] = vm['ETIME']
|
94
|
+
data['machine_name'] = vm['DEPLOY_ID']
|
95
|
+
data['user_id'] = vm['UID']
|
96
|
+
data['group_id'] = vm['GID']
|
97
|
+
data['user_dn'] = vm['USER_TEMPLATE/USER_X509_DN']
|
98
|
+
data['user_dn'] ||= user_map[data['user_id']]
|
99
|
+
data['user_name'] = vm['UNAME']
|
100
|
+
data['group_name'] = vm['GNAME']
|
101
|
+
data['status_code'] = vm['STATE']
|
102
|
+
data['status'] = vm.state_str
|
103
|
+
data['cpu_count'] = vm['TEMPLATE/VCPU']
|
104
|
+
data['network_inbound'] = vm['NET_TX']
|
105
|
+
data['network_outbound'] = vm['NET_RX']
|
106
|
+
data['memory'] = vm['TEMPLATE/MEMORY']
|
107
|
+
data['image_name'] = vm['TEMPLATE/DISK[1]/VMCATCHER_EVENT_AD_MPURI']
|
108
|
+
data['image_name'] ||= image_map[vm['TEMPLATE/DISK[1]/IMAGE_ID']]
|
109
|
+
data['image_name'] ||= mixin(vm)
|
110
|
+
data['image_name'] ||= vm['TEMPLATE/DISK[1]/IMAGE_ID']
|
111
|
+
data['history'] = history_records(vm)
|
112
|
+
data['disks'] = disk_records(vm)
|
76
113
|
|
77
|
-
data
|
78
|
-
|
79
|
-
logger.error('Skipping a malformed record. '\
|
80
|
-
"VM with id #{data['vm_uuid']} has no StartTime.")
|
81
|
-
return nil
|
82
|
-
end
|
114
|
+
data
|
115
|
+
end
|
83
116
|
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
117
|
+
# Returns an array of history records from vm
|
118
|
+
#
|
119
|
+
# @param [OpenNebula::VirtualMachine] vm virtual machine
|
120
|
+
#
|
121
|
+
# @return [Array] array of hashes representing vm's history records
|
122
|
+
def history_records(vm)
|
123
|
+
history = []
|
124
|
+
vm.each 'HISTORY_RECORDS/HISTORY' do |h|
|
125
|
+
history_record = {}
|
126
|
+
history_record['start_time'] = h['STIME']
|
127
|
+
history_record['end_time'] = h['ETIME']
|
128
|
+
history_record['rstart_time'] = h['RSTIME']
|
129
|
+
history_record['rend_time'] = h['RETIME']
|
130
|
+
history_record['seq'] = h['SEQ']
|
131
|
+
history_record['hostname'] = h['HOSTNAME']
|
132
|
+
|
133
|
+
history << history_record
|
99
134
|
end
|
100
135
|
|
101
|
-
|
102
|
-
|
103
|
-
data['group_id'] = parse(vm['GID'], STRING)
|
104
|
-
data['user_name'] = parse(vm['USER_TEMPLATE/USER_X509_DN'], STRING, nil)
|
105
|
-
data['user_name'] = parse(user_map[data['user_id']], STRING) unless data['user_name']
|
106
|
-
data['fqan'] = parse(vm['GNAME'], STRING, nil)
|
107
|
-
|
108
|
-
if vm['STATE']
|
109
|
-
data['status'] = parse(STATES[vm['STATE'].to_i], STRING)
|
110
|
-
else
|
111
|
-
data['status'] = 'NULL'
|
112
|
-
end
|
136
|
+
history
|
137
|
+
end
|
113
138
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
139
|
+
# Returns an array of disk records from vm
|
140
|
+
#
|
141
|
+
# @param [OpenNebula::VirtualMachine] vm virtual machine
|
142
|
+
#
|
143
|
+
# @return [Array] array of hashes representing vm's disk records
|
144
|
+
def disk_records(vm)
|
145
|
+
disks = []
|
146
|
+
vm.each 'TEMPLATE/DISK' do |d|
|
147
|
+
disk = {}
|
148
|
+
disk['size'] = d['SIZE']
|
149
|
+
|
150
|
+
disks << disk
|
118
151
|
end
|
119
152
|
|
120
|
-
|
121
|
-
return nil unless rstime
|
122
|
-
|
123
|
-
data['duration'] = parse(rstime.to_s, NON_ZERO)
|
124
|
-
|
125
|
-
suspend = (end_time - start_time) - data['duration'].to_i unless end_time == 0
|
126
|
-
data['suspend'] = parse(suspend.to_s, NUMBER)
|
127
|
-
|
128
|
-
data['cpu_count'] = parse(vm['TEMPLATE/VCPU'], NON_ZERO, '1')
|
129
|
-
|
130
|
-
net_tx = parse(vm['NET_TX'], NUMBER, 0)
|
131
|
-
data['network_inbound'] = (net_tx.to_i / B_IN_GB).round
|
132
|
-
net_rx = parse(vm['NET_RX'], NUMBER, 0)
|
133
|
-
data['network_outbound'] = (net_rx.to_i / B_IN_GB).round
|
134
|
-
|
135
|
-
data['memory'] = parse(vm['TEMPLATE/MEMORY'], NUMBER, '0')
|
136
|
-
|
137
|
-
data['image_name'] = parse(vm['TEMPLATE/DISK[1]/VMCATCHER_EVENT_AD_MPURI'], STRING, nil)
|
138
|
-
data['image_name'] = parse(image_map[vm['TEMPLATE/DISK[1]/IMAGE_ID']], STRING, nil) unless data['image_name']
|
139
|
-
data['image_name'] = parse(mixin(vm), STRING, nil) unless data['image_name']
|
140
|
-
data['image_name'] = parse(vm['TEMPLATE/DISK[1]/IMAGE_ID'], STRING) unless data['image_name']
|
141
|
-
|
142
|
-
data['disk_size'] = sum_disk_size(vm)
|
143
|
-
|
144
|
-
data
|
153
|
+
disks
|
145
154
|
end
|
146
155
|
|
147
156
|
# Look for 'os_tpl' OCCI mixin to better identifie virtual machine's image
|
@@ -163,56 +172,10 @@ class OneWorker
|
|
163
172
|
nil # nothing found
|
164
173
|
end
|
165
174
|
|
166
|
-
# Sums RSTIME (time when virtual machine was actually running)
|
167
|
-
#
|
168
|
-
# @param [OpenNebula::VirtualMachine] vm virtual machine
|
169
|
-
#
|
170
|
-
# @return [Integer] RSTIME
|
171
|
-
def sum_rstime(vm)
|
172
|
-
rstime = 0
|
173
|
-
vm.each 'HISTORY_RECORDS/HISTORY' do |h|
|
174
|
-
next unless h['RSTIME'] && h['RETIME'] && h['RSTIME'] != '0'
|
175
|
-
if h['RETIME'] == '0' && STATES[vm['STATE'].to_i] != 'completed'
|
176
|
-
rstime += Time.now.to_i - h['RSTIME'].to_i
|
177
|
-
next
|
178
|
-
end
|
179
|
-
if h['RSTIME'].to_i > h['RETIME'].to_i
|
180
|
-
logger.warn('Skipping malformed record. '\
|
181
|
-
"VM with id #{vm['ID']} has wrong CpuDuration.")
|
182
|
-
rstime = nil
|
183
|
-
break
|
184
|
-
end
|
185
|
-
rstime += h['RETIME'].to_i - h['RSTIME'].to_i
|
186
|
-
end
|
187
|
-
|
188
|
-
rstime
|
189
|
-
end
|
190
|
-
|
191
|
-
# Sums disk size of all disks within the virtual machine
|
192
|
-
#
|
193
|
-
# @param [OpenNebula::VirtualMachine] vm virtual machine
|
194
|
-
#
|
195
|
-
# @return [Integer] sum of disk sizes in GB rounded up
|
196
|
-
def sum_disk_size(vm)
|
197
|
-
disk_size = 'NULL'
|
198
|
-
vm.each 'TEMPLATE/DISK' do |disk|
|
199
|
-
return 'NULL' unless disk['SIZE']
|
200
|
-
|
201
|
-
size = parse(disk['SIZE'], NUMBER, nil)
|
202
|
-
unless size
|
203
|
-
logger.warn("Disk size invalid for VM with id #{vm['ID']}.")
|
204
|
-
return 'NULL'
|
205
|
-
end
|
206
|
-
disk_size = disk_size.to_i + size.to_i
|
207
|
-
end
|
208
|
-
|
209
|
-
disk_size = (disk_size/1000.0).ceil unless disk_size.to_i == 0
|
210
|
-
disk_size
|
211
|
-
end
|
212
|
-
|
213
175
|
# Sidekiq specific method, specifies the purpose of the worker
|
214
176
|
#
|
215
|
-
# @param [String] vms IDs of virtual machines to process in form of numbers separated by '|'
|
177
|
+
# @param [String] vms IDs of virtual machines to process in form of numbers separated by '|'
|
178
|
+
# (easier for cooperation with redis)
|
216
179
|
# @param [String] file_number number of the output file
|
217
180
|
def perform(vms, file_number)
|
218
181
|
OneacctExporter::Log.setup_log_level(logger)
|
@@ -229,18 +192,31 @@ class OneWorker
|
|
229
192
|
vm = load_vm(vm_id, oda)
|
230
193
|
next unless vm
|
231
194
|
|
232
|
-
|
233
|
-
|
234
|
-
|
195
|
+
begin
|
196
|
+
logger.debug("Processing vm with id: #{vm_id}.")
|
197
|
+
vm_data = process_vm(vm, user_map, image_map)
|
198
|
+
|
199
|
+
validator = DataValidators::ApelDataValidator.new(logger) if Settings.output['output_type'] == APEL_OT
|
200
|
+
validator = DataValidators::PbsDataValidator.new(logger) if Settings.output['output_type'] == PBS_OT
|
201
|
+
validator = DataValidators::LogstashDataValidator.new(logger) if Settings.output['output_type'] == LOGSTASH_OT
|
202
|
+
|
203
|
+
vm_data = validator.validate_data(vm_data) if validator
|
204
|
+
rescue Errors::ValidationError => e
|
205
|
+
logger.error("Error occured during processing of vm with id: #{vm_id}. #{e.message}")
|
206
|
+
next
|
207
|
+
end
|
235
208
|
|
236
209
|
logger.debug("Adding vm with data: #{vm_data} for export.")
|
237
210
|
data << vm_data
|
238
211
|
end
|
239
212
|
|
240
|
-
write_data(data, file_number)
|
213
|
+
write_data(data, file_number) unless data.empty?
|
241
214
|
end
|
242
215
|
|
243
216
|
# Write processed data into output directory
|
217
|
+
#
|
218
|
+
# @param [Hash] data data to be written into file
|
219
|
+
# @param [Fixnum] file_number sequence number of file data will be written to
|
244
220
|
def write_data(data, file_number)
|
245
221
|
logger.debug('Creating writer...')
|
246
222
|
ow = OneWriter.new(data, file_number, logger)
|
@@ -250,8 +226,4 @@ class OneWorker
|
|
250
226
|
logger.error(msg)
|
251
227
|
raise msg
|
252
228
|
end
|
253
|
-
|
254
|
-
def parse(value, regex, substitute = 'NULL')
|
255
|
-
regex =~ value ? value : substitute
|
256
|
-
end
|
257
229
|
end
|
data/lib/oneacct_exporter.rb
CHANGED
@@ -8,12 +8,13 @@ require 'sidekiq/api'
|
|
8
8
|
#
|
9
9
|
# @attr_reader [any logger] log logger for the class
|
10
10
|
# @attr_reader [Hash] range range of dates, requesting only virtual machines within the range
|
11
|
-
# @attr_reader [Hash] groups user groups, requesting only virtual machines with owners that
|
11
|
+
# @attr_reader [Hash] groups user groups, requesting only virtual machines with owners that
|
12
|
+
# belong to one of the group
|
12
13
|
# @attr_reader [TrueClass, FalseClass] blocking says whether to run export in blocking mode or not
|
13
14
|
# @attr_reader [Integer] timeout timeout for blocking mode
|
14
|
-
# @attr_reader [TrueClass, FalseClass] compatibility says whether to run export in compatibility
|
15
|
+
# @attr_reader [TrueClass, FalseClass] compatibility says whether to run export in compatibility
|
16
|
+
# mode or not
|
15
17
|
class OneacctExporter
|
16
|
-
|
17
18
|
attr_reader :log, :range, :groups, :blocking, :timeout, :compatibility
|
18
19
|
|
19
20
|
def initialize(options, log)
|
@@ -36,11 +37,11 @@ class OneacctExporter
|
|
36
37
|
oda = OneDataAccessor.new(@compatibility, @log)
|
37
38
|
|
38
39
|
vms = []
|
39
|
-
#load records of virtual machines in batches
|
40
|
+
# load records of virtual machines in batches
|
40
41
|
while vms = oda.vms(batch_number, @range, @groups)
|
41
42
|
@log.info("Starting worker with batch number: #{batch_number}.")
|
42
43
|
unless vms.empty?
|
43
|
-
#add a new job for every batch to the Sidekiq's queue
|
44
|
+
# add a new job for every batch to the Sidekiq's queue
|
44
45
|
OneWorker.perform_async(vms.join('|'), new_file_number)
|
45
46
|
new_file_number += 1
|
46
47
|
end
|
@@ -94,8 +95,9 @@ class OneacctExporter
|
|
94
95
|
# Clean output directory of previous entries
|
95
96
|
def clean_output_dir
|
96
97
|
output_dir = Dir.new(Settings.output['output_dir'])
|
97
|
-
output_dir.entries.
|
98
|
-
|
98
|
+
entries = output_dir.entries.select { |entry| entry != '.' && entry != '..' }
|
99
|
+
entries.each do |entry|
|
100
|
+
File.delete("#{output_dir.path}/#{entry}")
|
99
101
|
end
|
100
102
|
end
|
101
103
|
end
|
data/lib/oneacct_opts.rb
CHANGED
@@ -6,6 +6,8 @@ require 'settings'
|
|
6
6
|
|
7
7
|
# Class for parsing command line arguments
|
8
8
|
class OneacctOpts
|
9
|
+
include OutputTypes
|
10
|
+
|
9
11
|
BLOCKING_DEFAULT = false
|
10
12
|
TIMEOUT_DEFAULT = 60 * 60
|
11
13
|
COMPATIBILITY_DEFAULT = false
|
@@ -84,9 +86,7 @@ class OneacctOpts
|
|
84
86
|
# Set default values for not specified options
|
85
87
|
def self.set_defaults(options)
|
86
88
|
options.blocking = BLOCKING_DEFAULT unless options.blocking
|
87
|
-
unless options.timeout
|
88
|
-
options.timeout = TIMEOUT_DEFAULT if options.blocking
|
89
|
-
end
|
89
|
+
options.timeout = TIMEOUT_DEFAULT if options.blocking unless options.timeout
|
90
90
|
options.compatibility = COMPATIBILITY_DEFAULT unless options.compatibility
|
91
91
|
end
|
92
92
|
|
@@ -97,24 +97,24 @@ class OneacctOpts
|
|
97
97
|
|
98
98
|
# Make sure command line parameters are sane
|
99
99
|
def self.check_options_restrictions(options)
|
100
|
-
#make sure date range make sense
|
100
|
+
# make sure date range make sense
|
101
101
|
if options.records_from && options.records_to && options.records_from >= options.records_to
|
102
102
|
fail ArgumentError, 'Wrong time range for records retrieval.'
|
103
103
|
end
|
104
104
|
|
105
|
-
#make sure only one group restriction is used
|
105
|
+
# make sure only one group restriction is used
|
106
106
|
if options.include_groups && options.exclude_groups
|
107
107
|
fail ArgumentError, 'Mixing of group options is not possible.'
|
108
108
|
end
|
109
109
|
|
110
|
-
#make sure group file option is not used without specifying group restriction type
|
110
|
+
# make sure group file option is not used without specifying group restriction type
|
111
111
|
unless options.include_groups || options.exclude_groups
|
112
112
|
if options.groups_file
|
113
113
|
fail ArgumentError, 'Cannot use group file without specifying group restriction type.'
|
114
114
|
end
|
115
115
|
end
|
116
116
|
|
117
|
-
#make sure that timeout option is not used without blocking option
|
117
|
+
# make sure that timeout option is not used without blocking option
|
118
118
|
if options.timeout && !options.blocking
|
119
119
|
fail ArgumentError, 'Cannot set timeout without a blocking mode.'
|
120
120
|
end
|
@@ -122,22 +122,45 @@ class OneacctOpts
|
|
122
122
|
|
123
123
|
# Make sure configuration is sane
|
124
124
|
def self.check_settings_restrictions
|
125
|
-
#make sure all mandatory parameters are set
|
126
|
-
unless Settings['
|
127
|
-
Settings['output'] && Settings.output['output_dir'] && Settings.output['output_type']
|
125
|
+
# make sure all mandatory parameters are set
|
126
|
+
unless Settings['output'] && Settings.output['output_dir'] && Settings.output['output_type']
|
128
127
|
fail ArgumentError, 'Missing some mandatory parameters. Check your configuration file.'
|
129
128
|
end
|
130
129
|
|
131
|
-
#make sure log file is specified while loggin to file
|
130
|
+
# make sure log file is specified while loggin to file
|
132
131
|
if Settings['logging'] && Settings.logging['log_type'] == 'file' &&
|
133
|
-
|
132
|
+
!Settings.logging['log_file']
|
134
133
|
fail ArgumentError, 'Missing file for logging. Check your configuration file.'
|
135
134
|
end
|
136
135
|
|
137
|
-
|
136
|
+
check_output_type_specific_settings
|
137
|
+
|
138
|
+
# make sure specified template really exists
|
138
139
|
template_filename = OneWriter.template_filename(Settings.output['output_type'])
|
139
140
|
unless File.exist?(template_filename)
|
140
141
|
fail ArgumentError, "Non-existing template #{Settings.output['output_type']}."
|
141
142
|
end
|
142
143
|
end
|
144
|
+
|
145
|
+
def self.check_output_type_specific_settings
|
146
|
+
if Settings.output['output_type'] == APEL_OT
|
147
|
+
unless Settings.output['apel'] && Settings.output.apel['site_name'] &&
|
148
|
+
Settings.output.apel['cloud_type'] && Settings.output.apel['endpoint']
|
149
|
+
fail ArgumentError, 'Missing some mandatory parameters for APEL output type. Check your configuration file.'
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
if Settings.output['output_type'] == PBS_OT && Settings.output['pbs']
|
154
|
+
Settings.output.pbs['realm'] ||= 'META'
|
155
|
+
Settings.output.pbs['queue'] ||= 'cloud'
|
156
|
+
Settings.output.pbs['scratch_type'] ||= 'local'
|
157
|
+
Settings.output.pbs['host_identifier'] ||= 'on_localhost'
|
158
|
+
end
|
159
|
+
|
160
|
+
if Settings.output['output_type'] == LOGSTASH_OT
|
161
|
+
unless Settings.output['logstash'] && Settings.output.logstash['host'] && Settings.output.logstash['port']
|
162
|
+
fail ArgumentError, 'Missing some mandatory parameters for logstash output type. Check your configuration file.'
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
143
166
|
end
|
data/lib/output_types.rb
ADDED
data/lib/redis_conf.rb
CHANGED
@@ -8,7 +8,7 @@ class RedisConf
|
|
8
8
|
|
9
9
|
# Read and parse Redis server configuration options
|
10
10
|
#
|
11
|
-
# @return [Hash] redis server options ready for use
|
11
|
+
# @return [Hash] redis server options ready for use
|
12
12
|
def self.options
|
13
13
|
options = {}
|
14
14
|
if Settings['redis']
|
@@ -20,7 +20,7 @@ class RedisConf
|
|
20
20
|
options[:url] ||= 'redis://localhost:6379'
|
21
21
|
|
22
22
|
fail ArgumentError, "#{options[:url]} is not a valid URL."\
|
23
|
-
unless
|
23
|
+
unless uri?(options[:url])
|
24
24
|
|
25
25
|
if Settings['redis'] && Settings.redis['password']
|
26
26
|
fail ArgumentError, 'Redis password cannot be empty'\
|
data/lib/settings.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
require 'settingslogic'
|
2
2
|
|
3
|
-
# Class representing OneacctExport settings
|
3
|
+
# Class representing OneacctExport settings
|
4
4
|
class Settings < Settingslogic
|
5
5
|
CONF_NAME = 'conf.yml'
|
6
6
|
|
7
|
-
#three possible configuration file locations in order by preference
|
8
|
-
#if configuration file is found rest of the locations are ignored
|
7
|
+
# three possible configuration file locations in order by preference
|
8
|
+
# if configuration file is found rest of the locations are ignored
|
9
9
|
source "#{ENV['HOME']}/.oneacct-export/#{CONF_NAME}"\
|
10
10
|
if File.exist?("#{ENV['HOME']}/.oneacct-export/#{CONF_NAME}")
|
11
11
|
source "/etc/oneacct-export/#{CONF_NAME}"\
|
data/lib/templates/apel-0.2.erb
CHANGED
@@ -5,9 +5,9 @@ SiteName: <%= vm['site_name']%>
|
|
5
5
|
MachineName: <%= vm['machine_name']%>
|
6
6
|
LocalUserId: <%= vm['user_id']%>
|
7
7
|
LocalGroupId: <%= vm['group_id']%>
|
8
|
-
GlobalUserName: <%= vm['
|
9
|
-
<% if vm['
|
10
|
-
FQAN: /<%= vm['
|
8
|
+
GlobalUserName: <%= vm['user_dn']%>
|
9
|
+
<% if vm['group_name']-%>
|
10
|
+
FQAN: /<%= vm['group_name']%>/Role=NULL/Capability=NULL
|
11
11
|
<% else -%>
|
12
12
|
FQAN: NULL
|
13
13
|
<% end -%>
|
@@ -15,14 +15,14 @@ Status: <%= vm['status']%>
|
|
15
15
|
StartTime: <%= vm['start_time'].to_i%>
|
16
16
|
EndTime: <%= vm['end_time'].to_s == vm['end_time'] ? vm['end_time'] : vm['end_time'].to_i%>
|
17
17
|
SuspendDuration: <%= vm['suspend']%>
|
18
|
-
WallDuration: <%= vm['duration']%>
|
19
|
-
CpuDuration: <%= vm['duration']%>
|
18
|
+
WallDuration: <%= vm['duration'].to_i != 0 ? vm['duration'].to_i : 'NULL'%>
|
19
|
+
CpuDuration: <%= vm['duration'].to_i != 0 ? vm['duration'].to_i : 'NULL'%>
|
20
20
|
CpuCount: <%= vm['cpu_count']%>
|
21
21
|
NetworkType: NULL
|
22
22
|
NetworkInbound: <%= vm['network_inbound']%>
|
23
23
|
NetworkOutbound: <%= vm['network_outbound']%>
|
24
24
|
Memory: <%= vm['memory']%>
|
25
|
-
Disk: <%= vm['disk_size']%>
|
25
|
+
Disk: <%= vm['disk_size'].to_i != 0 ? (vm['disk_size']/1000.0).ceil : vm['disk_size']%>
|
26
26
|
StorageRecordId: NULL
|
27
27
|
ImageId: <%= vm['image_name']%>
|
28
28
|
CloudType: OpenNebula
|
@@ -0,0 +1,6 @@
|
|
1
|
+
<%- for vm in @data -%>
|
2
|
+
<%- for history_record in vm['history'] -%>
|
3
|
+
<%= history_record['start_time'].strftime('%D %T') %>;<%= history_record['state']%>;<%= vm['host']%>_<%= vm['machine_name']%>-<%= history_record['seq']%>;user=<%= vm['user_name']%> group=<%= vm['group_name']%> jobname=<%= vm['machine_name']%> queue=<%= vm['pbs_queue']%> ctime=<%= history_record['start_time'].to_i%> qtime=<%= history_record['start_time'].to_i%> etime=<%= history_record['start_time'].to_i%> start=<%= history_record['start_time'].to_i%> end=<%= history_record['end_time'].to_i%> owner=<%= vm['user_name']%>@<%= vm['realm']%> sched_nodespec=host=<%= history_record['hostname']%>:ppn=<%= vm['cpu_count']%>:mem=<%= vm['memory'].to_i*1024%>KB:vmem=<%= vm['memory'].to_i*1024%>KB<% if vm['scratch_type'] -%>:scratch_type=<%= vm['scratch_type']%><% end -%><% if vm['disk_size'] -%>:scratch_volume=<%= vm['disk_size']%><% end -%>mb Resource_List.mem=<%= vm['memory']%>mb Resource_List.nodect=1 <% if vm['disk_size'] -%>Resource_List.scratch=<%= vm['disk_size']%>mb <% end -%>Resource_List.vmem=<%= (vm['memory'].to_i/1024.0).ceil%>gb Resource_List.walltime=<%= (vm['duration'].to_i/3600).floor%>:<%= vm['duration'].utc.strftime('%M:%S')%> resc_req_total.mem=<%= vm['memory'].to_i*1024%>kb resc_req_total.nodect=1 resc_req_total.procs=<%= vm['cpu_count']%> resc_req_total.scratch=<%= vm['disk_size']%>mb resc_req_total.vmem=<%= vm['memory'].to_i*1024%>kb resc_req_total.walltime=<%= (vm['duration'].to_i/3600).floor%>:<%= vm['duration'].utc.strftime('%M:%S')%> Exit_status=0 resources_used.cput=<%= (vm['duration'].to_i/3600).floor%>:<%= vm['duration'].utc.strftime('%M:%S')%> resources_used.mem=<%= vm['memory'].to_i*1024%>kb resources_used.vmem=<%= vm['memory'].to_i*1024%>kb resources_used.walltime=<%= (vm['duration'].to_i/3600).floor%>:<%= vm['duration'].utc.strftime('%M:%S')%>
|
4
|
+
<%- end -%>
|
5
|
+
<%- end -%>
|
6
|
+
|