oneacct-export 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rspec +1 -0
- data/.travis.yml +1 -21
- data/bin/oneacct-export +7 -6
- data/config/conf.yml +25 -7
- data/lib/data_validators/apel_data_validator.rb +99 -0
- data/lib/data_validators/data_compute.rb +57 -0
- data/lib/data_validators/data_validator.rb +12 -0
- data/lib/data_validators/data_validator_helper.rb +15 -0
- data/lib/data_validators/logstash_data_validator.rb +82 -0
- data/lib/data_validators/pbs_data_validator.rb +86 -0
- data/lib/errors/not_implemented_error.rb +3 -0
- data/lib/errors/validation_error.rb +3 -0
- data/lib/errors.rb +2 -0
- data/lib/input_validator.rb +12 -2
- data/lib/one_data_accessor.rb +11 -10
- data/lib/one_worker.rb +109 -137
- data/lib/oneacct_exporter/version.rb +1 -1
- data/lib/oneacct_exporter.rb +9 -7
- data/lib/oneacct_opts.rb +36 -13
- data/lib/output_types.rb +5 -0
- data/lib/redis_conf.rb +2 -2
- data/lib/settings.rb +3 -3
- data/lib/templates/apel-0.2.erb +6 -6
- data/lib/templates/logstash-0.1.erb +3 -0
- data/lib/templates/pbs-0.1.erb +6 -0
- data/mock/{one_worker_vm8.xml → one_worker_vm_dn01.xml} +76 -74
- data/mock/one_worker_vm_dn02.xml +174 -0
- data/mock/{one_worker_DISK_missing.xml → one_worker_vm_empty_disk_records.xml} +10 -6
- data/mock/one_worker_vm_empty_history_records.xml +131 -0
- data/mock/one_worker_vm_image_name01.xml +175 -0
- data/mock/{one_worker_valid_machine.xml → one_worker_vm_image_name02.xml} +35 -7
- data/mock/one_worker_vm_image_name03.xml +167 -0
- data/mock/{one_worker_vm2.xml → one_worker_vm_image_name04.xml} +38 -9
- data/mock/{one_worker_vm1.xml → one_worker_vm_image_name05.xml} +36 -8
- data/mock/{one_worker_vm9.xml → one_worker_vm_image_name06.xml} +8 -5
- data/oneacct-export.gemspec +1 -0
- data/spec/data_validators/apel_data_validator_spec.rb +497 -0
- data/spec/data_validators/data_compute_spec.rb +193 -0
- data/spec/data_validators/data_validator_helper_spec.rb +66 -0
- data/spec/data_validators/data_validator_spec.rb +14 -0
- data/spec/data_validators/logstash_data_validator_spec.rb +469 -0
- data/spec/data_validators/pbs_data_validator_spec.rb +353 -0
- data/spec/one_worker_spec.rb +234 -542
- data/spec/oneacct_exporter_spec.rb +1 -41
- data/spec/oneacct_opts_spec.rb +135 -32
- data/spec/spec_helper.rb +18 -1
- metadata +51 -52
- data/mock/one_worker_DEPLOY_ID_missing.xml +0 -136
- data/mock/one_worker_DISK_SIZE_nan.xml +0 -147
- data/mock/one_worker_ETIME_0.xml +0 -137
- data/mock/one_worker_ETIME_missing.xml +0 -136
- data/mock/one_worker_ETIME_nan.xml +0 -137
- data/mock/one_worker_GID_missing.xml +0 -136
- data/mock/one_worker_GNAME_missing.xml +0 -136
- data/mock/one_worker_HISTORY_RECORDS_missing.xml +0 -91
- data/mock/one_worker_HISTORY_many.xml +0 -137
- data/mock/one_worker_HISTORY_missing.xml +0 -93
- data/mock/one_worker_HISTORY_one.xml +0 -115
- data/mock/one_worker_IMAGE_ID_missing.xml +0 -136
- data/mock/one_worker_MEMORY_0.xml +0 -137
- data/mock/one_worker_MEMORY_missing.xml +0 -135
- data/mock/one_worker_MEMORY_nan.xml +0 -137
- data/mock/one_worker_NET_RX_0.xml +0 -137
- data/mock/one_worker_NET_RX_missing.xml +0 -136
- data/mock/one_worker_NET_RX_nan.xml +0 -137
- data/mock/one_worker_NET_TX_0.xml +0 -137
- data/mock/one_worker_NET_TX_missing.xml +0 -136
- data/mock/one_worker_NET_TX_nan.xml +0 -137
- data/mock/one_worker_RETIME_0_RUNNING.xml +0 -115
- data/mock/one_worker_RETIME_0_STOPPED.xml +0 -115
- data/mock/one_worker_RETIME_missing.xml +0 -114
- data/mock/one_worker_RSTIME_0.xml +0 -115
- data/mock/one_worker_RSTIME_>_RETIME.xml +0 -115
- data/mock/one_worker_RSTIME_missing.xml +0 -114
- data/mock/one_worker_STATE_missing.xml +0 -136
- data/mock/one_worker_STATE_out_of_range.xml +0 -137
- data/mock/one_worker_STIME_>_ETIME.xml +0 -137
- data/mock/one_worker_STIME_missing.xml +0 -136
- data/mock/one_worker_STIME_nan.xml +0 -137
- data/mock/one_worker_TEMPLATE_missing.xml +0 -79
- data/mock/one_worker_UID_missing.xml +0 -136
- data/mock/one_worker_VCPU_0.xml +0 -137
- data/mock/one_worker_VCPU_missing.xml +0 -136
- data/mock/one_worker_VCPU_nan.xml +0 -137
- data/mock/one_worker_malformed_vm.xml +0 -136
- data/mock/one_worker_vm3.xml +0 -137
- data/mock/one_worker_vm4.xml +0 -106
- data/mock/one_worker_vm5.xml +0 -106
- data/mock/one_worker_vm6.xml +0 -107
- data/mock/one_worker_vm7.xml +0 -147
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
module DataValidators
|
|
4
|
+
describe PbsDataValidator do
|
|
5
|
+
|
|
6
|
+
subject { PbsDataValidator.new(Logger.new('/dev/null')) }
|
|
7
|
+
|
|
8
|
+
describe '.validate_data' do
|
|
9
|
+
let(:data) do
|
|
10
|
+
data = {}
|
|
11
|
+
|
|
12
|
+
data['pbs_queue'] = 'queue'
|
|
13
|
+
data['realm'] = 'REALM'
|
|
14
|
+
data['host'] = 'on_localhost'
|
|
15
|
+
data['scratch_type'] = 'local'
|
|
16
|
+
|
|
17
|
+
data['vm_uuid'] = '36551'
|
|
18
|
+
data['start_time'] = '1383741160'
|
|
19
|
+
data['end_time'] = '1383742270'
|
|
20
|
+
data['machine_name'] = 'one-36551'
|
|
21
|
+
data['user_id'] = '120'
|
|
22
|
+
data['group_id'] = '0'
|
|
23
|
+
data['user_name'] = 'uname'
|
|
24
|
+
data['group_name'] = 'gname'
|
|
25
|
+
data['status'] = '3'
|
|
26
|
+
data['cpu_count'] = '1'
|
|
27
|
+
data['network_inbound'] = '4154845418'
|
|
28
|
+
data['network_outbound'] = '6326418701'
|
|
29
|
+
data['memory'] = '1736960'
|
|
30
|
+
history = []
|
|
31
|
+
rec = {}
|
|
32
|
+
rec['start_time'] = '1383741169'
|
|
33
|
+
rec['end_time'] = '1383741259'
|
|
34
|
+
rec['rstart_time'] = '1383741278'
|
|
35
|
+
rec['rend_time'] = '1383741367'
|
|
36
|
+
rec['seq'] = '0'
|
|
37
|
+
rec['hostname'] = 'supermachine1.somewhere.com'
|
|
38
|
+
history << rec
|
|
39
|
+
rec = {}
|
|
40
|
+
rec['start_time'] = '1383741589'
|
|
41
|
+
rec['end_time'] = '1383742270'
|
|
42
|
+
rec['rstart_time'] = '1383741674'
|
|
43
|
+
rec['rend_time'] = '1383742270'
|
|
44
|
+
rec['seq'] = '1'
|
|
45
|
+
rec['hostname'] = 'supermachine2.somewhere.com'
|
|
46
|
+
history << rec
|
|
47
|
+
data['history'] = history
|
|
48
|
+
data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
|
|
49
|
+
|
|
50
|
+
data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
|
|
51
|
+
data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
|
|
52
|
+
|
|
53
|
+
data
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
let(:valid_data) do
|
|
57
|
+
valid_data = {}
|
|
58
|
+
|
|
59
|
+
valid_data['pbs_queue'] = 'queue'
|
|
60
|
+
valid_data['realm'] = 'REALM'
|
|
61
|
+
valid_data['host'] = 'on_localhost'
|
|
62
|
+
valid_data['scratch_type'] = 'local'
|
|
63
|
+
|
|
64
|
+
valid_data['vm_uuid'] = '36551'
|
|
65
|
+
valid_data['start_time'] = '1383741160'
|
|
66
|
+
valid_data['end_time'] = '1383742270'
|
|
67
|
+
valid_data['machine_name'] = 'one-36551'
|
|
68
|
+
valid_data['user_id'] = '120'
|
|
69
|
+
valid_data['group_id'] = '0'
|
|
70
|
+
valid_data['user_name'] = 'uname'
|
|
71
|
+
valid_data['group_name'] = 'gname'
|
|
72
|
+
valid_data['status'] = '3'
|
|
73
|
+
valid_data['cpu_count'] = '1'
|
|
74
|
+
valid_data['network_inbound'] = '4154845418'
|
|
75
|
+
valid_data['network_outbound'] = '6326418701'
|
|
76
|
+
valid_data['memory'] = '1736960'
|
|
77
|
+
history = []
|
|
78
|
+
rec = {}
|
|
79
|
+
rec['start_time'] = Time.at(1383741169)
|
|
80
|
+
rec['end_time'] = Time.at(1383741259)
|
|
81
|
+
rec['rstart_time'] = '1383741278'
|
|
82
|
+
rec['rend_time'] = '1383741367'
|
|
83
|
+
rec['seq'] = '0'
|
|
84
|
+
rec['hostname'] = 'supermachine1.somewhere.com'
|
|
85
|
+
rec['state'] = 'U'
|
|
86
|
+
history << rec
|
|
87
|
+
rec = {}
|
|
88
|
+
rec['start_time'] = Time.at(1383741589)
|
|
89
|
+
rec['end_time'] = Time.at(1383742270)
|
|
90
|
+
rec['rstart_time'] = '1383741674'
|
|
91
|
+
rec['rend_time'] = '1383742270'
|
|
92
|
+
rec['seq'] = '1'
|
|
93
|
+
rec['hostname'] = 'supermachine2.somewhere.com'
|
|
94
|
+
rec['state'] = 'U'
|
|
95
|
+
history << rec
|
|
96
|
+
valid_data['history'] = history
|
|
97
|
+
valid_data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
|
|
98
|
+
|
|
99
|
+
valid_data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
|
|
100
|
+
valid_data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
|
|
101
|
+
|
|
102
|
+
valid_data['duration'] = Time.at(685)
|
|
103
|
+
valid_data['disk_size'] = 52608
|
|
104
|
+
|
|
105
|
+
valid_data
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
context 'with all data valid' do
|
|
109
|
+
it 'returns the same data transformed if needed' do
|
|
110
|
+
expect(subject.validate_data(data)).to eq(valid_data)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
context 'without any data' do
|
|
115
|
+
let(:data) { nil }
|
|
116
|
+
|
|
117
|
+
it 'fails with ValidationError' do
|
|
118
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
context 'with empty data' do
|
|
123
|
+
let(:data) { {} }
|
|
124
|
+
|
|
125
|
+
it 'fails with ValidationError' do
|
|
126
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
context 'without host' do
|
|
131
|
+
before :example do
|
|
132
|
+
data['host'] = nil
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
it 'fails with ValidationError' do
|
|
136
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
context 'without queue' do
|
|
141
|
+
before :example do
|
|
142
|
+
data['pbs_queue'] = nil
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
it 'fails with ValidationError' do
|
|
146
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
context 'without realm' do
|
|
151
|
+
before :example do
|
|
152
|
+
data['realm'] = nil
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
it 'fails with ValidationError' do
|
|
156
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
context 'without vm uuid' do
|
|
161
|
+
before :example do
|
|
162
|
+
data['vm_uuid'] = nil
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
it 'fails with ValidationError' do
|
|
166
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
context 'without user name' do
|
|
171
|
+
before :example do
|
|
172
|
+
data['user_name'] = nil
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
it 'fails with ValidationError' do
|
|
176
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
context 'without group name' do
|
|
181
|
+
before :example do
|
|
182
|
+
data['group_name'] = nil
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
it 'fails with ValidationError' do
|
|
186
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
context 'without cpu count' do
|
|
191
|
+
before :example do
|
|
192
|
+
data['cpu_count'] = nil
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
it 'fails with ValidationError' do
|
|
196
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
context 'with cpu count that is not a number' do
|
|
201
|
+
before :example do
|
|
202
|
+
data['cpu_count'] = 'string'
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
it 'fails with ValidationError' do
|
|
206
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
context 'without memory' do
|
|
211
|
+
before :example do
|
|
212
|
+
data['memory'] = nil
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
it 'fails with ValidationError' do
|
|
216
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
context 'with memory that is not a number' do
|
|
221
|
+
before :example do
|
|
222
|
+
data['memory'] = 'string'
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
it 'fails with ValidationError' do
|
|
226
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
context 'without history records' do
|
|
231
|
+
before :example do
|
|
232
|
+
data['history'] = nil
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
it 'fails with ValidationError' do
|
|
236
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
context 'with empty history records' do
|
|
241
|
+
before :example do
|
|
242
|
+
data['history'] = []
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
it 'fails with ValidationError' do
|
|
246
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
context 'without start time in history record' do
|
|
251
|
+
before :example do
|
|
252
|
+
data['history'][0]['start_time'] = nil
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
it 'fails with ValidationError' do
|
|
256
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
context 'with start time in history record that is not a number' do
|
|
261
|
+
before :example do
|
|
262
|
+
data['history'][0]['start_time'] = 'string'
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
it 'fails with ValidationError' do
|
|
266
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
context 'with start time in history record that is zero' do
|
|
271
|
+
before :example do
|
|
272
|
+
data['history'][0]['start_time'] = '0'
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
it 'fails with ValidationError' do
|
|
276
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
context 'without end time in history record' do
|
|
281
|
+
before :example do
|
|
282
|
+
data['history'][0]['end_time'] = nil
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
it 'fails with ValidationError' do
|
|
286
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
context 'with end time in history record that is not a number' do
|
|
291
|
+
before :example do
|
|
292
|
+
data['history'][0]['end_time'] = 'string'
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
it 'fails with ValidationError' do
|
|
296
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
context 'without sequence number in history record' do
|
|
301
|
+
before :example do
|
|
302
|
+
data['history'][0]['seq'] = nil
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
it 'fails with ValidationError' do
|
|
306
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
context 'with sequence number in history record that is not a number' do
|
|
311
|
+
before :example do
|
|
312
|
+
data['history'][0]['seq'] = 'string'
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
it 'fails with ValidationError' do
|
|
316
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
317
|
+
end
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
context 'without hostname in history record' do
|
|
321
|
+
before :example do
|
|
322
|
+
data['history'][0]['hostname'] = nil
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
it 'fails with ValidationError' do
|
|
326
|
+
expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
context 'with stopped virtual machine' do
|
|
331
|
+
before :example do
|
|
332
|
+
data['status_code'] = '6'
|
|
333
|
+
valid_data['history'][1]['state'] = 'E'
|
|
334
|
+
valid_data['status_code'] = '6'
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
it 'sets last history record\'s state to "E"' do
|
|
338
|
+
expect(subject.validate_data(data)).to eq(valid_data)
|
|
339
|
+
end
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
context 'without machine name' do
|
|
343
|
+
before :example do
|
|
344
|
+
data['machine_name'] = nil
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
it 'replaces machine name with string created from id and prefix "one-"' do
|
|
348
|
+
expect(subject.validate_data(data)).to eq(valid_data)
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
end
|
|
352
|
+
end
|
|
353
|
+
end
|