oneacct-export 0.2.7 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rspec +1 -0
  4. data/.travis.yml +1 -21
  5. data/bin/oneacct-export +7 -6
  6. data/config/conf.yml +25 -7
  7. data/lib/data_validators/apel_data_validator.rb +99 -0
  8. data/lib/data_validators/data_compute.rb +57 -0
  9. data/lib/data_validators/data_validator.rb +12 -0
  10. data/lib/data_validators/data_validator_helper.rb +15 -0
  11. data/lib/data_validators/logstash_data_validator.rb +82 -0
  12. data/lib/data_validators/pbs_data_validator.rb +86 -0
  13. data/lib/errors/not_implemented_error.rb +3 -0
  14. data/lib/errors/validation_error.rb +3 -0
  15. data/lib/errors.rb +2 -0
  16. data/lib/input_validator.rb +12 -2
  17. data/lib/one_data_accessor.rb +11 -10
  18. data/lib/one_worker.rb +109 -137
  19. data/lib/oneacct_exporter/version.rb +1 -1
  20. data/lib/oneacct_exporter.rb +9 -7
  21. data/lib/oneacct_opts.rb +36 -13
  22. data/lib/output_types.rb +5 -0
  23. data/lib/redis_conf.rb +2 -2
  24. data/lib/settings.rb +3 -3
  25. data/lib/templates/apel-0.2.erb +6 -6
  26. data/lib/templates/logstash-0.1.erb +3 -0
  27. data/lib/templates/pbs-0.1.erb +6 -0
  28. data/mock/{one_worker_vm8.xml → one_worker_vm_dn01.xml} +76 -74
  29. data/mock/one_worker_vm_dn02.xml +174 -0
  30. data/mock/{one_worker_DISK_missing.xml → one_worker_vm_empty_disk_records.xml} +10 -6
  31. data/mock/one_worker_vm_empty_history_records.xml +131 -0
  32. data/mock/one_worker_vm_image_name01.xml +175 -0
  33. data/mock/{one_worker_valid_machine.xml → one_worker_vm_image_name02.xml} +35 -7
  34. data/mock/one_worker_vm_image_name03.xml +167 -0
  35. data/mock/{one_worker_vm2.xml → one_worker_vm_image_name04.xml} +38 -9
  36. data/mock/{one_worker_vm1.xml → one_worker_vm_image_name05.xml} +36 -8
  37. data/mock/{one_worker_vm9.xml → one_worker_vm_image_name06.xml} +8 -5
  38. data/oneacct-export.gemspec +1 -0
  39. data/spec/data_validators/apel_data_validator_spec.rb +497 -0
  40. data/spec/data_validators/data_compute_spec.rb +193 -0
  41. data/spec/data_validators/data_validator_helper_spec.rb +66 -0
  42. data/spec/data_validators/data_validator_spec.rb +14 -0
  43. data/spec/data_validators/logstash_data_validator_spec.rb +469 -0
  44. data/spec/data_validators/pbs_data_validator_spec.rb +353 -0
  45. data/spec/one_worker_spec.rb +234 -542
  46. data/spec/oneacct_exporter_spec.rb +1 -41
  47. data/spec/oneacct_opts_spec.rb +135 -32
  48. data/spec/spec_helper.rb +18 -1
  49. metadata +51 -52
  50. data/mock/one_worker_DEPLOY_ID_missing.xml +0 -136
  51. data/mock/one_worker_DISK_SIZE_nan.xml +0 -147
  52. data/mock/one_worker_ETIME_0.xml +0 -137
  53. data/mock/one_worker_ETIME_missing.xml +0 -136
  54. data/mock/one_worker_ETIME_nan.xml +0 -137
  55. data/mock/one_worker_GID_missing.xml +0 -136
  56. data/mock/one_worker_GNAME_missing.xml +0 -136
  57. data/mock/one_worker_HISTORY_RECORDS_missing.xml +0 -91
  58. data/mock/one_worker_HISTORY_many.xml +0 -137
  59. data/mock/one_worker_HISTORY_missing.xml +0 -93
  60. data/mock/one_worker_HISTORY_one.xml +0 -115
  61. data/mock/one_worker_IMAGE_ID_missing.xml +0 -136
  62. data/mock/one_worker_MEMORY_0.xml +0 -137
  63. data/mock/one_worker_MEMORY_missing.xml +0 -135
  64. data/mock/one_worker_MEMORY_nan.xml +0 -137
  65. data/mock/one_worker_NET_RX_0.xml +0 -137
  66. data/mock/one_worker_NET_RX_missing.xml +0 -136
  67. data/mock/one_worker_NET_RX_nan.xml +0 -137
  68. data/mock/one_worker_NET_TX_0.xml +0 -137
  69. data/mock/one_worker_NET_TX_missing.xml +0 -136
  70. data/mock/one_worker_NET_TX_nan.xml +0 -137
  71. data/mock/one_worker_RETIME_0_RUNNING.xml +0 -115
  72. data/mock/one_worker_RETIME_0_STOPPED.xml +0 -115
  73. data/mock/one_worker_RETIME_missing.xml +0 -114
  74. data/mock/one_worker_RSTIME_0.xml +0 -115
  75. data/mock/one_worker_RSTIME_>_RETIME.xml +0 -115
  76. data/mock/one_worker_RSTIME_missing.xml +0 -114
  77. data/mock/one_worker_STATE_missing.xml +0 -136
  78. data/mock/one_worker_STATE_out_of_range.xml +0 -137
  79. data/mock/one_worker_STIME_>_ETIME.xml +0 -137
  80. data/mock/one_worker_STIME_missing.xml +0 -136
  81. data/mock/one_worker_STIME_nan.xml +0 -137
  82. data/mock/one_worker_TEMPLATE_missing.xml +0 -79
  83. data/mock/one_worker_UID_missing.xml +0 -136
  84. data/mock/one_worker_VCPU_0.xml +0 -137
  85. data/mock/one_worker_VCPU_missing.xml +0 -136
  86. data/mock/one_worker_VCPU_nan.xml +0 -137
  87. data/mock/one_worker_malformed_vm.xml +0 -136
  88. data/mock/one_worker_vm3.xml +0 -137
  89. data/mock/one_worker_vm4.xml +0 -106
  90. data/mock/one_worker_vm5.xml +0 -106
  91. data/mock/one_worker_vm6.xml +0 -107
  92. data/mock/one_worker_vm7.xml +0 -147
@@ -0,0 +1,66 @@
1
+ require 'spec_helper'
2
+
3
+ module DataValidators
4
+ describe DataValidatorHelper do
5
+ subject { data_validator_helper }
6
+
7
+ let(:data_validator_helper) do
8
+ data_validator_helper = Object.new
9
+ data_validator_helper.extend(Errors)
10
+ data_validator_helper.extend(InputValidator)
11
+ data_validator_helper.extend(DataValidatorHelper)
12
+
13
+ data_validator_helper
14
+ end
15
+
16
+ describe '.fail_validation' do
17
+ it 'fails with ValidationError' do
18
+ expect { subject.fail_validation('field') }.to raise_error(Errors::ValidationError)
19
+ end
20
+ end
21
+
22
+ describe '.default' do
23
+ context 'with :string condition method' do
24
+ context 'with string value' do
25
+ it 'returns back value' do
26
+ expect(subject.default('string', :string, 'default')).to eq('string')
27
+ end
28
+ end
29
+ end
30
+
31
+ context 'with :number condition method' do
32
+ context 'with number value' do
33
+ it 'returns back value' do
34
+ expect(subject.default(42, :number, 'default')).to eq(42)
35
+ end
36
+ end
37
+
38
+ context 'with non-number value' do
39
+ it 'returns default value' do
40
+ expect(subject.default('string', :number, 'default')).to eq('default')
41
+ end
42
+ end
43
+ end
44
+
45
+ context 'with :nzn condition method' do
46
+ context 'with non-zero number value' do
47
+ it 'returns back value' do
48
+ expect(subject.default(42, :nzn, 'default')).to eq(42)
49
+ end
50
+ end
51
+
52
+ context 'with zero value' do
53
+ it 'returns default value' do
54
+ expect(subject.default(0, :nzn, 'default')).to eq('default')
55
+ end
56
+ end
57
+
58
+ context 'with non-number value' do
59
+ it 'returns default value' do
60
+ expect(subject.default('string', :nzn, 'default')).to eq('default')
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,14 @@
1
+ require 'spec_helper'
2
+
3
+ module DataValidators
4
+ describe DataValidator do
5
+
6
+ subject { DataValidator.new }
7
+
8
+ describe '.validate_data' do
9
+ it 'fails with NotImplementedError' do
10
+ expect { subject.validate_data }.to raise_error(Errors::NotImplementedError)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,469 @@
1
+ require 'rspec'
2
+
3
+ module DataValidators
4
+ describe LogstashDataValidator do
5
+
6
+ subject { LogstashDataValidator.new(Logger.new('/dev/null')) }
7
+
8
+ describe '.validate_data' do
9
+ let(:data) do
10
+ data = {}
11
+
12
+ data['endpoint'] = 'machine.hogwarts.co.uk'
13
+ data['site_name'] = 'Hogwarts'
14
+ data['cloud_type'] = 'OpenNebula'
15
+
16
+ data['vm_uuid'] = '36551'
17
+ data['start_time'] = '1383741160'
18
+ data['end_time'] = '1383742270'
19
+ data['machine_name'] = 'one-36551'
20
+ data['user_id'] = '120'
21
+ data['group_id'] = '0'
22
+ data['user_name'] = 'uname'
23
+ data['group_name'] = 'gname'
24
+ data['status_code'] = '3'
25
+ data['status'] = 'STATUS'
26
+ data['cpu_count'] = '1'
27
+ data['network_inbound'] = '4154845418'
28
+ data['network_outbound'] = '6326418701'
29
+ data['memory'] = '1736960'
30
+ history = []
31
+ rec = {}
32
+ rec['start_time'] = '1383741169'
33
+ rec['end_time'] = '1383741259'
34
+ rec['rstart_time'] = '1383741278'
35
+ rec['rend_time'] = '1383741367'
36
+ rec['seq'] = '0'
37
+ rec['hostname'] = 'supermachine1.somewhere.com'
38
+ history << rec
39
+ rec = {}
40
+ rec['start_time'] = '1383741589'
41
+ rec['end_time'] = '1383742270'
42
+ rec['rstart_time'] = '1383741674'
43
+ rec['rend_time'] = '1383742270'
44
+ rec['seq'] = '1'
45
+ rec['hostname'] = 'supermachine2.somewhere.com'
46
+ history << rec
47
+ data['history'] = history
48
+ data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
49
+
50
+ data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
51
+ data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
52
+
53
+ data
54
+ end
55
+
56
+ let(:valid_data) do
57
+ valid_data = {}
58
+
59
+ valid_data['endpoint'] = 'machine.hogwarts.co.uk'
60
+ valid_data['site_name'] = 'Hogwarts'
61
+ valid_data['cloud_type'] = 'OpenNebula'
62
+
63
+ valid_data['vm_uuid'] = '36551'
64
+ valid_data['start_time'] = 1383741160
65
+ valid_data['end_time'] = 1383742270
66
+ valid_data['machine_name'] = 'one-36551'
67
+ valid_data['user_id'] = 120
68
+ valid_data['group_id'] = 0
69
+ valid_data['user_name'] = 'uname'
70
+ valid_data['group_name'] = 'gname'
71
+ valid_data['status_code'] = 3
72
+ valid_data['status'] = 'STATUS'
73
+ valid_data['cpu_count'] = 1
74
+ valid_data['network_inbound'] = 4154845418
75
+ valid_data['network_outbound'] = 6326418701
76
+ valid_data['memory'] = 1736960
77
+ history = []
78
+ rec = {}
79
+ rec['start_time'] = 1383741169
80
+ rec['end_time'] = 1383741259
81
+ rec['rstart_time'] = 1383741278
82
+ rec['rend_time'] = 1383741367
83
+ rec['seq'] = 0
84
+ rec['hostname'] = 'supermachine1.somewhere.com'
85
+ history << rec
86
+ rec = {}
87
+ rec['start_time'] = 1383741589
88
+ rec['end_time'] = 1383742270
89
+ rec['rstart_time'] = 1383741674
90
+ rec['rend_time'] = 1383742270
91
+ rec['seq'] = 1
92
+ rec['hostname'] = 'supermachine2.somewhere.com'
93
+ history << rec
94
+ valid_data['history'] = history
95
+ valid_data['disks'] = [{'size' => 10240}, {'size' => 42368}]
96
+
97
+ valid_data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
98
+ valid_data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
99
+
100
+ valid_data
101
+ end
102
+
103
+ context 'with all data valid' do
104
+ it 'returns the same data transformed if needed' do
105
+ expect(subject.validate_data(data)).to eq(valid_data)
106
+ end
107
+ end
108
+
109
+ context 'without any data' do
110
+ let(:data) { nil }
111
+
112
+ it 'fails with ValidationError' do
113
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
114
+ end
115
+ end
116
+
117
+ context 'with empty data' do
118
+ let(:data) { {} }
119
+
120
+ it 'fails with ValidationError' do
121
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
122
+ end
123
+ end
124
+
125
+ context 'without start time' do
126
+ before :example do
127
+ data['start_time'] = nil
128
+ end
129
+
130
+ it 'it fails with ValidationError' do
131
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
132
+ end
133
+ end
134
+
135
+ context 'with zero start time' do
136
+ before :example do
137
+ data['start_time'] = '0'
138
+ end
139
+
140
+ it 'it fails with ValidationError' do
141
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
142
+ end
143
+ end
144
+
145
+ context 'with start time that is not a number' do
146
+ before :example do
147
+ data['start_time'] = 'string'
148
+ end
149
+
150
+ it 'it fails with ValidationError' do
151
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
152
+ end
153
+ end
154
+
155
+ context 'without end time' do
156
+ before :example do
157
+ data['end_time'] = nil
158
+ end
159
+
160
+ it 'it fails with ValidationError' do
161
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
162
+ end
163
+ end
164
+
165
+ context 'with end time that is not a number' do
166
+ before :example do
167
+ data['end_time'] = 'string'
168
+ end
169
+
170
+ it 'it fails with ValidationError' do
171
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
172
+ end
173
+ end
174
+
175
+ context 'with start time bigger than end time' do
176
+ before :example do
177
+ time = data['end_time']
178
+ data['end_time'] = data['start_time']
179
+ data['start_time'] = time
180
+ end
181
+
182
+ it 'it fails with ValidationError' do
183
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
184
+ end
185
+ end
186
+
187
+ context 'without user id' do
188
+ before :example do
189
+ data['user_id'] = nil
190
+ end
191
+
192
+ it 'it fails with ValidationError' do
193
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
194
+ end
195
+ end
196
+
197
+ context 'with user id that is not a number' do
198
+ before :example do
199
+ data['user_id'] = 'string'
200
+ end
201
+
202
+ it 'it fails with ValidationError' do
203
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
204
+ end
205
+ end
206
+
207
+ context 'without group id' do
208
+ before :example do
209
+ data['group_id'] = nil
210
+ end
211
+
212
+ it 'it fails with ValidationError' do
213
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
214
+ end
215
+ end
216
+
217
+ context 'with group id that is not a number' do
218
+ before :example do
219
+ data['group_id'] = 'string'
220
+ end
221
+
222
+ it 'it fails with ValidationError' do
223
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
224
+ end
225
+ end
226
+
227
+ context 'without status_code' do
228
+ before :example do
229
+ data['status_code'] = nil
230
+ end
231
+
232
+ it 'it fails with ValidationError' do
233
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
234
+ end
235
+ end
236
+
237
+
238
+ context 'with status_code that is not a number' do
239
+ before :example do
240
+ data['status_code'] = 'string'
241
+ end
242
+
243
+ it 'it fails with ValidationError' do
244
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
245
+ end
246
+ end
247
+
248
+ context 'without cpu count' do
249
+ before :example do
250
+ data['cpu_count'] = nil
251
+ end
252
+
253
+ it 'it fails with ValidationError' do
254
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
255
+ end
256
+ end
257
+
258
+ context 'with cpu count that is not a number' do
259
+ before :example do
260
+ data['cpu_count'] = 'string'
261
+ end
262
+
263
+ it 'it fails with ValidationError' do
264
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
265
+ end
266
+ end
267
+
268
+ context 'without network inbound' do
269
+ before :example do
270
+ data['network_inbound'] = nil
271
+ end
272
+
273
+ it 'it fails with ValidationError' do
274
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
275
+ end
276
+ end
277
+
278
+ context 'with network inbound that is not a number' do
279
+ before :example do
280
+ data['network_inbound'] = 'string'
281
+ end
282
+
283
+ it 'it fails with ValidationError' do
284
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
285
+ end
286
+ end
287
+
288
+ context 'without network outbound' do
289
+ before :example do
290
+ data['network_outbound'] = nil
291
+ end
292
+
293
+ it 'it fails with ValidationError' do
294
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
295
+ end
296
+ end
297
+
298
+ context 'with network outbound that is not a number' do
299
+ before :example do
300
+ data['network_outbound'] = 'string'
301
+ end
302
+
303
+ it 'it fails with ValidationError' do
304
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
305
+ end
306
+ end
307
+
308
+ context 'without memory' do
309
+ before :example do
310
+ data['memory'] = nil
311
+ end
312
+
313
+ it 'it fails with ValidationError' do
314
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
315
+ end
316
+ end
317
+
318
+ context 'with memory that is not a number' do
319
+ before :example do
320
+ data['memory'] = 'string'
321
+ end
322
+
323
+ it 'it fails with ValidationError' do
324
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
325
+ end
326
+ end
327
+
328
+ context 'without disks' do
329
+ before :example do
330
+ data['disks'] = nil
331
+ end
332
+
333
+ it 'it fails with ValidationError' do
334
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
335
+ end
336
+ end
337
+
338
+ context 'without history' do
339
+ before :example do
340
+ data['history'] = nil
341
+ end
342
+
343
+ it 'it fails with ValidationError' do
344
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
345
+ end
346
+ end
347
+
348
+ context 'without start time in history record' do
349
+ before :example do
350
+ data['history'][0]['start_time'] = nil
351
+ end
352
+
353
+ it 'fails with ValidationError' do
354
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
355
+ end
356
+ end
357
+
358
+ context 'with start time in history record that is not a number' do
359
+ before :example do
360
+ data['history'][0]['start_time'] = 'string'
361
+ end
362
+
363
+ it 'fails with ValidationError' do
364
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
365
+ end
366
+ end
367
+
368
+ context 'with start time in history record that is zero' do
369
+ before :example do
370
+ data['history'][0]['start_time'] = '0'
371
+ end
372
+
373
+ it 'fails with ValidationError' do
374
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
375
+ end
376
+ end
377
+
378
+ context 'without end time in history record' do
379
+ before :example do
380
+ data['history'][0]['end_time'] = nil
381
+ end
382
+
383
+ it 'fails with ValidationError' do
384
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
385
+ end
386
+ end
387
+
388
+ context 'with end time in history record that is not a number' do
389
+ before :example do
390
+ data['history'][0]['end_time'] = 'string'
391
+ end
392
+
393
+ it 'fails with ValidationError' do
394
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
395
+ end
396
+ end
397
+
398
+ context 'without rstart time in history record' do
399
+ before :example do
400
+ data['history'][0]['rstart_time'] = nil
401
+ end
402
+
403
+ it 'fails with ValidationError' do
404
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
405
+ end
406
+ end
407
+
408
+ context 'with rstart time in history record that is not a number' do
409
+ before :example do
410
+ data['history'][0]['rstart_time'] = 'string'
411
+ end
412
+
413
+ it 'fails with ValidationError' do
414
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
415
+ end
416
+ end
417
+
418
+ context 'with rstart time in history record that is zero' do
419
+ before :example do
420
+ data['history'][0]['rstart_time'] = '0'
421
+ end
422
+
423
+ it 'fails with ValidationError' do
424
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
425
+ end
426
+ end
427
+
428
+ context 'without rend time in history record' do
429
+ before :example do
430
+ data['history'][0]['rend_time'] = nil
431
+ end
432
+
433
+ it 'fails with ValidationError' do
434
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
435
+ end
436
+ end
437
+
438
+ context 'with rend time in history record that is not a number' do
439
+ before :example do
440
+ data['history'][0]['rend_time'] = 'string'
441
+ end
442
+
443
+ it 'fails with ValidationError' do
444
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
445
+ end
446
+ end
447
+
448
+ context 'without sequence number in history record' do
449
+ before :example do
450
+ data['history'][0]['seq'] = nil
451
+ end
452
+
453
+ it 'fails with ValidationError' do
454
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
455
+ end
456
+ end
457
+
458
+ context 'with sequence number in history record that is not a number' do
459
+ before :example do
460
+ data['history'][0]['seq'] = 'string'
461
+ end
462
+
463
+ it 'fails with ValidationError' do
464
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
465
+ end
466
+ end
467
+ end
468
+ end
469
+ end