oneacct-export 0.2.7 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rspec +1 -0
  4. data/.travis.yml +1 -21
  5. data/bin/oneacct-export +7 -6
  6. data/config/conf.yml +25 -7
  7. data/lib/data_validators/apel_data_validator.rb +99 -0
  8. data/lib/data_validators/data_compute.rb +57 -0
  9. data/lib/data_validators/data_validator.rb +12 -0
  10. data/lib/data_validators/data_validator_helper.rb +15 -0
  11. data/lib/data_validators/logstash_data_validator.rb +82 -0
  12. data/lib/data_validators/pbs_data_validator.rb +86 -0
  13. data/lib/errors/not_implemented_error.rb +3 -0
  14. data/lib/errors/validation_error.rb +3 -0
  15. data/lib/errors.rb +2 -0
  16. data/lib/input_validator.rb +12 -2
  17. data/lib/one_data_accessor.rb +11 -10
  18. data/lib/one_worker.rb +109 -137
  19. data/lib/oneacct_exporter/version.rb +1 -1
  20. data/lib/oneacct_exporter.rb +9 -7
  21. data/lib/oneacct_opts.rb +36 -13
  22. data/lib/output_types.rb +5 -0
  23. data/lib/redis_conf.rb +2 -2
  24. data/lib/settings.rb +3 -3
  25. data/lib/templates/apel-0.2.erb +6 -6
  26. data/lib/templates/logstash-0.1.erb +3 -0
  27. data/lib/templates/pbs-0.1.erb +6 -0
  28. data/mock/{one_worker_vm8.xml → one_worker_vm_dn01.xml} +76 -74
  29. data/mock/one_worker_vm_dn02.xml +174 -0
  30. data/mock/{one_worker_DISK_missing.xml → one_worker_vm_empty_disk_records.xml} +10 -6
  31. data/mock/one_worker_vm_empty_history_records.xml +131 -0
  32. data/mock/one_worker_vm_image_name01.xml +175 -0
  33. data/mock/{one_worker_valid_machine.xml → one_worker_vm_image_name02.xml} +35 -7
  34. data/mock/one_worker_vm_image_name03.xml +167 -0
  35. data/mock/{one_worker_vm2.xml → one_worker_vm_image_name04.xml} +38 -9
  36. data/mock/{one_worker_vm1.xml → one_worker_vm_image_name05.xml} +36 -8
  37. data/mock/{one_worker_vm9.xml → one_worker_vm_image_name06.xml} +8 -5
  38. data/oneacct-export.gemspec +1 -0
  39. data/spec/data_validators/apel_data_validator_spec.rb +497 -0
  40. data/spec/data_validators/data_compute_spec.rb +193 -0
  41. data/spec/data_validators/data_validator_helper_spec.rb +66 -0
  42. data/spec/data_validators/data_validator_spec.rb +14 -0
  43. data/spec/data_validators/logstash_data_validator_spec.rb +469 -0
  44. data/spec/data_validators/pbs_data_validator_spec.rb +353 -0
  45. data/spec/one_worker_spec.rb +234 -542
  46. data/spec/oneacct_exporter_spec.rb +1 -41
  47. data/spec/oneacct_opts_spec.rb +135 -32
  48. data/spec/spec_helper.rb +18 -1
  49. metadata +51 -52
  50. data/mock/one_worker_DEPLOY_ID_missing.xml +0 -136
  51. data/mock/one_worker_DISK_SIZE_nan.xml +0 -147
  52. data/mock/one_worker_ETIME_0.xml +0 -137
  53. data/mock/one_worker_ETIME_missing.xml +0 -136
  54. data/mock/one_worker_ETIME_nan.xml +0 -137
  55. data/mock/one_worker_GID_missing.xml +0 -136
  56. data/mock/one_worker_GNAME_missing.xml +0 -136
  57. data/mock/one_worker_HISTORY_RECORDS_missing.xml +0 -91
  58. data/mock/one_worker_HISTORY_many.xml +0 -137
  59. data/mock/one_worker_HISTORY_missing.xml +0 -93
  60. data/mock/one_worker_HISTORY_one.xml +0 -115
  61. data/mock/one_worker_IMAGE_ID_missing.xml +0 -136
  62. data/mock/one_worker_MEMORY_0.xml +0 -137
  63. data/mock/one_worker_MEMORY_missing.xml +0 -135
  64. data/mock/one_worker_MEMORY_nan.xml +0 -137
  65. data/mock/one_worker_NET_RX_0.xml +0 -137
  66. data/mock/one_worker_NET_RX_missing.xml +0 -136
  67. data/mock/one_worker_NET_RX_nan.xml +0 -137
  68. data/mock/one_worker_NET_TX_0.xml +0 -137
  69. data/mock/one_worker_NET_TX_missing.xml +0 -136
  70. data/mock/one_worker_NET_TX_nan.xml +0 -137
  71. data/mock/one_worker_RETIME_0_RUNNING.xml +0 -115
  72. data/mock/one_worker_RETIME_0_STOPPED.xml +0 -115
  73. data/mock/one_worker_RETIME_missing.xml +0 -114
  74. data/mock/one_worker_RSTIME_0.xml +0 -115
  75. data/mock/one_worker_RSTIME_>_RETIME.xml +0 -115
  76. data/mock/one_worker_RSTIME_missing.xml +0 -114
  77. data/mock/one_worker_STATE_missing.xml +0 -136
  78. data/mock/one_worker_STATE_out_of_range.xml +0 -137
  79. data/mock/one_worker_STIME_>_ETIME.xml +0 -137
  80. data/mock/one_worker_STIME_missing.xml +0 -136
  81. data/mock/one_worker_STIME_nan.xml +0 -137
  82. data/mock/one_worker_TEMPLATE_missing.xml +0 -79
  83. data/mock/one_worker_UID_missing.xml +0 -136
  84. data/mock/one_worker_VCPU_0.xml +0 -137
  85. data/mock/one_worker_VCPU_missing.xml +0 -136
  86. data/mock/one_worker_VCPU_nan.xml +0 -137
  87. data/mock/one_worker_malformed_vm.xml +0 -136
  88. data/mock/one_worker_vm3.xml +0 -137
  89. data/mock/one_worker_vm4.xml +0 -106
  90. data/mock/one_worker_vm5.xml +0 -106
  91. data/mock/one_worker_vm6.xml +0 -107
  92. data/mock/one_worker_vm7.xml +0 -147
@@ -0,0 +1,66 @@
1
+ require 'spec_helper'
2
+
3
+ module DataValidators
4
+ describe DataValidatorHelper do
5
+ subject { data_validator_helper }
6
+
7
+ let(:data_validator_helper) do
8
+ data_validator_helper = Object.new
9
+ data_validator_helper.extend(Errors)
10
+ data_validator_helper.extend(InputValidator)
11
+ data_validator_helper.extend(DataValidatorHelper)
12
+
13
+ data_validator_helper
14
+ end
15
+
16
+ describe '.fail_validation' do
17
+ it 'fails with ValidationError' do
18
+ expect { subject.fail_validation('field') }.to raise_error(Errors::ValidationError)
19
+ end
20
+ end
21
+
22
+ describe '.default' do
23
+ context 'with :string condition method' do
24
+ context 'with string value' do
25
+ it 'returns back value' do
26
+ expect(subject.default('string', :string, 'default')).to eq('string')
27
+ end
28
+ end
29
+ end
30
+
31
+ context 'with :number condition method' do
32
+ context 'with number value' do
33
+ it 'returns back value' do
34
+ expect(subject.default(42, :number, 'default')).to eq(42)
35
+ end
36
+ end
37
+
38
+ context 'with non-number value' do
39
+ it 'returns default value' do
40
+ expect(subject.default('string', :number, 'default')).to eq('default')
41
+ end
42
+ end
43
+ end
44
+
45
+ context 'with :nzn condition method' do
46
+ context 'with non-zero number value' do
47
+ it 'returns back value' do
48
+ expect(subject.default(42, :nzn, 'default')).to eq(42)
49
+ end
50
+ end
51
+
52
+ context 'with zero value' do
53
+ it 'returns default value' do
54
+ expect(subject.default(0, :nzn, 'default')).to eq('default')
55
+ end
56
+ end
57
+
58
+ context 'with non-number value' do
59
+ it 'returns default value' do
60
+ expect(subject.default('string', :nzn, 'default')).to eq('default')
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,14 @@
1
+ require 'spec_helper'
2
+
3
+ module DataValidators
4
+ describe DataValidator do
5
+
6
+ subject { DataValidator.new }
7
+
8
+ describe '.validate_data' do
9
+ it 'fails with NotImplementedError' do
10
+ expect { subject.validate_data }.to raise_error(Errors::NotImplementedError)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,469 @@
1
+ require 'rspec'
2
+
3
+ module DataValidators
4
+ describe LogstashDataValidator do
5
+
6
+ subject { LogstashDataValidator.new(Logger.new('/dev/null')) }
7
+
8
+ describe '.validate_data' do
9
+ let(:data) do
10
+ data = {}
11
+
12
+ data['endpoint'] = 'machine.hogwarts.co.uk'
13
+ data['site_name'] = 'Hogwarts'
14
+ data['cloud_type'] = 'OpenNebula'
15
+
16
+ data['vm_uuid'] = '36551'
17
+ data['start_time'] = '1383741160'
18
+ data['end_time'] = '1383742270'
19
+ data['machine_name'] = 'one-36551'
20
+ data['user_id'] = '120'
21
+ data['group_id'] = '0'
22
+ data['user_name'] = 'uname'
23
+ data['group_name'] = 'gname'
24
+ data['status_code'] = '3'
25
+ data['status'] = 'STATUS'
26
+ data['cpu_count'] = '1'
27
+ data['network_inbound'] = '4154845418'
28
+ data['network_outbound'] = '6326418701'
29
+ data['memory'] = '1736960'
30
+ history = []
31
+ rec = {}
32
+ rec['start_time'] = '1383741169'
33
+ rec['end_time'] = '1383741259'
34
+ rec['rstart_time'] = '1383741278'
35
+ rec['rend_time'] = '1383741367'
36
+ rec['seq'] = '0'
37
+ rec['hostname'] = 'supermachine1.somewhere.com'
38
+ history << rec
39
+ rec = {}
40
+ rec['start_time'] = '1383741589'
41
+ rec['end_time'] = '1383742270'
42
+ rec['rstart_time'] = '1383741674'
43
+ rec['rend_time'] = '1383742270'
44
+ rec['seq'] = '1'
45
+ rec['hostname'] = 'supermachine2.somewhere.com'
46
+ history << rec
47
+ data['history'] = history
48
+ data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
49
+
50
+ data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
51
+ data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
52
+
53
+ data
54
+ end
55
+
56
+ let(:valid_data) do
57
+ valid_data = {}
58
+
59
+ valid_data['endpoint'] = 'machine.hogwarts.co.uk'
60
+ valid_data['site_name'] = 'Hogwarts'
61
+ valid_data['cloud_type'] = 'OpenNebula'
62
+
63
+ valid_data['vm_uuid'] = '36551'
64
+ valid_data['start_time'] = 1383741160
65
+ valid_data['end_time'] = 1383742270
66
+ valid_data['machine_name'] = 'one-36551'
67
+ valid_data['user_id'] = 120
68
+ valid_data['group_id'] = 0
69
+ valid_data['user_name'] = 'uname'
70
+ valid_data['group_name'] = 'gname'
71
+ valid_data['status_code'] = 3
72
+ valid_data['status'] = 'STATUS'
73
+ valid_data['cpu_count'] = 1
74
+ valid_data['network_inbound'] = 4154845418
75
+ valid_data['network_outbound'] = 6326418701
76
+ valid_data['memory'] = 1736960
77
+ history = []
78
+ rec = {}
79
+ rec['start_time'] = 1383741169
80
+ rec['end_time'] = 1383741259
81
+ rec['rstart_time'] = 1383741278
82
+ rec['rend_time'] = 1383741367
83
+ rec['seq'] = 0
84
+ rec['hostname'] = 'supermachine1.somewhere.com'
85
+ history << rec
86
+ rec = {}
87
+ rec['start_time'] = 1383741589
88
+ rec['end_time'] = 1383742270
89
+ rec['rstart_time'] = 1383741674
90
+ rec['rend_time'] = 1383742270
91
+ rec['seq'] = 1
92
+ rec['hostname'] = 'supermachine2.somewhere.com'
93
+ history << rec
94
+ valid_data['history'] = history
95
+ valid_data['disks'] = [{'size' => 10240}, {'size' => 42368}]
96
+
97
+ valid_data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
98
+ valid_data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
99
+
100
+ valid_data
101
+ end
102
+
103
+ context 'with all data valid' do
104
+ it 'returns the same data transformed if needed' do
105
+ expect(subject.validate_data(data)).to eq(valid_data)
106
+ end
107
+ end
108
+
109
+ context 'without any data' do
110
+ let(:data) { nil }
111
+
112
+ it 'fails with ValidationError' do
113
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
114
+ end
115
+ end
116
+
117
+ context 'with empty data' do
118
+ let(:data) { {} }
119
+
120
+ it 'fails with ValidationError' do
121
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
122
+ end
123
+ end
124
+
125
+ context 'without start time' do
126
+ before :example do
127
+ data['start_time'] = nil
128
+ end
129
+
130
+ it 'it fails with ValidationError' do
131
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
132
+ end
133
+ end
134
+
135
+ context 'with zero start time' do
136
+ before :example do
137
+ data['start_time'] = '0'
138
+ end
139
+
140
+ it 'it fails with ValidationError' do
141
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
142
+ end
143
+ end
144
+
145
+ context 'with start time that is not a number' do
146
+ before :example do
147
+ data['start_time'] = 'string'
148
+ end
149
+
150
+ it 'it fails with ValidationError' do
151
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
152
+ end
153
+ end
154
+
155
+ context 'without end time' do
156
+ before :example do
157
+ data['end_time'] = nil
158
+ end
159
+
160
+ it 'it fails with ValidationError' do
161
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
162
+ end
163
+ end
164
+
165
+ context 'with end time that is not a number' do
166
+ before :example do
167
+ data['end_time'] = 'string'
168
+ end
169
+
170
+ it 'it fails with ValidationError' do
171
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
172
+ end
173
+ end
174
+
175
+ context 'with start time bigger than end time' do
176
+ before :example do
177
+ time = data['end_time']
178
+ data['end_time'] = data['start_time']
179
+ data['start_time'] = time
180
+ end
181
+
182
+ it 'it fails with ValidationError' do
183
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
184
+ end
185
+ end
186
+
187
+ context 'without user id' do
188
+ before :example do
189
+ data['user_id'] = nil
190
+ end
191
+
192
+ it 'it fails with ValidationError' do
193
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
194
+ end
195
+ end
196
+
197
+ context 'with user id that is not a number' do
198
+ before :example do
199
+ data['user_id'] = 'string'
200
+ end
201
+
202
+ it 'it fails with ValidationError' do
203
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
204
+ end
205
+ end
206
+
207
+ context 'without group id' do
208
+ before :example do
209
+ data['group_id'] = nil
210
+ end
211
+
212
+ it 'it fails with ValidationError' do
213
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
214
+ end
215
+ end
216
+
217
+ context 'with group id that is not a number' do
218
+ before :example do
219
+ data['group_id'] = 'string'
220
+ end
221
+
222
+ it 'it fails with ValidationError' do
223
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
224
+ end
225
+ end
226
+
227
+ context 'without status_code' do
228
+ before :example do
229
+ data['status_code'] = nil
230
+ end
231
+
232
+ it 'it fails with ValidationError' do
233
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
234
+ end
235
+ end
236
+
237
+
238
+ context 'with status_code that is not a number' do
239
+ before :example do
240
+ data['status_code'] = 'string'
241
+ end
242
+
243
+ it 'it fails with ValidationError' do
244
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
245
+ end
246
+ end
247
+
248
+ context 'without cpu count' do
249
+ before :example do
250
+ data['cpu_count'] = nil
251
+ end
252
+
253
+ it 'it fails with ValidationError' do
254
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
255
+ end
256
+ end
257
+
258
+ context 'with cpu count that is not a number' do
259
+ before :example do
260
+ data['cpu_count'] = 'string'
261
+ end
262
+
263
+ it 'it fails with ValidationError' do
264
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
265
+ end
266
+ end
267
+
268
+ context 'without network inbound' do
269
+ before :example do
270
+ data['network_inbound'] = nil
271
+ end
272
+
273
+ it 'it fails with ValidationError' do
274
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
275
+ end
276
+ end
277
+
278
+ context 'with network inbound that is not a number' do
279
+ before :example do
280
+ data['network_inbound'] = 'string'
281
+ end
282
+
283
+ it 'it fails with ValidationError' do
284
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
285
+ end
286
+ end
287
+
288
+ context 'without network outbound' do
289
+ before :example do
290
+ data['network_outbound'] = nil
291
+ end
292
+
293
+ it 'it fails with ValidationError' do
294
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
295
+ end
296
+ end
297
+
298
+ context 'with network outbound that is not a number' do
299
+ before :example do
300
+ data['network_outbound'] = 'string'
301
+ end
302
+
303
+ it 'it fails with ValidationError' do
304
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
305
+ end
306
+ end
307
+
308
+ context 'without memory' do
309
+ before :example do
310
+ data['memory'] = nil
311
+ end
312
+
313
+ it 'it fails with ValidationError' do
314
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
315
+ end
316
+ end
317
+
318
+ context 'with memory that is not a number' do
319
+ before :example do
320
+ data['memory'] = 'string'
321
+ end
322
+
323
+ it 'it fails with ValidationError' do
324
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
325
+ end
326
+ end
327
+
328
+ context 'without disks' do
329
+ before :example do
330
+ data['disks'] = nil
331
+ end
332
+
333
+ it 'it fails with ValidationError' do
334
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
335
+ end
336
+ end
337
+
338
+ context 'without history' do
339
+ before :example do
340
+ data['history'] = nil
341
+ end
342
+
343
+ it 'it fails with ValidationError' do
344
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
345
+ end
346
+ end
347
+
348
+ context 'without start time in history record' do
349
+ before :example do
350
+ data['history'][0]['start_time'] = nil
351
+ end
352
+
353
+ it 'fails with ValidationError' do
354
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
355
+ end
356
+ end
357
+
358
+ context 'with start time in history record that is not a number' do
359
+ before :example do
360
+ data['history'][0]['start_time'] = 'string'
361
+ end
362
+
363
+ it 'fails with ValidationError' do
364
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
365
+ end
366
+ end
367
+
368
+ context 'with start time in history record that is zero' do
369
+ before :example do
370
+ data['history'][0]['start_time'] = '0'
371
+ end
372
+
373
+ it 'fails with ValidationError' do
374
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
375
+ end
376
+ end
377
+
378
+ context 'without end time in history record' do
379
+ before :example do
380
+ data['history'][0]['end_time'] = nil
381
+ end
382
+
383
+ it 'fails with ValidationError' do
384
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
385
+ end
386
+ end
387
+
388
+ context 'with end time in history record that is not a number' do
389
+ before :example do
390
+ data['history'][0]['end_time'] = 'string'
391
+ end
392
+
393
+ it 'fails with ValidationError' do
394
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
395
+ end
396
+ end
397
+
398
+ context 'without rstart time in history record' do
399
+ before :example do
400
+ data['history'][0]['rstart_time'] = nil
401
+ end
402
+
403
+ it 'fails with ValidationError' do
404
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
405
+ end
406
+ end
407
+
408
+ context 'with rstart time in history record that is not a number' do
409
+ before :example do
410
+ data['history'][0]['rstart_time'] = 'string'
411
+ end
412
+
413
+ it 'fails with ValidationError' do
414
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
415
+ end
416
+ end
417
+
418
+ context 'with rstart time in history record that is zero' do
419
+ before :example do
420
+ data['history'][0]['rstart_time'] = '0'
421
+ end
422
+
423
+ it 'fails with ValidationError' do
424
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
425
+ end
426
+ end
427
+
428
+ context 'without rend time in history record' do
429
+ before :example do
430
+ data['history'][0]['rend_time'] = nil
431
+ end
432
+
433
+ it 'fails with ValidationError' do
434
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
435
+ end
436
+ end
437
+
438
+ context 'with rend time in history record that is not a number' do
439
+ before :example do
440
+ data['history'][0]['rend_time'] = 'string'
441
+ end
442
+
443
+ it 'fails with ValidationError' do
444
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
445
+ end
446
+ end
447
+
448
+ context 'without sequence number in history record' do
449
+ before :example do
450
+ data['history'][0]['seq'] = nil
451
+ end
452
+
453
+ it 'fails with ValidationError' do
454
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
455
+ end
456
+ end
457
+
458
+ context 'with sequence number in history record that is not a number' do
459
+ before :example do
460
+ data['history'][0]['seq'] = 'string'
461
+ end
462
+
463
+ it 'fails with ValidationError' do
464
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
465
+ end
466
+ end
467
+ end
468
+ end
469
+ end