oneacct-export 0.2.7 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rspec +1 -0
  4. data/.travis.yml +1 -21
  5. data/bin/oneacct-export +7 -6
  6. data/config/conf.yml +25 -7
  7. data/lib/data_validators/apel_data_validator.rb +99 -0
  8. data/lib/data_validators/data_compute.rb +57 -0
  9. data/lib/data_validators/data_validator.rb +12 -0
  10. data/lib/data_validators/data_validator_helper.rb +15 -0
  11. data/lib/data_validators/logstash_data_validator.rb +82 -0
  12. data/lib/data_validators/pbs_data_validator.rb +86 -0
  13. data/lib/errors/not_implemented_error.rb +3 -0
  14. data/lib/errors/validation_error.rb +3 -0
  15. data/lib/errors.rb +2 -0
  16. data/lib/input_validator.rb +12 -2
  17. data/lib/one_data_accessor.rb +11 -10
  18. data/lib/one_worker.rb +109 -137
  19. data/lib/oneacct_exporter/version.rb +1 -1
  20. data/lib/oneacct_exporter.rb +9 -7
  21. data/lib/oneacct_opts.rb +36 -13
  22. data/lib/output_types.rb +5 -0
  23. data/lib/redis_conf.rb +2 -2
  24. data/lib/settings.rb +3 -3
  25. data/lib/templates/apel-0.2.erb +6 -6
  26. data/lib/templates/logstash-0.1.erb +3 -0
  27. data/lib/templates/pbs-0.1.erb +6 -0
  28. data/mock/{one_worker_vm8.xml → one_worker_vm_dn01.xml} +76 -74
  29. data/mock/one_worker_vm_dn02.xml +174 -0
  30. data/mock/{one_worker_DISK_missing.xml → one_worker_vm_empty_disk_records.xml} +10 -6
  31. data/mock/one_worker_vm_empty_history_records.xml +131 -0
  32. data/mock/one_worker_vm_image_name01.xml +175 -0
  33. data/mock/{one_worker_valid_machine.xml → one_worker_vm_image_name02.xml} +35 -7
  34. data/mock/one_worker_vm_image_name03.xml +167 -0
  35. data/mock/{one_worker_vm2.xml → one_worker_vm_image_name04.xml} +38 -9
  36. data/mock/{one_worker_vm1.xml → one_worker_vm_image_name05.xml} +36 -8
  37. data/mock/{one_worker_vm9.xml → one_worker_vm_image_name06.xml} +8 -5
  38. data/oneacct-export.gemspec +1 -0
  39. data/spec/data_validators/apel_data_validator_spec.rb +497 -0
  40. data/spec/data_validators/data_compute_spec.rb +193 -0
  41. data/spec/data_validators/data_validator_helper_spec.rb +66 -0
  42. data/spec/data_validators/data_validator_spec.rb +14 -0
  43. data/spec/data_validators/logstash_data_validator_spec.rb +469 -0
  44. data/spec/data_validators/pbs_data_validator_spec.rb +353 -0
  45. data/spec/one_worker_spec.rb +234 -542
  46. data/spec/oneacct_exporter_spec.rb +1 -41
  47. data/spec/oneacct_opts_spec.rb +135 -32
  48. data/spec/spec_helper.rb +18 -1
  49. metadata +51 -52
  50. data/mock/one_worker_DEPLOY_ID_missing.xml +0 -136
  51. data/mock/one_worker_DISK_SIZE_nan.xml +0 -147
  52. data/mock/one_worker_ETIME_0.xml +0 -137
  53. data/mock/one_worker_ETIME_missing.xml +0 -136
  54. data/mock/one_worker_ETIME_nan.xml +0 -137
  55. data/mock/one_worker_GID_missing.xml +0 -136
  56. data/mock/one_worker_GNAME_missing.xml +0 -136
  57. data/mock/one_worker_HISTORY_RECORDS_missing.xml +0 -91
  58. data/mock/one_worker_HISTORY_many.xml +0 -137
  59. data/mock/one_worker_HISTORY_missing.xml +0 -93
  60. data/mock/one_worker_HISTORY_one.xml +0 -115
  61. data/mock/one_worker_IMAGE_ID_missing.xml +0 -136
  62. data/mock/one_worker_MEMORY_0.xml +0 -137
  63. data/mock/one_worker_MEMORY_missing.xml +0 -135
  64. data/mock/one_worker_MEMORY_nan.xml +0 -137
  65. data/mock/one_worker_NET_RX_0.xml +0 -137
  66. data/mock/one_worker_NET_RX_missing.xml +0 -136
  67. data/mock/one_worker_NET_RX_nan.xml +0 -137
  68. data/mock/one_worker_NET_TX_0.xml +0 -137
  69. data/mock/one_worker_NET_TX_missing.xml +0 -136
  70. data/mock/one_worker_NET_TX_nan.xml +0 -137
  71. data/mock/one_worker_RETIME_0_RUNNING.xml +0 -115
  72. data/mock/one_worker_RETIME_0_STOPPED.xml +0 -115
  73. data/mock/one_worker_RETIME_missing.xml +0 -114
  74. data/mock/one_worker_RSTIME_0.xml +0 -115
  75. data/mock/one_worker_RSTIME_>_RETIME.xml +0 -115
  76. data/mock/one_worker_RSTIME_missing.xml +0 -114
  77. data/mock/one_worker_STATE_missing.xml +0 -136
  78. data/mock/one_worker_STATE_out_of_range.xml +0 -137
  79. data/mock/one_worker_STIME_>_ETIME.xml +0 -137
  80. data/mock/one_worker_STIME_missing.xml +0 -136
  81. data/mock/one_worker_STIME_nan.xml +0 -137
  82. data/mock/one_worker_TEMPLATE_missing.xml +0 -79
  83. data/mock/one_worker_UID_missing.xml +0 -136
  84. data/mock/one_worker_VCPU_0.xml +0 -137
  85. data/mock/one_worker_VCPU_missing.xml +0 -136
  86. data/mock/one_worker_VCPU_nan.xml +0 -137
  87. data/mock/one_worker_malformed_vm.xml +0 -136
  88. data/mock/one_worker_vm3.xml +0 -137
  89. data/mock/one_worker_vm4.xml +0 -106
  90. data/mock/one_worker_vm5.xml +0 -106
  91. data/mock/one_worker_vm6.xml +0 -107
  92. data/mock/one_worker_vm7.xml +0 -147
@@ -0,0 +1,353 @@
1
+ require 'spec_helper'
2
+
3
+ module DataValidators
4
+ describe PbsDataValidator do
5
+
6
+ subject { PbsDataValidator.new(Logger.new('/dev/null')) }
7
+
8
+ describe '.validate_data' do
9
+ let(:data) do
10
+ data = {}
11
+
12
+ data['pbs_queue'] = 'queue'
13
+ data['realm'] = 'REALM'
14
+ data['host'] = 'on_localhost'
15
+ data['scratch_type'] = 'local'
16
+
17
+ data['vm_uuid'] = '36551'
18
+ data['start_time'] = '1383741160'
19
+ data['end_time'] = '1383742270'
20
+ data['machine_name'] = 'one-36551'
21
+ data['user_id'] = '120'
22
+ data['group_id'] = '0'
23
+ data['user_name'] = 'uname'
24
+ data['group_name'] = 'gname'
25
+ data['status'] = '3'
26
+ data['cpu_count'] = '1'
27
+ data['network_inbound'] = '4154845418'
28
+ data['network_outbound'] = '6326418701'
29
+ data['memory'] = '1736960'
30
+ history = []
31
+ rec = {}
32
+ rec['start_time'] = '1383741169'
33
+ rec['end_time'] = '1383741259'
34
+ rec['rstart_time'] = '1383741278'
35
+ rec['rend_time'] = '1383741367'
36
+ rec['seq'] = '0'
37
+ rec['hostname'] = 'supermachine1.somewhere.com'
38
+ history << rec
39
+ rec = {}
40
+ rec['start_time'] = '1383741589'
41
+ rec['end_time'] = '1383742270'
42
+ rec['rstart_time'] = '1383741674'
43
+ rec['rend_time'] = '1383742270'
44
+ rec['seq'] = '1'
45
+ rec['hostname'] = 'supermachine2.somewhere.com'
46
+ history << rec
47
+ data['history'] = history
48
+ data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
49
+
50
+ data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
51
+ data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
52
+
53
+ data
54
+ end
55
+
56
+ let(:valid_data) do
57
+ valid_data = {}
58
+
59
+ valid_data['pbs_queue'] = 'queue'
60
+ valid_data['realm'] = 'REALM'
61
+ valid_data['host'] = 'on_localhost'
62
+ valid_data['scratch_type'] = 'local'
63
+
64
+ valid_data['vm_uuid'] = '36551'
65
+ valid_data['start_time'] = '1383741160'
66
+ valid_data['end_time'] = '1383742270'
67
+ valid_data['machine_name'] = 'one-36551'
68
+ valid_data['user_id'] = '120'
69
+ valid_data['group_id'] = '0'
70
+ valid_data['user_name'] = 'uname'
71
+ valid_data['group_name'] = 'gname'
72
+ valid_data['status'] = '3'
73
+ valid_data['cpu_count'] = '1'
74
+ valid_data['network_inbound'] = '4154845418'
75
+ valid_data['network_outbound'] = '6326418701'
76
+ valid_data['memory'] = '1736960'
77
+ history = []
78
+ rec = {}
79
+ rec['start_time'] = Time.at(1383741169)
80
+ rec['end_time'] = Time.at(1383741259)
81
+ rec['rstart_time'] = '1383741278'
82
+ rec['rend_time'] = '1383741367'
83
+ rec['seq'] = '0'
84
+ rec['hostname'] = 'supermachine1.somewhere.com'
85
+ rec['state'] = 'U'
86
+ history << rec
87
+ rec = {}
88
+ rec['start_time'] = Time.at(1383741589)
89
+ rec['end_time'] = Time.at(1383742270)
90
+ rec['rstart_time'] = '1383741674'
91
+ rec['rend_time'] = '1383742270'
92
+ rec['seq'] = '1'
93
+ rec['hostname'] = 'supermachine2.somewhere.com'
94
+ rec['state'] = 'U'
95
+ history << rec
96
+ valid_data['history'] = history
97
+ valid_data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
98
+
99
+ valid_data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
100
+ valid_data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
101
+
102
+ valid_data['duration'] = Time.at(685)
103
+ valid_data['disk_size'] = 52608
104
+
105
+ valid_data
106
+ end
107
+
108
+ context 'with all data valid' do
109
+ it 'returns the same data transformed if needed' do
110
+ expect(subject.validate_data(data)).to eq(valid_data)
111
+ end
112
+ end
113
+
114
+ context 'without any data' do
115
+ let(:data) { nil }
116
+
117
+ it 'fails with ValidationError' do
118
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
119
+ end
120
+ end
121
+
122
+ context 'with empty data' do
123
+ let(:data) { {} }
124
+
125
+ it 'fails with ValidationError' do
126
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
127
+ end
128
+ end
129
+
130
+ context 'without host' do
131
+ before :example do
132
+ data['host'] = nil
133
+ end
134
+
135
+ it 'fails with ValidationError' do
136
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
137
+ end
138
+ end
139
+
140
+ context 'without queue' do
141
+ before :example do
142
+ data['pbs_queue'] = nil
143
+ end
144
+
145
+ it 'fails with ValidationError' do
146
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
147
+ end
148
+ end
149
+
150
+ context 'without realm' do
151
+ before :example do
152
+ data['realm'] = nil
153
+ end
154
+
155
+ it 'fails with ValidationError' do
156
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
157
+ end
158
+ end
159
+
160
+ context 'without vm uuid' do
161
+ before :example do
162
+ data['vm_uuid'] = nil
163
+ end
164
+
165
+ it 'fails with ValidationError' do
166
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
167
+ end
168
+ end
169
+
170
+ context 'without user name' do
171
+ before :example do
172
+ data['user_name'] = nil
173
+ end
174
+
175
+ it 'fails with ValidationError' do
176
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
177
+ end
178
+ end
179
+
180
+ context 'without group name' do
181
+ before :example do
182
+ data['group_name'] = nil
183
+ end
184
+
185
+ it 'fails with ValidationError' do
186
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
187
+ end
188
+ end
189
+
190
+ context 'without cpu count' do
191
+ before :example do
192
+ data['cpu_count'] = nil
193
+ end
194
+
195
+ it 'fails with ValidationError' do
196
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
197
+ end
198
+ end
199
+
200
+ context 'with cpu count that is not a number' do
201
+ before :example do
202
+ data['cpu_count'] = 'string'
203
+ end
204
+
205
+ it 'fails with ValidationError' do
206
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
207
+ end
208
+ end
209
+
210
+ context 'without memory' do
211
+ before :example do
212
+ data['memory'] = nil
213
+ end
214
+
215
+ it 'fails with ValidationError' do
216
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
217
+ end
218
+ end
219
+
220
+ context 'with memory that is not a number' do
221
+ before :example do
222
+ data['memory'] = 'string'
223
+ end
224
+
225
+ it 'fails with ValidationError' do
226
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
227
+ end
228
+ end
229
+
230
+ context 'without history records' do
231
+ before :example do
232
+ data['history'] = nil
233
+ end
234
+
235
+ it 'fails with ValidationError' do
236
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
237
+ end
238
+ end
239
+
240
+ context 'with empty history records' do
241
+ before :example do
242
+ data['history'] = []
243
+ end
244
+
245
+ it 'fails with ValidationError' do
246
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
247
+ end
248
+ end
249
+
250
+ context 'without start time in history record' do
251
+ before :example do
252
+ data['history'][0]['start_time'] = nil
253
+ end
254
+
255
+ it 'fails with ValidationError' do
256
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
257
+ end
258
+ end
259
+
260
+ context 'with start time in history record that is not a number' do
261
+ before :example do
262
+ data['history'][0]['start_time'] = 'string'
263
+ end
264
+
265
+ it 'fails with ValidationError' do
266
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
267
+ end
268
+ end
269
+
270
+ context 'with start time in history record that is zero' do
271
+ before :example do
272
+ data['history'][0]['start_time'] = '0'
273
+ end
274
+
275
+ it 'fails with ValidationError' do
276
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
277
+ end
278
+ end
279
+
280
+ context 'without end time in history record' do
281
+ before :example do
282
+ data['history'][0]['end_time'] = nil
283
+ end
284
+
285
+ it 'fails with ValidationError' do
286
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
287
+ end
288
+ end
289
+
290
+ context 'with end time in history record that is not a number' do
291
+ before :example do
292
+ data['history'][0]['end_time'] = 'string'
293
+ end
294
+
295
+ it 'fails with ValidationError' do
296
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
297
+ end
298
+ end
299
+
300
+ context 'without sequence number in history record' do
301
+ before :example do
302
+ data['history'][0]['seq'] = nil
303
+ end
304
+
305
+ it 'fails with ValidationError' do
306
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
307
+ end
308
+ end
309
+
310
+ context 'with sequence number in history record that is not a number' do
311
+ before :example do
312
+ data['history'][0]['seq'] = 'string'
313
+ end
314
+
315
+ it 'fails with ValidationError' do
316
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
317
+ end
318
+ end
319
+
320
+ context 'without hostname in history record' do
321
+ before :example do
322
+ data['history'][0]['hostname'] = nil
323
+ end
324
+
325
+ it 'fails with ValidationError' do
326
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
327
+ end
328
+ end
329
+
330
+ context 'with stopped virtual machine' do
331
+ before :example do
332
+ data['status_code'] = '6'
333
+ valid_data['history'][1]['state'] = 'E'
334
+ valid_data['status_code'] = '6'
335
+ end
336
+
337
+ it 'sets last history record\'s state to "E"' do
338
+ expect(subject.validate_data(data)).to eq(valid_data)
339
+ end
340
+ end
341
+
342
+ context 'without machine name' do
343
+ before :example do
344
+ data['machine_name'] = nil
345
+ end
346
+
347
+ it 'replaces machine name with string created from id and prefix "one-"' do
348
+ expect(subject.validate_data(data)).to eq(valid_data)
349
+ end
350
+ end
351
+ end
352
+ end
353
+ end