oneacct-export 0.2.7 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rspec +1 -0
  4. data/.travis.yml +1 -21
  5. data/bin/oneacct-export +7 -6
  6. data/config/conf.yml +25 -7
  7. data/lib/data_validators/apel_data_validator.rb +99 -0
  8. data/lib/data_validators/data_compute.rb +57 -0
  9. data/lib/data_validators/data_validator.rb +12 -0
  10. data/lib/data_validators/data_validator_helper.rb +15 -0
  11. data/lib/data_validators/logstash_data_validator.rb +82 -0
  12. data/lib/data_validators/pbs_data_validator.rb +86 -0
  13. data/lib/errors/not_implemented_error.rb +3 -0
  14. data/lib/errors/validation_error.rb +3 -0
  15. data/lib/errors.rb +2 -0
  16. data/lib/input_validator.rb +12 -2
  17. data/lib/one_data_accessor.rb +11 -10
  18. data/lib/one_worker.rb +109 -137
  19. data/lib/oneacct_exporter/version.rb +1 -1
  20. data/lib/oneacct_exporter.rb +9 -7
  21. data/lib/oneacct_opts.rb +36 -13
  22. data/lib/output_types.rb +5 -0
  23. data/lib/redis_conf.rb +2 -2
  24. data/lib/settings.rb +3 -3
  25. data/lib/templates/apel-0.2.erb +6 -6
  26. data/lib/templates/logstash-0.1.erb +3 -0
  27. data/lib/templates/pbs-0.1.erb +6 -0
  28. data/mock/{one_worker_vm8.xml → one_worker_vm_dn01.xml} +76 -74
  29. data/mock/one_worker_vm_dn02.xml +174 -0
  30. data/mock/{one_worker_DISK_missing.xml → one_worker_vm_empty_disk_records.xml} +10 -6
  31. data/mock/one_worker_vm_empty_history_records.xml +131 -0
  32. data/mock/one_worker_vm_image_name01.xml +175 -0
  33. data/mock/{one_worker_valid_machine.xml → one_worker_vm_image_name02.xml} +35 -7
  34. data/mock/one_worker_vm_image_name03.xml +167 -0
  35. data/mock/{one_worker_vm2.xml → one_worker_vm_image_name04.xml} +38 -9
  36. data/mock/{one_worker_vm1.xml → one_worker_vm_image_name05.xml} +36 -8
  37. data/mock/{one_worker_vm9.xml → one_worker_vm_image_name06.xml} +8 -5
  38. data/oneacct-export.gemspec +1 -0
  39. data/spec/data_validators/apel_data_validator_spec.rb +497 -0
  40. data/spec/data_validators/data_compute_spec.rb +193 -0
  41. data/spec/data_validators/data_validator_helper_spec.rb +66 -0
  42. data/spec/data_validators/data_validator_spec.rb +14 -0
  43. data/spec/data_validators/logstash_data_validator_spec.rb +469 -0
  44. data/spec/data_validators/pbs_data_validator_spec.rb +353 -0
  45. data/spec/one_worker_spec.rb +234 -542
  46. data/spec/oneacct_exporter_spec.rb +1 -41
  47. data/spec/oneacct_opts_spec.rb +135 -32
  48. data/spec/spec_helper.rb +18 -1
  49. metadata +51 -52
  50. data/mock/one_worker_DEPLOY_ID_missing.xml +0 -136
  51. data/mock/one_worker_DISK_SIZE_nan.xml +0 -147
  52. data/mock/one_worker_ETIME_0.xml +0 -137
  53. data/mock/one_worker_ETIME_missing.xml +0 -136
  54. data/mock/one_worker_ETIME_nan.xml +0 -137
  55. data/mock/one_worker_GID_missing.xml +0 -136
  56. data/mock/one_worker_GNAME_missing.xml +0 -136
  57. data/mock/one_worker_HISTORY_RECORDS_missing.xml +0 -91
  58. data/mock/one_worker_HISTORY_many.xml +0 -137
  59. data/mock/one_worker_HISTORY_missing.xml +0 -93
  60. data/mock/one_worker_HISTORY_one.xml +0 -115
  61. data/mock/one_worker_IMAGE_ID_missing.xml +0 -136
  62. data/mock/one_worker_MEMORY_0.xml +0 -137
  63. data/mock/one_worker_MEMORY_missing.xml +0 -135
  64. data/mock/one_worker_MEMORY_nan.xml +0 -137
  65. data/mock/one_worker_NET_RX_0.xml +0 -137
  66. data/mock/one_worker_NET_RX_missing.xml +0 -136
  67. data/mock/one_worker_NET_RX_nan.xml +0 -137
  68. data/mock/one_worker_NET_TX_0.xml +0 -137
  69. data/mock/one_worker_NET_TX_missing.xml +0 -136
  70. data/mock/one_worker_NET_TX_nan.xml +0 -137
  71. data/mock/one_worker_RETIME_0_RUNNING.xml +0 -115
  72. data/mock/one_worker_RETIME_0_STOPPED.xml +0 -115
  73. data/mock/one_worker_RETIME_missing.xml +0 -114
  74. data/mock/one_worker_RSTIME_0.xml +0 -115
  75. data/mock/one_worker_RSTIME_>_RETIME.xml +0 -115
  76. data/mock/one_worker_RSTIME_missing.xml +0 -114
  77. data/mock/one_worker_STATE_missing.xml +0 -136
  78. data/mock/one_worker_STATE_out_of_range.xml +0 -137
  79. data/mock/one_worker_STIME_>_ETIME.xml +0 -137
  80. data/mock/one_worker_STIME_missing.xml +0 -136
  81. data/mock/one_worker_STIME_nan.xml +0 -137
  82. data/mock/one_worker_TEMPLATE_missing.xml +0 -79
  83. data/mock/one_worker_UID_missing.xml +0 -136
  84. data/mock/one_worker_VCPU_0.xml +0 -137
  85. data/mock/one_worker_VCPU_missing.xml +0 -136
  86. data/mock/one_worker_VCPU_nan.xml +0 -137
  87. data/mock/one_worker_malformed_vm.xml +0 -136
  88. data/mock/one_worker_vm3.xml +0 -137
  89. data/mock/one_worker_vm4.xml +0 -106
  90. data/mock/one_worker_vm5.xml +0 -106
  91. data/mock/one_worker_vm6.xml +0 -107
  92. data/mock/one_worker_vm7.xml +0 -147
@@ -0,0 +1,353 @@
1
+ require 'spec_helper'
2
+
3
+ module DataValidators
4
+ describe PbsDataValidator do
5
+
6
+ subject { PbsDataValidator.new(Logger.new('/dev/null')) }
7
+
8
+ describe '.validate_data' do
9
+ let(:data) do
10
+ data = {}
11
+
12
+ data['pbs_queue'] = 'queue'
13
+ data['realm'] = 'REALM'
14
+ data['host'] = 'on_localhost'
15
+ data['scratch_type'] = 'local'
16
+
17
+ data['vm_uuid'] = '36551'
18
+ data['start_time'] = '1383741160'
19
+ data['end_time'] = '1383742270'
20
+ data['machine_name'] = 'one-36551'
21
+ data['user_id'] = '120'
22
+ data['group_id'] = '0'
23
+ data['user_name'] = 'uname'
24
+ data['group_name'] = 'gname'
25
+ data['status'] = '3'
26
+ data['cpu_count'] = '1'
27
+ data['network_inbound'] = '4154845418'
28
+ data['network_outbound'] = '6326418701'
29
+ data['memory'] = '1736960'
30
+ history = []
31
+ rec = {}
32
+ rec['start_time'] = '1383741169'
33
+ rec['end_time'] = '1383741259'
34
+ rec['rstart_time'] = '1383741278'
35
+ rec['rend_time'] = '1383741367'
36
+ rec['seq'] = '0'
37
+ rec['hostname'] = 'supermachine1.somewhere.com'
38
+ history << rec
39
+ rec = {}
40
+ rec['start_time'] = '1383741589'
41
+ rec['end_time'] = '1383742270'
42
+ rec['rstart_time'] = '1383741674'
43
+ rec['rend_time'] = '1383742270'
44
+ rec['seq'] = '1'
45
+ rec['hostname'] = 'supermachine2.somewhere.com'
46
+ history << rec
47
+ data['history'] = history
48
+ data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
49
+
50
+ data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
51
+ data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
52
+
53
+ data
54
+ end
55
+
56
+ let(:valid_data) do
57
+ valid_data = {}
58
+
59
+ valid_data['pbs_queue'] = 'queue'
60
+ valid_data['realm'] = 'REALM'
61
+ valid_data['host'] = 'on_localhost'
62
+ valid_data['scratch_type'] = 'local'
63
+
64
+ valid_data['vm_uuid'] = '36551'
65
+ valid_data['start_time'] = '1383741160'
66
+ valid_data['end_time'] = '1383742270'
67
+ valid_data['machine_name'] = 'one-36551'
68
+ valid_data['user_id'] = '120'
69
+ valid_data['group_id'] = '0'
70
+ valid_data['user_name'] = 'uname'
71
+ valid_data['group_name'] = 'gname'
72
+ valid_data['status'] = '3'
73
+ valid_data['cpu_count'] = '1'
74
+ valid_data['network_inbound'] = '4154845418'
75
+ valid_data['network_outbound'] = '6326418701'
76
+ valid_data['memory'] = '1736960'
77
+ history = []
78
+ rec = {}
79
+ rec['start_time'] = Time.at(1383741169)
80
+ rec['end_time'] = Time.at(1383741259)
81
+ rec['rstart_time'] = '1383741278'
82
+ rec['rend_time'] = '1383741367'
83
+ rec['seq'] = '0'
84
+ rec['hostname'] = 'supermachine1.somewhere.com'
85
+ rec['state'] = 'U'
86
+ history << rec
87
+ rec = {}
88
+ rec['start_time'] = Time.at(1383741589)
89
+ rec['end_time'] = Time.at(1383742270)
90
+ rec['rstart_time'] = '1383741674'
91
+ rec['rend_time'] = '1383742270'
92
+ rec['seq'] = '1'
93
+ rec['hostname'] = 'supermachine2.somewhere.com'
94
+ rec['state'] = 'U'
95
+ history << rec
96
+ valid_data['history'] = history
97
+ valid_data['disks'] = [{'size' => '10240'}, {'size' => '42368'}]
98
+
99
+ valid_data['user_dn'] = '/MY=STuPID/CN=DN/CN=HERE'
100
+ valid_data['image_name'] = 'https://appdb.egi.eu/store/vo/image/blablablabla/'
101
+
102
+ valid_data['duration'] = Time.at(685)
103
+ valid_data['disk_size'] = 52608
104
+
105
+ valid_data
106
+ end
107
+
108
+ context 'with all data valid' do
109
+ it 'returns the same data transformed if needed' do
110
+ expect(subject.validate_data(data)).to eq(valid_data)
111
+ end
112
+ end
113
+
114
+ context 'without any data' do
115
+ let(:data) { nil }
116
+
117
+ it 'fails with ValidationError' do
118
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
119
+ end
120
+ end
121
+
122
+ context 'with empty data' do
123
+ let(:data) { {} }
124
+
125
+ it 'fails with ValidationError' do
126
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
127
+ end
128
+ end
129
+
130
+ context 'without host' do
131
+ before :example do
132
+ data['host'] = nil
133
+ end
134
+
135
+ it 'fails with ValidationError' do
136
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
137
+ end
138
+ end
139
+
140
+ context 'without queue' do
141
+ before :example do
142
+ data['pbs_queue'] = nil
143
+ end
144
+
145
+ it 'fails with ValidationError' do
146
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
147
+ end
148
+ end
149
+
150
+ context 'without realm' do
151
+ before :example do
152
+ data['realm'] = nil
153
+ end
154
+
155
+ it 'fails with ValidationError' do
156
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
157
+ end
158
+ end
159
+
160
+ context 'without vm uuid' do
161
+ before :example do
162
+ data['vm_uuid'] = nil
163
+ end
164
+
165
+ it 'fails with ValidationError' do
166
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
167
+ end
168
+ end
169
+
170
+ context 'without user name' do
171
+ before :example do
172
+ data['user_name'] = nil
173
+ end
174
+
175
+ it 'fails with ValidationError' do
176
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
177
+ end
178
+ end
179
+
180
+ context 'without group name' do
181
+ before :example do
182
+ data['group_name'] = nil
183
+ end
184
+
185
+ it 'fails with ValidationError' do
186
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
187
+ end
188
+ end
189
+
190
+ context 'without cpu count' do
191
+ before :example do
192
+ data['cpu_count'] = nil
193
+ end
194
+
195
+ it 'fails with ValidationError' do
196
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
197
+ end
198
+ end
199
+
200
+ context 'with cpu count that is not a number' do
201
+ before :example do
202
+ data['cpu_count'] = 'string'
203
+ end
204
+
205
+ it 'fails with ValidationError' do
206
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
207
+ end
208
+ end
209
+
210
+ context 'without memory' do
211
+ before :example do
212
+ data['memory'] = nil
213
+ end
214
+
215
+ it 'fails with ValidationError' do
216
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
217
+ end
218
+ end
219
+
220
+ context 'with memory that is not a number' do
221
+ before :example do
222
+ data['memory'] = 'string'
223
+ end
224
+
225
+ it 'fails with ValidationError' do
226
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
227
+ end
228
+ end
229
+
230
+ context 'without history records' do
231
+ before :example do
232
+ data['history'] = nil
233
+ end
234
+
235
+ it 'fails with ValidationError' do
236
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
237
+ end
238
+ end
239
+
240
+ context 'with empty history records' do
241
+ before :example do
242
+ data['history'] = []
243
+ end
244
+
245
+ it 'fails with ValidationError' do
246
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
247
+ end
248
+ end
249
+
250
+ context 'without start time in history record' do
251
+ before :example do
252
+ data['history'][0]['start_time'] = nil
253
+ end
254
+
255
+ it 'fails with ValidationError' do
256
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
257
+ end
258
+ end
259
+
260
+ context 'with start time in history record that is not a number' do
261
+ before :example do
262
+ data['history'][0]['start_time'] = 'string'
263
+ end
264
+
265
+ it 'fails with ValidationError' do
266
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
267
+ end
268
+ end
269
+
270
+ context 'with start time in history record that is zero' do
271
+ before :example do
272
+ data['history'][0]['start_time'] = '0'
273
+ end
274
+
275
+ it 'fails with ValidationError' do
276
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
277
+ end
278
+ end
279
+
280
+ context 'without end time in history record' do
281
+ before :example do
282
+ data['history'][0]['end_time'] = nil
283
+ end
284
+
285
+ it 'fails with ValidationError' do
286
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
287
+ end
288
+ end
289
+
290
+ context 'with end time in history record that is not a number' do
291
+ before :example do
292
+ data['history'][0]['end_time'] = 'string'
293
+ end
294
+
295
+ it 'fails with ValidationError' do
296
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
297
+ end
298
+ end
299
+
300
+ context 'without sequence number in history record' do
301
+ before :example do
302
+ data['history'][0]['seq'] = nil
303
+ end
304
+
305
+ it 'fails with ValidationError' do
306
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
307
+ end
308
+ end
309
+
310
+ context 'with sequence number in history record that is not a number' do
311
+ before :example do
312
+ data['history'][0]['seq'] = 'string'
313
+ end
314
+
315
+ it 'fails with ValidationError' do
316
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
317
+ end
318
+ end
319
+
320
+ context 'without hostname in history record' do
321
+ before :example do
322
+ data['history'][0]['hostname'] = nil
323
+ end
324
+
325
+ it 'fails with ValidationError' do
326
+ expect { subject.validate_data(data) }.to raise_error(Errors::ValidationError)
327
+ end
328
+ end
329
+
330
+ context 'with stopped virtual machine' do
331
+ before :example do
332
+ data['status_code'] = '6'
333
+ valid_data['history'][1]['state'] = 'E'
334
+ valid_data['status_code'] = '6'
335
+ end
336
+
337
+ it 'sets last history record\'s state to "E"' do
338
+ expect(subject.validate_data(data)).to eq(valid_data)
339
+ end
340
+ end
341
+
342
+ context 'without machine name' do
343
+ before :example do
344
+ data['machine_name'] = nil
345
+ end
346
+
347
+ it 'replaces machine name with string created from id and prefix "one-"' do
348
+ expect(subject.validate_data(data)).to eq(valid_data)
349
+ end
350
+ end
351
+ end
352
+ end
353
+ end