neptune 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +4 -0
- data/doc/BabelHelper.html +393 -376
- data/doc/BadConfigurationException.html +121 -127
- data/doc/CommonFunctions.html +237 -265
- data/doc/ExodusHelper.html +820 -0
- data/doc/ExodusTaskInfo.html +263 -0
- data/doc/FileNotFoundException.html +121 -127
- data/doc/NeptuneHelper.html +527 -592
- data/doc/NeptuneManagerClient.html +696 -0
- data/doc/NeptuneManagerException.html +139 -0
- data/doc/Object.html +334 -236
- data/doc/TaskInfo.html +428 -0
- data/doc/created.rid +8 -5
- data/doc/images/add.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/index.html +74 -142
- data/doc/js/darkfish.js +99 -62
- data/doc/js/jquery.js +15 -29
- data/doc/js/navigation.js +142 -0
- data/doc/js/search.js +94 -0
- data/doc/js/search_index.js +1 -0
- data/doc/js/searcher.js +228 -0
- data/doc/table_of_contents.html +226 -0
- data/lib/babel.rb +116 -50
- data/lib/custom_exceptions.rb +2 -2
- data/lib/exodus.rb +311 -0
- data/lib/exodus_task_info.rb +36 -0
- data/lib/neptune.rb +52 -18
- data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
- data/lib/task_info.rb +155 -0
- data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
- data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
- data/test/test_exodus.rb +687 -0
- data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
- data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
- data/test/test_task_info.rb +32 -0
- data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
- metadata +30 -34
- data/doc/AppControllerClient.html +0 -702
- data/doc/AppControllerException.html +0 -145
- data/doc/bin/neptune.html +0 -56
- data/doc/js/quicksearch.js +0 -114
- data/doc/js/thickbox-compressed.js +0 -10
- data/doc/lib/app_controller_client_rb.html +0 -60
- data/doc/lib/babel_rb.html +0 -68
- data/doc/lib/common_functions_rb.html +0 -70
- data/doc/lib/custom_exceptions_rb.html +0 -54
- data/doc/lib/neptune_rb.html +0 -60
- data/test/integration/tc_c.rb +0 -57
- data/test/integration/tc_dfsp.rb +0 -37
- data/test/integration/tc_dwssa.rb +0 -38
- data/test/integration/tc_erlang.rb +0 -183
- data/test/integration/tc_mapreduce.rb +0 -282
- data/test/integration/tc_mpi.rb +0 -160
- data/test/integration/tc_storage.rb +0 -209
- data/test/integration/tc_upc.rb +0 -75
- data/test/integration/tc_x10.rb +0 -94
- data/test/integration/test_helper.rb +0 -135
- data/test/integration/ts_neptune.rb +0 -40
data/test/test_exodus.rb
ADDED
@@ -0,0 +1,687 @@
|
|
1
|
+
# Programmer: Chris Bunch (cgb@cs.ucsb.edu)
|
2
|
+
|
3
|
+
|
4
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
5
|
+
require 'exodus'
|
6
|
+
|
7
|
+
|
8
|
+
require 'rubygems'
|
9
|
+
require 'flexmock/test_unit'
|
10
|
+
|
11
|
+
|
12
|
+
class TestExodus < Test::Unit::TestCase
|
13
|
+
|
14
|
+
|
15
|
+
def setup
|
16
|
+
# all sleeps should immediately return
|
17
|
+
flexmock(Kernel).should_receive(:sleep).and_return()
|
18
|
+
|
19
|
+
# assume that appscale is always running for keyname=appscale
|
20
|
+
location_file = File.expand_path("~/.appscale/locations-appscale.yaml")
|
21
|
+
flexmock(File).should_receive(:exists?).with(location_file).
|
22
|
+
and_return(true)
|
23
|
+
|
24
|
+
# set up some dummy data that will get read when we try to read the
|
25
|
+
# locations file
|
26
|
+
yaml_info = {
|
27
|
+
:shadow => "127.0.0.1",
|
28
|
+
:secret => "secret"
|
29
|
+
}
|
30
|
+
flexmock(YAML).should_receive(:load_file).and_return(yaml_info)
|
31
|
+
|
32
|
+
@ec2_credentials = {
|
33
|
+
:EC2_ACCESS_KEY => "boo",
|
34
|
+
:EC2_SECRET_KEY => "baz",
|
35
|
+
:EC2_URL => "http://ec2.url",
|
36
|
+
:S3_URL => "http://s3.url",
|
37
|
+
:S3_bucket_name => "bazbucket"
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
def test_exodus_job_format_validation
|
43
|
+
# calling exodus with something that's not an Array or Hash should fail
|
44
|
+
assert_raises(BadConfigurationException) {
|
45
|
+
exodus(2)
|
46
|
+
}
|
47
|
+
|
48
|
+
# also, if we give exodus an Array, it had better be an array of Hashes
|
49
|
+
assert_raises(BadConfigurationException) {
|
50
|
+
exodus([2])
|
51
|
+
}
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
def test_ensure_all_params_are_present
|
56
|
+
# calling exodus without specifying :clouds should fail
|
57
|
+
assert_raises(BadConfigurationException) {
|
58
|
+
ExodusHelper.ensure_all_params_are_present({})
|
59
|
+
}
|
60
|
+
|
61
|
+
# calling exodus with invalid clouds specified should fail
|
62
|
+
assert_raises(BadConfigurationException) {
|
63
|
+
ExodusHelper.ensure_all_params_are_present({
|
64
|
+
:clouds_to_use => :BazCloud,
|
65
|
+
:credentials => {}
|
66
|
+
})
|
67
|
+
}
|
68
|
+
|
69
|
+
# doing the same but with an array should also fail
|
70
|
+
assert_raises(BadConfigurationException) {
|
71
|
+
ExodusHelper.ensure_all_params_are_present({
|
72
|
+
:clouds_to_use => [:BazCloud],
|
73
|
+
:credentials => {}
|
74
|
+
})
|
75
|
+
}
|
76
|
+
|
77
|
+
# giving an array of not strings should fail
|
78
|
+
assert_raises(BadConfigurationException) {
|
79
|
+
ExodusHelper.ensure_all_params_are_present({:clouds_to_use => [1, 2, 3]})
|
80
|
+
}
|
81
|
+
|
82
|
+
# giving not a string should fail
|
83
|
+
assert_raises(BadConfigurationException) {
|
84
|
+
ExodusHelper.ensure_all_params_are_present({:clouds_to_use => 1})
|
85
|
+
}
|
86
|
+
|
87
|
+
# giving an acceptable cloud but with no credentials should fail
|
88
|
+
assert_raises(BadConfigurationException) {
|
89
|
+
ExodusHelper.ensure_all_params_are_present({:clouds_to_use => :GoogleAppEngine})
|
90
|
+
}
|
91
|
+
|
92
|
+
# similarly, specifying credentials in a non-Hash format should fail
|
93
|
+
assert_raises(BadConfigurationException) {
|
94
|
+
ExodusHelper.ensure_all_params_are_present({
|
95
|
+
:clouds_to_use => :GoogleAppEngine,
|
96
|
+
:credentials => 1
|
97
|
+
})
|
98
|
+
}
|
99
|
+
|
100
|
+
# if a credential is nil or empty, it should fail
|
101
|
+
assert_raises(BadConfigurationException) {
|
102
|
+
ExodusHelper.ensure_all_params_are_present({
|
103
|
+
:clouds_to_use => :AmazonEC2,
|
104
|
+
:credentials => {
|
105
|
+
:EC2_ACCESS_KEY => nil,
|
106
|
+
:EC2_SECRET_KEY => "baz"
|
107
|
+
}
|
108
|
+
})
|
109
|
+
}
|
110
|
+
|
111
|
+
assert_raises(BadConfigurationException) {
|
112
|
+
ExodusHelper.ensure_all_params_are_present({
|
113
|
+
:clouds_to_use => :AmazonEC2,
|
114
|
+
:credentials => {
|
115
|
+
:EC2_ACCESS_KEY => "",
|
116
|
+
:EC2_SECRET_KEY => "baz"
|
117
|
+
}
|
118
|
+
})
|
119
|
+
}
|
120
|
+
|
121
|
+
# make sure that the user tells us to optimize their task for either
|
122
|
+
# performance or cost
|
123
|
+
assert_raises(BadConfigurationException) {
|
124
|
+
ExodusHelper.ensure_all_params_are_present({
|
125
|
+
:clouds_to_use => :AmazonEC2,
|
126
|
+
:credentials => @ec2_credentials
|
127
|
+
})
|
128
|
+
}
|
129
|
+
|
130
|
+
assert_raises(BadConfigurationException) {
|
131
|
+
ExodusHelper.ensure_all_params_are_present({
|
132
|
+
:clouds_to_use => :AmazonEC2,
|
133
|
+
:credentials => @ec2_credentials,
|
134
|
+
:optimize_for => 2
|
135
|
+
})
|
136
|
+
}
|
137
|
+
|
138
|
+
# failing to specify files, argv, or executable
|
139
|
+
# should fail
|
140
|
+
|
141
|
+
# first, files
|
142
|
+
assert_raises(BadConfigurationException) {
|
143
|
+
ExodusHelper.ensure_all_params_are_present({
|
144
|
+
:clouds_to_use => :AmazonEC2,
|
145
|
+
:credentials => @ec2_credentials,
|
146
|
+
:optimize_for => :cost
|
147
|
+
})
|
148
|
+
}
|
149
|
+
|
150
|
+
# next, argv
|
151
|
+
assert_raises(BadConfigurationException) {
|
152
|
+
ExodusHelper.ensure_all_params_are_present({
|
153
|
+
:clouds_to_use => :AmazonEC2,
|
154
|
+
:credentials => @ec2_credentials,
|
155
|
+
:optimize_for => :cost,
|
156
|
+
:code => "/foo/bar.rb"
|
157
|
+
})
|
158
|
+
}
|
159
|
+
|
160
|
+
# finally, executable
|
161
|
+
assert_raises(BadConfigurationException) {
|
162
|
+
ExodusHelper.ensure_all_params_are_present({
|
163
|
+
:clouds_to_use => :AmazonEC2,
|
164
|
+
:credentials => @ec2_credentials,
|
165
|
+
:optimize_for => :cost,
|
166
|
+
:code => "/foo/bar.rb",
|
167
|
+
:argv => []
|
168
|
+
})
|
169
|
+
}
|
170
|
+
|
171
|
+
# and of course, calling this function the right way should not fail
|
172
|
+
assert_nothing_raised(BadConfigurationException) {
|
173
|
+
ExodusHelper.ensure_all_params_are_present({
|
174
|
+
:clouds_to_use => :AmazonEC2,
|
175
|
+
:credentials => @ec2_credentials,
|
176
|
+
:optimize_for => :cost,
|
177
|
+
:code => "/foo/bar.rb",
|
178
|
+
:argv => [],
|
179
|
+
:executable => "ruby"
|
180
|
+
})
|
181
|
+
}
|
182
|
+
|
183
|
+
# similarly, if the credentials were in the user's environment instead
|
184
|
+
# of in the job specification, exodus should pull them in for us
|
185
|
+
ENV['EC2_ACCESS_KEY'] = "boo"
|
186
|
+
ENV['EC2_SECRET_KEY'] = "baz"
|
187
|
+
ENV['EC2_URL'] = "http://ec2.url"
|
188
|
+
ENV['S3_URL'] = "http://s3.url"
|
189
|
+
ENV['S3_bucket_name'] = "bazbucket"
|
190
|
+
assert_nothing_raised(BadConfigurationException) {
|
191
|
+
ExodusHelper.ensure_all_params_are_present({
|
192
|
+
:clouds_to_use => :AmazonEC2,
|
193
|
+
:credentials => {
|
194
|
+
},
|
195
|
+
:optimize_for => :cost,
|
196
|
+
:code => "/foo/bar.rb",
|
197
|
+
:argv => [],
|
198
|
+
:executable => "ruby"
|
199
|
+
})
|
200
|
+
}
|
201
|
+
ENV['EC2_ACCESS_KEY'] = nil
|
202
|
+
ENV['EC2_SECRET_KEY'] = nil
|
203
|
+
ENV['EC2_URL'] = nil
|
204
|
+
ENV['S3_URL'] = nil
|
205
|
+
ENV['S3_bucket_name'] = nil
|
206
|
+
end
|
207
|
+
|
208
|
+
|
209
|
+
def test_get_task_info_from_neptune_manager
|
210
|
+
# this test sets the expectations for what should happen if
|
211
|
+
# we ask the neptune manager for info about a job and it has
|
212
|
+
# never seen this job before
|
213
|
+
|
214
|
+
job = {
|
215
|
+
:clouds_to_use => :AmazonEC2,
|
216
|
+
:credentials => {
|
217
|
+
:EC2_ACCESS_KEY => "boo",
|
218
|
+
:EC2_SECRET_KEY => "baz"
|
219
|
+
},
|
220
|
+
:optimize_for => :cost,
|
221
|
+
:code => "/foo/bar.rb",
|
222
|
+
:argv => [],
|
223
|
+
:executable => "ruby"
|
224
|
+
}
|
225
|
+
|
226
|
+
# mock out the SOAP call for get_profiling_info
|
227
|
+
no_job_data = {
|
228
|
+
}
|
229
|
+
flexmock(NeptuneManagerClient).new_instances { |instance|
|
230
|
+
instance.should_receive(:get_profiling_info).with(job[:code]).
|
231
|
+
and_return(no_job_data)
|
232
|
+
}
|
233
|
+
|
234
|
+
profiling_info = ExodusHelper.get_profiling_info(job)
|
235
|
+
assert_equal(true, profiling_info.empty?)
|
236
|
+
end
|
237
|
+
|
238
|
+
|
239
|
+
def test_get_clouds_to_run_task_on_with_no_data
|
240
|
+
job = {
|
241
|
+
:clouds_to_use => [:AmazonEC2, :Eucalyptus],
|
242
|
+
:credentials => {
|
243
|
+
:EC2_ACCESS_KEY => "boo",
|
244
|
+
:EC2_SECRET_KEY => "baz"
|
245
|
+
},
|
246
|
+
:code => "/foo/bar.rb",
|
247
|
+
:argv => [],
|
248
|
+
:executable => "ruby"
|
249
|
+
}
|
250
|
+
|
251
|
+
# in this entirely hypothetical example, let us assume we have never run
|
252
|
+
# our task previously, so we have no data on what its performance and
|
253
|
+
# cost characteristics look like
|
254
|
+
profiling_info = {
|
255
|
+
}
|
256
|
+
|
257
|
+
job[:optimize_for] = :performance
|
258
|
+
clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
|
259
|
+
profiling_info)
|
260
|
+
assert_equal(job[:clouds_to_use], clouds_to_run_task_on)
|
261
|
+
|
262
|
+
job[:optimize_for] = :cost
|
263
|
+
clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
|
264
|
+
profiling_info)
|
265
|
+
assert_equal(job[:clouds_to_use], clouds_to_run_task_on)
|
266
|
+
end
|
267
|
+
|
268
|
+
def test_get_clouds_to_run_task_on_with_data
|
269
|
+
job = {
|
270
|
+
:clouds_to_use => [:AmazonEC2, :Eucalyptus],
|
271
|
+
:credentials => {
|
272
|
+
:EC2_ACCESS_KEY => "boo",
|
273
|
+
:EC2_SECRET_KEY => "baz"
|
274
|
+
},
|
275
|
+
:code => "/foo/bar.rb",
|
276
|
+
:argv => [],
|
277
|
+
:executable => "ruby"
|
278
|
+
}
|
279
|
+
|
280
|
+
# in this entirely hypothetical example, let us assume we have run our
|
281
|
+
# task previously on Amazon EC2 and Eucalyptus, and that it runs
|
282
|
+
# faster on EC2, but cheaper on Eucalyptus (that is, for free).
|
283
|
+
profiling_info = {
|
284
|
+
'AmazonEC2' => {
|
285
|
+
'performance' => [30.2, 40.6, 35.8, 41.2, 38.9],
|
286
|
+
'cost' => [0.40, 0.40, 0.40, 0.40, 0.40]
|
287
|
+
},
|
288
|
+
'Eucalyptus' => {
|
289
|
+
'performance' => [60.0, 65.0, 55.0, 70.3, 63.2],
|
290
|
+
'cost' => [0.0, 0.0, 0.0, 0.0, 0.0]
|
291
|
+
}
|
292
|
+
}
|
293
|
+
|
294
|
+
job[:optimize_for] = :performance
|
295
|
+
clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
|
296
|
+
profiling_info)
|
297
|
+
assert_equal([:AmazonEC2], clouds_to_run_task_on)
|
298
|
+
|
299
|
+
job[:optimize_for] = :cost
|
300
|
+
clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
|
301
|
+
profiling_info)
|
302
|
+
assert_equal([:Eucalyptus], clouds_to_run_task_on)
|
303
|
+
end
|
304
|
+
|
305
|
+
|
306
|
+
def test_get_clouds_to_run_task_on_when_profiling_lacks_data
|
307
|
+
job = {
|
308
|
+
:clouds_to_use => [:AmazonEC2, :Eucalyptus, :GoogleAppEngine],
|
309
|
+
:credentials => {
|
310
|
+
:EC2_ACCESS_KEY => "boo",
|
311
|
+
:EC2_SECRET_KEY => "baz"
|
312
|
+
},
|
313
|
+
:code => "/foo/bar.rb",
|
314
|
+
:argv => [],
|
315
|
+
:executable => "ruby"
|
316
|
+
}
|
317
|
+
|
318
|
+
# this example is similar to the prior one, but this time, we have data
|
319
|
+
# on EC2 and Eucalyptus but not Google App Engine. Here, we expect to
|
320
|
+
# run the job on the best one that we're trying to optimize for as well
|
321
|
+
# as anywhere we've never run it before (that is, Google App Engine)
|
322
|
+
profiling_info = {
|
323
|
+
'AmazonEC2' => {
|
324
|
+
'performance' => [30.2, 40.6, 35.8, 41.2, 38.9],
|
325
|
+
'cost' => [0.40, 0.40, 0.40, 0.40, 0.40]
|
326
|
+
},
|
327
|
+
'Eucalyptus' => {
|
328
|
+
'performance' => [60.0, 65.0, 55.0, 70.3, 63.2],
|
329
|
+
'cost' => [0.0, 0.0, 0.0, 0.0, 0.0]
|
330
|
+
}
|
331
|
+
}
|
332
|
+
|
333
|
+
job[:optimize_for] = :performance
|
334
|
+
clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
|
335
|
+
profiling_info)
|
336
|
+
assert_equal(true, clouds_to_run_task_on.include?(:AmazonEC2))
|
337
|
+
assert_equal(true, clouds_to_run_task_on.include?(:GoogleAppEngine))
|
338
|
+
|
339
|
+
job[:optimize_for] = :cost
|
340
|
+
clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
|
341
|
+
profiling_info)
|
342
|
+
assert_equal(true, clouds_to_run_task_on.include?(:Eucalyptus))
|
343
|
+
assert_equal(true, clouds_to_run_task_on.include?(:GoogleAppEngine))
|
344
|
+
end
|
345
|
+
|
346
|
+
|
347
|
+
def test_generate_babel_tasks_one_task
|
348
|
+
job = {
|
349
|
+
:clouds_to_use => [:AmazonEC2, :Eucalyptus, :GoogleAppEngine],
|
350
|
+
:credentials => {
|
351
|
+
:EUCA_ACCESS_KEY => "boo",
|
352
|
+
:EUCA_SECRET_KEY => "baz",
|
353
|
+
:EUCA_URL => "http://euca.url",
|
354
|
+
:WALRUS_URL => "http://walrus.url",
|
355
|
+
:Walrus_bucket_name => "bazbucket"
|
356
|
+
},
|
357
|
+
:code => "/foo/bar.rb",
|
358
|
+
:argv => [2],
|
359
|
+
:executable => "ruby",
|
360
|
+
:optimize_for => :cost
|
361
|
+
}
|
362
|
+
|
363
|
+
clouds_to_run_task_on = [:Eucalyptus]
|
364
|
+
|
365
|
+
expected = [{
|
366
|
+
:type => "babel",
|
367
|
+
:EUCA_ACCESS_KEY => "boo",
|
368
|
+
:EUCA_SECRET_KEY => "baz",
|
369
|
+
:EUCA_URL => "http://euca.url",
|
370
|
+
:WALRUS_URL => "http://walrus.url",
|
371
|
+
:Walrus_bucket_name => "bazbucket",
|
372
|
+
:code => "/foo/bar.rb",
|
373
|
+
:argv => [2],
|
374
|
+
:executable => "ruby",
|
375
|
+
:is_remote => false,
|
376
|
+
:run_local => false,
|
377
|
+
:storage => "walrus",
|
378
|
+
:engine => "executor-rabbitmq"
|
379
|
+
}]
|
380
|
+
actual = ExodusHelper.generate_babel_tasks(job, clouds_to_run_task_on)
|
381
|
+
assert_equal(expected, actual)
|
382
|
+
end
|
383
|
+
|
384
|
+
|
385
|
+
def test_generate_babel_tasks_many_tasks
|
386
|
+
job = {
|
387
|
+
:clouds_to_use => [:AmazonEC2, :Eucalyptus, :GoogleAppEngine],
|
388
|
+
:credentials => {
|
389
|
+
# ec2 credentials
|
390
|
+
:EC2_ACCESS_KEY => "boo",
|
391
|
+
:EC2_SECRET_KEY => "baz",
|
392
|
+
:EC2_URL => "http://ec2.url",
|
393
|
+
:S3_URL => "http://s3.url",
|
394
|
+
:S3_bucket_name => "bazbucket1",
|
395
|
+
|
396
|
+
# google app engine credentials
|
397
|
+
:appid => "bazappid",
|
398
|
+
:appcfg_cookies => "~/.appcfg_cookies",
|
399
|
+
:function => "bazboo()",
|
400
|
+
:GStorage_bucket_name => "bazbucket2"
|
401
|
+
},
|
402
|
+
:code => "/foo/bar.rb",
|
403
|
+
:argv => [2],
|
404
|
+
:executable => "ruby",
|
405
|
+
:optimize_for => :cost
|
406
|
+
}
|
407
|
+
|
408
|
+
clouds_to_run_task_on = [:AmazonEC2, :GoogleAppEngine]
|
409
|
+
|
410
|
+
ec2_task = {
|
411
|
+
:type => "babel",
|
412
|
+
:EC2_ACCESS_KEY => "boo",
|
413
|
+
:EC2_SECRET_KEY => "baz",
|
414
|
+
:EC2_URL => "http://ec2.url",
|
415
|
+
:S3_URL => "http://s3.url",
|
416
|
+
:S3_bucket_name => "bazbucket1",
|
417
|
+
:code => "/foo/bar.rb",
|
418
|
+
:argv => [2],
|
419
|
+
:executable => "ruby",
|
420
|
+
:is_remote => false,
|
421
|
+
:run_local => false,
|
422
|
+
:storage => "s3",
|
423
|
+
:engine => "executor-sqs"
|
424
|
+
}
|
425
|
+
|
426
|
+
appengine_task = {
|
427
|
+
:type => "babel",
|
428
|
+
:appid => "bazappid",
|
429
|
+
:appcfg_cookies => "~/.appcfg_cookies",
|
430
|
+
:function => "bazboo()",
|
431
|
+
:GStorage_bucket_name => "bazbucket2",
|
432
|
+
:code => "/foo/bar.rb",
|
433
|
+
:argv => [2],
|
434
|
+
:executable => "ruby",
|
435
|
+
:is_remote => false,
|
436
|
+
:run_local => false,
|
437
|
+
:storage => "gstorage",
|
438
|
+
:engine => "appengine-push-q"
|
439
|
+
}
|
440
|
+
|
441
|
+
expected = [ec2_task, appengine_task]
|
442
|
+
actual = ExodusHelper.generate_babel_tasks(job, clouds_to_run_task_on)
|
443
|
+
assert_equal(expected, actual)
|
444
|
+
end
|
445
|
+
|
446
|
+
|
447
|
+
def test_exodus_run_job_with_one_babel_task
|
448
|
+
ec2_task = {
|
449
|
+
:type => "babel",
|
450
|
+
:EC2_ACCESS_KEY => "boo",
|
451
|
+
:EC2_SECRET_KEY => "baz",
|
452
|
+
:EC2_URL => "http://ec2.url",
|
453
|
+
:S3_URL => "http://s3.url",
|
454
|
+
:code => "/foo/bar.rb",
|
455
|
+
:argv => [2],
|
456
|
+
:executable => "ruby",
|
457
|
+
:is_remote => false,
|
458
|
+
:run_local => false,
|
459
|
+
:storage => "s3",
|
460
|
+
:engine => "executor-sqs",
|
461
|
+
:bucket_name => "bazbucket"
|
462
|
+
}
|
463
|
+
babel_tasks = [ec2_task]
|
464
|
+
|
465
|
+
# mock out calls to the NeptuneManager
|
466
|
+
flexmock(NeptuneManagerClient).new_instances { |instance|
|
467
|
+
# let's say that all checks to see if temp files exist tell us that
|
468
|
+
# the files don't exist
|
469
|
+
instance.should_receive(:does_file_exist?).
|
470
|
+
with(/\A\/bazbucket\/babel\/temp-[\w]+\Z/, Hash).
|
471
|
+
and_return(false)
|
472
|
+
|
473
|
+
# assume that our code got put in the remote datastore fine
|
474
|
+
instance.should_receive(:does_file_exist?).
|
475
|
+
with(/\A\/bazbucket\/babel\/foo\/bar.rb\Z/, Hash).
|
476
|
+
and_return(true)
|
477
|
+
|
478
|
+
# also, calls to put_input should succeed
|
479
|
+
instance.should_receive(:put_input).with(Hash).and_return(true)
|
480
|
+
|
481
|
+
# mock out the call to get_supported_babel_engines and put in
|
482
|
+
# SQS and rabbitmq (which is always supported)
|
483
|
+
instance.should_receive(:get_supported_babel_engines).with(Hash).
|
484
|
+
and_return(["executor-rabbitmq", "executor-sqs"])
|
485
|
+
|
486
|
+
# neptune jobs should start fine
|
487
|
+
instance.should_receive(:start_neptune_job).with(Hash).
|
488
|
+
and_return("babel job is now running")
|
489
|
+
|
490
|
+
# getting the output of the job should return it the first time
|
491
|
+
instance.should_receive(:get_output).with(Hash).
|
492
|
+
and_return("task output yay!")
|
493
|
+
}
|
494
|
+
|
495
|
+
# mock out filesystem checks
|
496
|
+
# we'll say that our code does exist
|
497
|
+
flexmock(File).should_receive(:exists?).with("/foo").and_return(true)
|
498
|
+
|
499
|
+
# mock out scp calls - assume they go through with no problems
|
500
|
+
flexmock(CommonFunctions).should_receive(:shell).with(/\Ascp/).
|
501
|
+
and_return()
|
502
|
+
|
503
|
+
dispatched_tasks = ExodusHelper.run_job(babel_tasks)
|
504
|
+
exodus_task = ExodusTaskInfo.new(dispatched_tasks)
|
505
|
+
|
506
|
+
expected = "task output yay!"
|
507
|
+
assert_equal(expected, exodus_task.to_s)
|
508
|
+
end
|
509
|
+
|
510
|
+
|
511
|
+
def test_exodus_job_that_generates_one_babel_task
|
512
|
+
job = {
|
513
|
+
:clouds_to_use => :AmazonEC2,
|
514
|
+
:credentials => {
|
515
|
+
:EC2_ACCESS_KEY => "boo",
|
516
|
+
:EC2_SECRET_KEY => "baz",
|
517
|
+
:EC2_URL => "http://ec2.url",
|
518
|
+
:S3_URL => "http://s3.url",
|
519
|
+
:S3_bucket_name => "bazbucket"
|
520
|
+
},
|
521
|
+
:optimize_for => :cost,
|
522
|
+
:code => "/foo/bar.rb",
|
523
|
+
:argv => [],
|
524
|
+
:executable => "ruby"
|
525
|
+
}
|
526
|
+
|
527
|
+
# mock out calls to the NeptuneManager
|
528
|
+
flexmock(NeptuneManagerClient).new_instances { |instance|
|
529
|
+
# for this test, let's say there's no data on this task right now
|
530
|
+
instance.should_receive(:get_profiling_info).with(String).
|
531
|
+
and_return({})
|
532
|
+
|
533
|
+
# let's say that all checks to see if temp files exist tell us that
|
534
|
+
# the files don't exist
|
535
|
+
instance.should_receive(:does_file_exist?).
|
536
|
+
with(/\A\/bazbucket\/babel\/temp-[\w]+\Z/, Hash).
|
537
|
+
and_return(false)
|
538
|
+
|
539
|
+
# assume that our code got put in the remote datastore fine
|
540
|
+
instance.should_receive(:does_file_exist?).
|
541
|
+
with(/\A\/bazbucket\/babel\/foo\/bar.rb\Z/, Hash).
|
542
|
+
and_return(true)
|
543
|
+
|
544
|
+
# also, calls to put_input should succeed
|
545
|
+
instance.should_receive(:put_input).with(Hash).and_return(true)
|
546
|
+
|
547
|
+
# mock out the call to get_supported_babel_engines and put in
|
548
|
+
# SQS and rabbitmq (which is always supported)
|
549
|
+
instance.should_receive(:get_supported_babel_engines).with(Hash).
|
550
|
+
and_return(["executor-rabbitmq", "executor-sqs"])
|
551
|
+
|
552
|
+
# neptune jobs should start fine
|
553
|
+
instance.should_receive(:start_neptune_job).with(Hash).
|
554
|
+
and_return("babel job is now running")
|
555
|
+
|
556
|
+
# getting the output of the job should return it the first time
|
557
|
+
instance.should_receive(:get_output).with(Hash).
|
558
|
+
and_return("task output yay!")
|
559
|
+
}
|
560
|
+
|
561
|
+
# mock out filesystem checks
|
562
|
+
# we'll say that our code does exist
|
563
|
+
flexmock(File).should_receive(:exists?).with("/foo").and_return(true)
|
564
|
+
|
565
|
+
# mock out scp calls - assume they go through with no problems
|
566
|
+
flexmock(CommonFunctions).should_receive(:shell).with(/\Ascp/).
|
567
|
+
and_return()
|
568
|
+
|
569
|
+
# first, let's make sure that exodus calls work fine if we give it
|
570
|
+
# a Hash, containing info on one job
|
571
|
+
expected = "task output yay!"
|
572
|
+
actual = exodus(job)
|
573
|
+
assert_equal(expected, actual.to_s)
|
574
|
+
assert_equal(expected, actual.stdout)
|
575
|
+
|
576
|
+
# now, let's make sure that exodus calls still work the same way if we
|
577
|
+
# give it an Array, containing info on the same job
|
578
|
+
actual2 = exodus([job])
|
579
|
+
assert_equal(expected, actual2[0].to_s)
|
580
|
+
assert_equal(expected, actual2[0].stdout)
|
581
|
+
end
|
582
|
+
|
583
|
+
|
584
|
+
def test_exodus_job_that_generates_two_babel_tasks
|
585
|
+
job = {
|
586
|
+
:clouds_to_use => [:AmazonEC2, :MicrosoftAzure],
|
587
|
+
:credentials => {
|
588
|
+
# for EC2
|
589
|
+
:EC2_ACCESS_KEY => "boo",
|
590
|
+
:EC2_SECRET_KEY => "baz",
|
591
|
+
:EC2_URL => "http://ec2.url",
|
592
|
+
:S3_URL => "http://s3.url",
|
593
|
+
:S3_bucket_name => "bazbucket",
|
594
|
+
|
595
|
+
# for azure
|
596
|
+
:WAZ_Account_Name => "wazboo",
|
597
|
+
:WAZ_Access_Key => "wazbaz",
|
598
|
+
:WAZ_Container_Name => "wazbucket"
|
599
|
+
},
|
600
|
+
:optimize_for => :cost,
|
601
|
+
:code => "/foo/bar.rb",
|
602
|
+
:argv => [],
|
603
|
+
:executable => "ruby"
|
604
|
+
}
|
605
|
+
|
606
|
+
job2 = job.dup
|
607
|
+
|
608
|
+
# mock out calls to the NeptuneManager
|
609
|
+
flexmock(NeptuneManagerClient).new_instances { |instance|
|
610
|
+
# for this test, let's say there's no data on this task right now
|
611
|
+
instance.should_receive(:get_profiling_info).with(String).
|
612
|
+
and_return({})
|
613
|
+
|
614
|
+
# let's say that all checks to see if temp files exist tell us that
|
615
|
+
# the files don't exist
|
616
|
+
instance.should_receive(:does_file_exist?).
|
617
|
+
with(/\A\/bazbucket\/babel\/temp-[\w]+\Z/, Hash).
|
618
|
+
and_return(false)
|
619
|
+
|
620
|
+
instance.should_receive(:does_file_exist?).
|
621
|
+
with(/\A\/wazbucket\/babel\/temp-[\w]+\Z/, Hash).
|
622
|
+
and_return(false)
|
623
|
+
|
624
|
+
# assume that our code got put in the remote datastore fine
|
625
|
+
instance.should_receive(:does_file_exist?).
|
626
|
+
with(/\A\/bazbucket\/babel\/foo\/bar.rb\Z/, Hash).
|
627
|
+
and_return(true)
|
628
|
+
|
629
|
+
instance.should_receive(:does_file_exist?).
|
630
|
+
with(/\A\/wazbucket\/babel\/foo\/bar.rb\Z/, Hash).
|
631
|
+
and_return(true)
|
632
|
+
|
633
|
+
# also, calls to put_input should succeed
|
634
|
+
instance.should_receive(:put_input).with(Hash).and_return(true)
|
635
|
+
|
636
|
+
# mock out the call to get_supported_babel_engines and put in
|
637
|
+
# SQS, WAZ-push-q, and rabbitmq (which is always supported)
|
638
|
+
instance.should_receive(:get_supported_babel_engines).with(Hash).
|
639
|
+
and_return(["executor-rabbitmq", "executor-sqs", "waz-push-q"])
|
640
|
+
|
641
|
+
# neptune jobs should start fine - instead of expecting a Hash (like
|
642
|
+
# in the last test), we're now expecting an Array, where each item
|
643
|
+
# is a Hash corresponding to each of the jobs that will be run
|
644
|
+
instance.should_receive(:start_neptune_job).with(Array).
|
645
|
+
and_return("babel job is now running")
|
646
|
+
|
647
|
+
# getting the output of the job shouldn't return it the first time
|
648
|
+
# (unlike the last test). this time, we want to make sure that
|
649
|
+
# ExodusTaskInfo is properly asking all the babel tasks it's hiding
|
650
|
+
# from us for the job's output, so we'll have the first babel task
|
651
|
+
# always fail to have the output ready, and the second one will have
|
652
|
+
# it the second time around.
|
653
|
+
instance.should_receive(:get_output).with(on { |job|
|
654
|
+
job["@output"] =~ /\A\/bazbucket\/babel\/temp-/
|
655
|
+
}).
|
656
|
+
and_return(DOES_NOT_EXIST)
|
657
|
+
|
658
|
+
instance.should_receive(:get_output).with(on { |job|
|
659
|
+
job["@output"] =~ /\A\/wazbucket\/babel\/temp-/
|
660
|
+
}).
|
661
|
+
and_return("task output yay!")
|
662
|
+
}
|
663
|
+
|
664
|
+
# mock out filesystem checks
|
665
|
+
# we'll say that our code does exist
|
666
|
+
flexmock(File).should_receive(:exists?).with("/foo").and_return(true)
|
667
|
+
|
668
|
+
# mock out scp calls - assume they go through with no problems
|
669
|
+
flexmock(CommonFunctions).should_receive(:shell).with(/\Ascp/).
|
670
|
+
and_return()
|
671
|
+
|
672
|
+
# first, let's make sure that exodus calls work fine if we give it
|
673
|
+
# a Hash, containing info on one job
|
674
|
+
expected = "task output yay!"
|
675
|
+
actual = exodus(job)
|
676
|
+
assert_equal(expected, actual.to_s)
|
677
|
+
assert_equal(expected, actual.stdout)
|
678
|
+
|
679
|
+
# now, let's make sure that exodus calls still work the same way if we
|
680
|
+
# give it an Array, containing info on the same job
|
681
|
+
actual2 = exodus([job2])
|
682
|
+
assert_equal(expected, actual2[0].to_s)
|
683
|
+
assert_equal(expected, actual2[0].stdout)
|
684
|
+
end
|
685
|
+
|
686
|
+
|
687
|
+
end
|