neptune 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/README +4 -0
  2. data/doc/BabelHelper.html +393 -376
  3. data/doc/BadConfigurationException.html +121 -127
  4. data/doc/CommonFunctions.html +237 -265
  5. data/doc/ExodusHelper.html +820 -0
  6. data/doc/ExodusTaskInfo.html +263 -0
  7. data/doc/FileNotFoundException.html +121 -127
  8. data/doc/NeptuneHelper.html +527 -592
  9. data/doc/NeptuneManagerClient.html +696 -0
  10. data/doc/NeptuneManagerException.html +139 -0
  11. data/doc/Object.html +334 -236
  12. data/doc/TaskInfo.html +428 -0
  13. data/doc/created.rid +8 -5
  14. data/doc/images/add.png +0 -0
  15. data/doc/images/delete.png +0 -0
  16. data/doc/images/tag_blue.png +0 -0
  17. data/doc/images/transparent.png +0 -0
  18. data/doc/index.html +74 -142
  19. data/doc/js/darkfish.js +99 -62
  20. data/doc/js/jquery.js +15 -29
  21. data/doc/js/navigation.js +142 -0
  22. data/doc/js/search.js +94 -0
  23. data/doc/js/search_index.js +1 -0
  24. data/doc/js/searcher.js +228 -0
  25. data/doc/table_of_contents.html +226 -0
  26. data/lib/babel.rb +116 -50
  27. data/lib/custom_exceptions.rb +2 -2
  28. data/lib/exodus.rb +311 -0
  29. data/lib/exodus_task_info.rb +36 -0
  30. data/lib/neptune.rb +52 -18
  31. data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
  32. data/lib/task_info.rb +155 -0
  33. data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
  34. data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
  35. data/test/test_exodus.rb +687 -0
  36. data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
  37. data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
  38. data/test/test_task_info.rb +32 -0
  39. data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
  40. metadata +30 -34
  41. data/doc/AppControllerClient.html +0 -702
  42. data/doc/AppControllerException.html +0 -145
  43. data/doc/bin/neptune.html +0 -56
  44. data/doc/js/quicksearch.js +0 -114
  45. data/doc/js/thickbox-compressed.js +0 -10
  46. data/doc/lib/app_controller_client_rb.html +0 -60
  47. data/doc/lib/babel_rb.html +0 -68
  48. data/doc/lib/common_functions_rb.html +0 -70
  49. data/doc/lib/custom_exceptions_rb.html +0 -54
  50. data/doc/lib/neptune_rb.html +0 -60
  51. data/test/integration/tc_c.rb +0 -57
  52. data/test/integration/tc_dfsp.rb +0 -37
  53. data/test/integration/tc_dwssa.rb +0 -38
  54. data/test/integration/tc_erlang.rb +0 -183
  55. data/test/integration/tc_mapreduce.rb +0 -282
  56. data/test/integration/tc_mpi.rb +0 -160
  57. data/test/integration/tc_storage.rb +0 -209
  58. data/test/integration/tc_upc.rb +0 -75
  59. data/test/integration/tc_x10.rb +0 -94
  60. data/test/integration/test_helper.rb +0 -135
  61. data/test/integration/ts_neptune.rb +0 -40
@@ -1,6 +1,6 @@
1
1
  # Programmer: Chris Bunch (cgb@cs.ucsb.edu)
2
2
 
3
- $:.unshift File.join(File.dirname(__FILE__), "..", "..", "lib")
3
+ $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
4
4
  require 'common_functions'
5
5
 
6
6
  require 'test/unit'
@@ -0,0 +1,687 @@
1
+ # Programmer: Chris Bunch (cgb@cs.ucsb.edu)
2
+
3
+
4
+ $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
5
+ require 'exodus'
6
+
7
+
8
+ require 'rubygems'
9
+ require 'flexmock/test_unit'
10
+
11
+
12
+ class TestExodus < Test::Unit::TestCase
13
+
14
+
15
+ def setup
16
+ # all sleeps should immediately return
17
+ flexmock(Kernel).should_receive(:sleep).and_return()
18
+
19
+ # assume that appscale is always running for keyname=appscale
20
+ location_file = File.expand_path("~/.appscale/locations-appscale.yaml")
21
+ flexmock(File).should_receive(:exists?).with(location_file).
22
+ and_return(true)
23
+
24
+ # set up some dummy data that will get read when we try to read the
25
+ # locations file
26
+ yaml_info = {
27
+ :shadow => "127.0.0.1",
28
+ :secret => "secret"
29
+ }
30
+ flexmock(YAML).should_receive(:load_file).and_return(yaml_info)
31
+
32
+ @ec2_credentials = {
33
+ :EC2_ACCESS_KEY => "boo",
34
+ :EC2_SECRET_KEY => "baz",
35
+ :EC2_URL => "http://ec2.url",
36
+ :S3_URL => "http://s3.url",
37
+ :S3_bucket_name => "bazbucket"
38
+ }
39
+ end
40
+
41
+
42
+ def test_exodus_job_format_validation
43
+ # calling exodus with something that's not an Array or Hash should fail
44
+ assert_raises(BadConfigurationException) {
45
+ exodus(2)
46
+ }
47
+
48
+ # also, if we give exodus an Array, it had better be an array of Hashes
49
+ assert_raises(BadConfigurationException) {
50
+ exodus([2])
51
+ }
52
+ end
53
+
54
+
55
+ def test_ensure_all_params_are_present
56
+ # calling exodus without specifying :clouds should fail
57
+ assert_raises(BadConfigurationException) {
58
+ ExodusHelper.ensure_all_params_are_present({})
59
+ }
60
+
61
+ # calling exodus with invalid clouds specified should fail
62
+ assert_raises(BadConfigurationException) {
63
+ ExodusHelper.ensure_all_params_are_present({
64
+ :clouds_to_use => :BazCloud,
65
+ :credentials => {}
66
+ })
67
+ }
68
+
69
+ # doing the same but with an array should also fail
70
+ assert_raises(BadConfigurationException) {
71
+ ExodusHelper.ensure_all_params_are_present({
72
+ :clouds_to_use => [:BazCloud],
73
+ :credentials => {}
74
+ })
75
+ }
76
+
77
+ # giving an array of not strings should fail
78
+ assert_raises(BadConfigurationException) {
79
+ ExodusHelper.ensure_all_params_are_present({:clouds_to_use => [1, 2, 3]})
80
+ }
81
+
82
+ # giving not a string should fail
83
+ assert_raises(BadConfigurationException) {
84
+ ExodusHelper.ensure_all_params_are_present({:clouds_to_use => 1})
85
+ }
86
+
87
+ # giving an acceptable cloud but with no credentials should fail
88
+ assert_raises(BadConfigurationException) {
89
+ ExodusHelper.ensure_all_params_are_present({:clouds_to_use => :GoogleAppEngine})
90
+ }
91
+
92
+ # similarly, specifying credentials in a non-Hash format should fail
93
+ assert_raises(BadConfigurationException) {
94
+ ExodusHelper.ensure_all_params_are_present({
95
+ :clouds_to_use => :GoogleAppEngine,
96
+ :credentials => 1
97
+ })
98
+ }
99
+
100
+ # if a credential is nil or empty, it should fail
101
+ assert_raises(BadConfigurationException) {
102
+ ExodusHelper.ensure_all_params_are_present({
103
+ :clouds_to_use => :AmazonEC2,
104
+ :credentials => {
105
+ :EC2_ACCESS_KEY => nil,
106
+ :EC2_SECRET_KEY => "baz"
107
+ }
108
+ })
109
+ }
110
+
111
+ assert_raises(BadConfigurationException) {
112
+ ExodusHelper.ensure_all_params_are_present({
113
+ :clouds_to_use => :AmazonEC2,
114
+ :credentials => {
115
+ :EC2_ACCESS_KEY => "",
116
+ :EC2_SECRET_KEY => "baz"
117
+ }
118
+ })
119
+ }
120
+
121
+ # make sure that the user tells us to optimize their task for either
122
+ # performance or cost
123
+ assert_raises(BadConfigurationException) {
124
+ ExodusHelper.ensure_all_params_are_present({
125
+ :clouds_to_use => :AmazonEC2,
126
+ :credentials => @ec2_credentials
127
+ })
128
+ }
129
+
130
+ assert_raises(BadConfigurationException) {
131
+ ExodusHelper.ensure_all_params_are_present({
132
+ :clouds_to_use => :AmazonEC2,
133
+ :credentials => @ec2_credentials,
134
+ :optimize_for => 2
135
+ })
136
+ }
137
+
138
+ # failing to specify files, argv, or executable
139
+ # should fail
140
+
141
+ # first, files
142
+ assert_raises(BadConfigurationException) {
143
+ ExodusHelper.ensure_all_params_are_present({
144
+ :clouds_to_use => :AmazonEC2,
145
+ :credentials => @ec2_credentials,
146
+ :optimize_for => :cost
147
+ })
148
+ }
149
+
150
+ # next, argv
151
+ assert_raises(BadConfigurationException) {
152
+ ExodusHelper.ensure_all_params_are_present({
153
+ :clouds_to_use => :AmazonEC2,
154
+ :credentials => @ec2_credentials,
155
+ :optimize_for => :cost,
156
+ :code => "/foo/bar.rb"
157
+ })
158
+ }
159
+
160
+ # finally, executable
161
+ assert_raises(BadConfigurationException) {
162
+ ExodusHelper.ensure_all_params_are_present({
163
+ :clouds_to_use => :AmazonEC2,
164
+ :credentials => @ec2_credentials,
165
+ :optimize_for => :cost,
166
+ :code => "/foo/bar.rb",
167
+ :argv => []
168
+ })
169
+ }
170
+
171
+ # and of course, calling this function the right way should not fail
172
+ assert_nothing_raised(BadConfigurationException) {
173
+ ExodusHelper.ensure_all_params_are_present({
174
+ :clouds_to_use => :AmazonEC2,
175
+ :credentials => @ec2_credentials,
176
+ :optimize_for => :cost,
177
+ :code => "/foo/bar.rb",
178
+ :argv => [],
179
+ :executable => "ruby"
180
+ })
181
+ }
182
+
183
+ # similarly, if the credentials were in the user's environment instead
184
+ # of in the job specification, exodus should pull them in for us
185
+ ENV['EC2_ACCESS_KEY'] = "boo"
186
+ ENV['EC2_SECRET_KEY'] = "baz"
187
+ ENV['EC2_URL'] = "http://ec2.url"
188
+ ENV['S3_URL'] = "http://s3.url"
189
+ ENV['S3_bucket_name'] = "bazbucket"
190
+ assert_nothing_raised(BadConfigurationException) {
191
+ ExodusHelper.ensure_all_params_are_present({
192
+ :clouds_to_use => :AmazonEC2,
193
+ :credentials => {
194
+ },
195
+ :optimize_for => :cost,
196
+ :code => "/foo/bar.rb",
197
+ :argv => [],
198
+ :executable => "ruby"
199
+ })
200
+ }
201
+ ENV['EC2_ACCESS_KEY'] = nil
202
+ ENV['EC2_SECRET_KEY'] = nil
203
+ ENV['EC2_URL'] = nil
204
+ ENV['S3_URL'] = nil
205
+ ENV['S3_bucket_name'] = nil
206
+ end
207
+
208
+
209
+ def test_get_task_info_from_neptune_manager
210
+ # this test sets the expectations for what should happen if
211
+ # we ask the neptune manager for info about a job and it has
212
+ # never seen this job before
213
+
214
+ job = {
215
+ :clouds_to_use => :AmazonEC2,
216
+ :credentials => {
217
+ :EC2_ACCESS_KEY => "boo",
218
+ :EC2_SECRET_KEY => "baz"
219
+ },
220
+ :optimize_for => :cost,
221
+ :code => "/foo/bar.rb",
222
+ :argv => [],
223
+ :executable => "ruby"
224
+ }
225
+
226
+ # mock out the SOAP call for get_profiling_info
227
+ no_job_data = {
228
+ }
229
+ flexmock(NeptuneManagerClient).new_instances { |instance|
230
+ instance.should_receive(:get_profiling_info).with(job[:code]).
231
+ and_return(no_job_data)
232
+ }
233
+
234
+ profiling_info = ExodusHelper.get_profiling_info(job)
235
+ assert_equal(true, profiling_info.empty?)
236
+ end
237
+
238
+
239
+ def test_get_clouds_to_run_task_on_with_no_data
240
+ job = {
241
+ :clouds_to_use => [:AmazonEC2, :Eucalyptus],
242
+ :credentials => {
243
+ :EC2_ACCESS_KEY => "boo",
244
+ :EC2_SECRET_KEY => "baz"
245
+ },
246
+ :code => "/foo/bar.rb",
247
+ :argv => [],
248
+ :executable => "ruby"
249
+ }
250
+
251
+ # in this entirely hypothetical example, let us assume we have never run
252
+ # our task previously, so we have no data on what its performance and
253
+ # cost characteristics look like
254
+ profiling_info = {
255
+ }
256
+
257
+ job[:optimize_for] = :performance
258
+ clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
259
+ profiling_info)
260
+ assert_equal(job[:clouds_to_use], clouds_to_run_task_on)
261
+
262
+ job[:optimize_for] = :cost
263
+ clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
264
+ profiling_info)
265
+ assert_equal(job[:clouds_to_use], clouds_to_run_task_on)
266
+ end
267
+
268
+ def test_get_clouds_to_run_task_on_with_data
269
+ job = {
270
+ :clouds_to_use => [:AmazonEC2, :Eucalyptus],
271
+ :credentials => {
272
+ :EC2_ACCESS_KEY => "boo",
273
+ :EC2_SECRET_KEY => "baz"
274
+ },
275
+ :code => "/foo/bar.rb",
276
+ :argv => [],
277
+ :executable => "ruby"
278
+ }
279
+
280
+ # in this entirely hypothetical example, let us assume we have run our
281
+ # task previously on Amazon EC2 and Eucalyptus, and that it runs
282
+ # faster on EC2, but cheaper on Eucalyptus (that is, for free).
283
+ profiling_info = {
284
+ 'AmazonEC2' => {
285
+ 'performance' => [30.2, 40.6, 35.8, 41.2, 38.9],
286
+ 'cost' => [0.40, 0.40, 0.40, 0.40, 0.40]
287
+ },
288
+ 'Eucalyptus' => {
289
+ 'performance' => [60.0, 65.0, 55.0, 70.3, 63.2],
290
+ 'cost' => [0.0, 0.0, 0.0, 0.0, 0.0]
291
+ }
292
+ }
293
+
294
+ job[:optimize_for] = :performance
295
+ clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
296
+ profiling_info)
297
+ assert_equal([:AmazonEC2], clouds_to_run_task_on)
298
+
299
+ job[:optimize_for] = :cost
300
+ clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
301
+ profiling_info)
302
+ assert_equal([:Eucalyptus], clouds_to_run_task_on)
303
+ end
304
+
305
+
306
+ def test_get_clouds_to_run_task_on_when_profiling_lacks_data
307
+ job = {
308
+ :clouds_to_use => [:AmazonEC2, :Eucalyptus, :GoogleAppEngine],
309
+ :credentials => {
310
+ :EC2_ACCESS_KEY => "boo",
311
+ :EC2_SECRET_KEY => "baz"
312
+ },
313
+ :code => "/foo/bar.rb",
314
+ :argv => [],
315
+ :executable => "ruby"
316
+ }
317
+
318
+ # this example is similar to the prior one, but this time, we have data
319
+ # on EC2 and Eucalyptus but not Google App Engine. Here, we expect to
320
+ # run the job on the best one that we're trying to optimize for as well
321
+ # as anywhere we've never run it before (that is, Google App Engine)
322
+ profiling_info = {
323
+ 'AmazonEC2' => {
324
+ 'performance' => [30.2, 40.6, 35.8, 41.2, 38.9],
325
+ 'cost' => [0.40, 0.40, 0.40, 0.40, 0.40]
326
+ },
327
+ 'Eucalyptus' => {
328
+ 'performance' => [60.0, 65.0, 55.0, 70.3, 63.2],
329
+ 'cost' => [0.0, 0.0, 0.0, 0.0, 0.0]
330
+ }
331
+ }
332
+
333
+ job[:optimize_for] = :performance
334
+ clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
335
+ profiling_info)
336
+ assert_equal(true, clouds_to_run_task_on.include?(:AmazonEC2))
337
+ assert_equal(true, clouds_to_run_task_on.include?(:GoogleAppEngine))
338
+
339
+ job[:optimize_for] = :cost
340
+ clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job,
341
+ profiling_info)
342
+ assert_equal(true, clouds_to_run_task_on.include?(:Eucalyptus))
343
+ assert_equal(true, clouds_to_run_task_on.include?(:GoogleAppEngine))
344
+ end
345
+
346
+
347
+ def test_generate_babel_tasks_one_task
348
+ job = {
349
+ :clouds_to_use => [:AmazonEC2, :Eucalyptus, :GoogleAppEngine],
350
+ :credentials => {
351
+ :EUCA_ACCESS_KEY => "boo",
352
+ :EUCA_SECRET_KEY => "baz",
353
+ :EUCA_URL => "http://euca.url",
354
+ :WALRUS_URL => "http://walrus.url",
355
+ :Walrus_bucket_name => "bazbucket"
356
+ },
357
+ :code => "/foo/bar.rb",
358
+ :argv => [2],
359
+ :executable => "ruby",
360
+ :optimize_for => :cost
361
+ }
362
+
363
+ clouds_to_run_task_on = [:Eucalyptus]
364
+
365
+ expected = [{
366
+ :type => "babel",
367
+ :EUCA_ACCESS_KEY => "boo",
368
+ :EUCA_SECRET_KEY => "baz",
369
+ :EUCA_URL => "http://euca.url",
370
+ :WALRUS_URL => "http://walrus.url",
371
+ :Walrus_bucket_name => "bazbucket",
372
+ :code => "/foo/bar.rb",
373
+ :argv => [2],
374
+ :executable => "ruby",
375
+ :is_remote => false,
376
+ :run_local => false,
377
+ :storage => "walrus",
378
+ :engine => "executor-rabbitmq"
379
+ }]
380
+ actual = ExodusHelper.generate_babel_tasks(job, clouds_to_run_task_on)
381
+ assert_equal(expected, actual)
382
+ end
383
+
384
+
385
+ def test_generate_babel_tasks_many_tasks
386
+ job = {
387
+ :clouds_to_use => [:AmazonEC2, :Eucalyptus, :GoogleAppEngine],
388
+ :credentials => {
389
+ # ec2 credentials
390
+ :EC2_ACCESS_KEY => "boo",
391
+ :EC2_SECRET_KEY => "baz",
392
+ :EC2_URL => "http://ec2.url",
393
+ :S3_URL => "http://s3.url",
394
+ :S3_bucket_name => "bazbucket1",
395
+
396
+ # google app engine credentials
397
+ :appid => "bazappid",
398
+ :appcfg_cookies => "~/.appcfg_cookies",
399
+ :function => "bazboo()",
400
+ :GStorage_bucket_name => "bazbucket2"
401
+ },
402
+ :code => "/foo/bar.rb",
403
+ :argv => [2],
404
+ :executable => "ruby",
405
+ :optimize_for => :cost
406
+ }
407
+
408
+ clouds_to_run_task_on = [:AmazonEC2, :GoogleAppEngine]
409
+
410
+ ec2_task = {
411
+ :type => "babel",
412
+ :EC2_ACCESS_KEY => "boo",
413
+ :EC2_SECRET_KEY => "baz",
414
+ :EC2_URL => "http://ec2.url",
415
+ :S3_URL => "http://s3.url",
416
+ :S3_bucket_name => "bazbucket1",
417
+ :code => "/foo/bar.rb",
418
+ :argv => [2],
419
+ :executable => "ruby",
420
+ :is_remote => false,
421
+ :run_local => false,
422
+ :storage => "s3",
423
+ :engine => "executor-sqs"
424
+ }
425
+
426
+ appengine_task = {
427
+ :type => "babel",
428
+ :appid => "bazappid",
429
+ :appcfg_cookies => "~/.appcfg_cookies",
430
+ :function => "bazboo()",
431
+ :GStorage_bucket_name => "bazbucket2",
432
+ :code => "/foo/bar.rb",
433
+ :argv => [2],
434
+ :executable => "ruby",
435
+ :is_remote => false,
436
+ :run_local => false,
437
+ :storage => "gstorage",
438
+ :engine => "appengine-push-q"
439
+ }
440
+
441
+ expected = [ec2_task, appengine_task]
442
+ actual = ExodusHelper.generate_babel_tasks(job, clouds_to_run_task_on)
443
+ assert_equal(expected, actual)
444
+ end
445
+
446
+
447
+ def test_exodus_run_job_with_one_babel_task
448
+ ec2_task = {
449
+ :type => "babel",
450
+ :EC2_ACCESS_KEY => "boo",
451
+ :EC2_SECRET_KEY => "baz",
452
+ :EC2_URL => "http://ec2.url",
453
+ :S3_URL => "http://s3.url",
454
+ :code => "/foo/bar.rb",
455
+ :argv => [2],
456
+ :executable => "ruby",
457
+ :is_remote => false,
458
+ :run_local => false,
459
+ :storage => "s3",
460
+ :engine => "executor-sqs",
461
+ :bucket_name => "bazbucket"
462
+ }
463
+ babel_tasks = [ec2_task]
464
+
465
+ # mock out calls to the NeptuneManager
466
+ flexmock(NeptuneManagerClient).new_instances { |instance|
467
+ # let's say that all checks to see if temp files exist tell us that
468
+ # the files don't exist
469
+ instance.should_receive(:does_file_exist?).
470
+ with(/\A\/bazbucket\/babel\/temp-[\w]+\Z/, Hash).
471
+ and_return(false)
472
+
473
+ # assume that our code got put in the remote datastore fine
474
+ instance.should_receive(:does_file_exist?).
475
+ with(/\A\/bazbucket\/babel\/foo\/bar.rb\Z/, Hash).
476
+ and_return(true)
477
+
478
+ # also, calls to put_input should succeed
479
+ instance.should_receive(:put_input).with(Hash).and_return(true)
480
+
481
+ # mock out the call to get_supported_babel_engines and put in
482
+ # SQS and rabbitmq (which is always supported)
483
+ instance.should_receive(:get_supported_babel_engines).with(Hash).
484
+ and_return(["executor-rabbitmq", "executor-sqs"])
485
+
486
+ # neptune jobs should start fine
487
+ instance.should_receive(:start_neptune_job).with(Hash).
488
+ and_return("babel job is now running")
489
+
490
+ # getting the output of the job should return it the first time
491
+ instance.should_receive(:get_output).with(Hash).
492
+ and_return("task output yay!")
493
+ }
494
+
495
+ # mock out filesystem checks
496
+ # we'll say that our code does exist
497
+ flexmock(File).should_receive(:exists?).with("/foo").and_return(true)
498
+
499
+ # mock out scp calls - assume they go through with no problems
500
+ flexmock(CommonFunctions).should_receive(:shell).with(/\Ascp/).
501
+ and_return()
502
+
503
+ dispatched_tasks = ExodusHelper.run_job(babel_tasks)
504
+ exodus_task = ExodusTaskInfo.new(dispatched_tasks)
505
+
506
+ expected = "task output yay!"
507
+ assert_equal(expected, exodus_task.to_s)
508
+ end
509
+
510
+
511
+ def test_exodus_job_that_generates_one_babel_task
512
+ job = {
513
+ :clouds_to_use => :AmazonEC2,
514
+ :credentials => {
515
+ :EC2_ACCESS_KEY => "boo",
516
+ :EC2_SECRET_KEY => "baz",
517
+ :EC2_URL => "http://ec2.url",
518
+ :S3_URL => "http://s3.url",
519
+ :S3_bucket_name => "bazbucket"
520
+ },
521
+ :optimize_for => :cost,
522
+ :code => "/foo/bar.rb",
523
+ :argv => [],
524
+ :executable => "ruby"
525
+ }
526
+
527
+ # mock out calls to the NeptuneManager
528
+ flexmock(NeptuneManagerClient).new_instances { |instance|
529
+ # for this test, let's say there's no data on this task right now
530
+ instance.should_receive(:get_profiling_info).with(String).
531
+ and_return({})
532
+
533
+ # let's say that all checks to see if temp files exist tell us that
534
+ # the files don't exist
535
+ instance.should_receive(:does_file_exist?).
536
+ with(/\A\/bazbucket\/babel\/temp-[\w]+\Z/, Hash).
537
+ and_return(false)
538
+
539
+ # assume that our code got put in the remote datastore fine
540
+ instance.should_receive(:does_file_exist?).
541
+ with(/\A\/bazbucket\/babel\/foo\/bar.rb\Z/, Hash).
542
+ and_return(true)
543
+
544
+ # also, calls to put_input should succeed
545
+ instance.should_receive(:put_input).with(Hash).and_return(true)
546
+
547
+ # mock out the call to get_supported_babel_engines and put in
548
+ # SQS and rabbitmq (which is always supported)
549
+ instance.should_receive(:get_supported_babel_engines).with(Hash).
550
+ and_return(["executor-rabbitmq", "executor-sqs"])
551
+
552
+ # neptune jobs should start fine
553
+ instance.should_receive(:start_neptune_job).with(Hash).
554
+ and_return("babel job is now running")
555
+
556
+ # getting the output of the job should return it the first time
557
+ instance.should_receive(:get_output).with(Hash).
558
+ and_return("task output yay!")
559
+ }
560
+
561
+ # mock out filesystem checks
562
+ # we'll say that our code does exist
563
+ flexmock(File).should_receive(:exists?).with("/foo").and_return(true)
564
+
565
+ # mock out scp calls - assume they go through with no problems
566
+ flexmock(CommonFunctions).should_receive(:shell).with(/\Ascp/).
567
+ and_return()
568
+
569
+ # first, let's make sure that exodus calls work fine if we give it
570
+ # a Hash, containing info on one job
571
+ expected = "task output yay!"
572
+ actual = exodus(job)
573
+ assert_equal(expected, actual.to_s)
574
+ assert_equal(expected, actual.stdout)
575
+
576
+ # now, let's make sure that exodus calls still work the same way if we
577
+ # give it an Array, containing info on the same job
578
+ actual2 = exodus([job])
579
+ assert_equal(expected, actual2[0].to_s)
580
+ assert_equal(expected, actual2[0].stdout)
581
+ end
582
+
583
+
584
+ def test_exodus_job_that_generates_two_babel_tasks
585
+ job = {
586
+ :clouds_to_use => [:AmazonEC2, :MicrosoftAzure],
587
+ :credentials => {
588
+ # for EC2
589
+ :EC2_ACCESS_KEY => "boo",
590
+ :EC2_SECRET_KEY => "baz",
591
+ :EC2_URL => "http://ec2.url",
592
+ :S3_URL => "http://s3.url",
593
+ :S3_bucket_name => "bazbucket",
594
+
595
+ # for azure
596
+ :WAZ_Account_Name => "wazboo",
597
+ :WAZ_Access_Key => "wazbaz",
598
+ :WAZ_Container_Name => "wazbucket"
599
+ },
600
+ :optimize_for => :cost,
601
+ :code => "/foo/bar.rb",
602
+ :argv => [],
603
+ :executable => "ruby"
604
+ }
605
+
606
+ job2 = job.dup
607
+
608
+ # mock out calls to the NeptuneManager
609
+ flexmock(NeptuneManagerClient).new_instances { |instance|
610
+ # for this test, let's say there's no data on this task right now
611
+ instance.should_receive(:get_profiling_info).with(String).
612
+ and_return({})
613
+
614
+ # let's say that all checks to see if temp files exist tell us that
615
+ # the files don't exist
616
+ instance.should_receive(:does_file_exist?).
617
+ with(/\A\/bazbucket\/babel\/temp-[\w]+\Z/, Hash).
618
+ and_return(false)
619
+
620
+ instance.should_receive(:does_file_exist?).
621
+ with(/\A\/wazbucket\/babel\/temp-[\w]+\Z/, Hash).
622
+ and_return(false)
623
+
624
+ # assume that our code got put in the remote datastore fine
625
+ instance.should_receive(:does_file_exist?).
626
+ with(/\A\/bazbucket\/babel\/foo\/bar.rb\Z/, Hash).
627
+ and_return(true)
628
+
629
+ instance.should_receive(:does_file_exist?).
630
+ with(/\A\/wazbucket\/babel\/foo\/bar.rb\Z/, Hash).
631
+ and_return(true)
632
+
633
+ # also, calls to put_input should succeed
634
+ instance.should_receive(:put_input).with(Hash).and_return(true)
635
+
636
+ # mock out the call to get_supported_babel_engines and put in
637
+ # SQS, WAZ-push-q, and rabbitmq (which is always supported)
638
+ instance.should_receive(:get_supported_babel_engines).with(Hash).
639
+ and_return(["executor-rabbitmq", "executor-sqs", "waz-push-q"])
640
+
641
+ # neptune jobs should start fine - instead of expecting a Hash (like
642
+ # in the last test), we're now expecting an Array, where each item
643
+ # is a Hash corresponding to each of the jobs that will be run
644
+ instance.should_receive(:start_neptune_job).with(Array).
645
+ and_return("babel job is now running")
646
+
647
+ # getting the output of the job shouldn't return it the first time
648
+ # (unlike the last test). this time, we want to make sure that
649
+ # ExodusTaskInfo is properly asking all the babel tasks it's hiding
650
+ # from us for the job's output, so we'll have the first babel task
651
+ # always fail to have the output ready, and the second one will have
652
+ # it the second time around.
653
+ instance.should_receive(:get_output).with(on { |job|
654
+ job["@output"] =~ /\A\/bazbucket\/babel\/temp-/
655
+ }).
656
+ and_return(DOES_NOT_EXIST)
657
+
658
+ instance.should_receive(:get_output).with(on { |job|
659
+ job["@output"] =~ /\A\/wazbucket\/babel\/temp-/
660
+ }).
661
+ and_return("task output yay!")
662
+ }
663
+
664
+ # mock out filesystem checks
665
+ # we'll say that our code does exist
666
+ flexmock(File).should_receive(:exists?).with("/foo").and_return(true)
667
+
668
+ # mock out scp calls - assume they go through with no problems
669
+ flexmock(CommonFunctions).should_receive(:shell).with(/\Ascp/).
670
+ and_return()
671
+
672
+ # first, let's make sure that exodus calls work fine if we give it
673
+ # a Hash, containing info on one job
674
+ expected = "task output yay!"
675
+ actual = exodus(job)
676
+ assert_equal(expected, actual.to_s)
677
+ assert_equal(expected, actual.stdout)
678
+
679
+ # now, let's make sure that exodus calls still work the same way if we
680
+ # give it an Array, containing info on the same job
681
+ actual2 = exodus([job2])
682
+ assert_equal(expected, actual2[0].to_s)
683
+ assert_equal(expected, actual2[0].stdout)
684
+ end
685
+
686
+
687
+ end