neptune 0.2.1 → 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. data/README +4 -0
  2. data/doc/BabelHelper.html +393 -376
  3. data/doc/BadConfigurationException.html +121 -127
  4. data/doc/CommonFunctions.html +237 -265
  5. data/doc/ExodusHelper.html +820 -0
  6. data/doc/ExodusTaskInfo.html +263 -0
  7. data/doc/FileNotFoundException.html +121 -127
  8. data/doc/NeptuneHelper.html +527 -592
  9. data/doc/NeptuneManagerClient.html +696 -0
  10. data/doc/NeptuneManagerException.html +139 -0
  11. data/doc/Object.html +334 -236
  12. data/doc/TaskInfo.html +428 -0
  13. data/doc/created.rid +8 -5
  14. data/doc/images/add.png +0 -0
  15. data/doc/images/delete.png +0 -0
  16. data/doc/images/tag_blue.png +0 -0
  17. data/doc/images/transparent.png +0 -0
  18. data/doc/index.html +74 -142
  19. data/doc/js/darkfish.js +99 -62
  20. data/doc/js/jquery.js +15 -29
  21. data/doc/js/navigation.js +142 -0
  22. data/doc/js/search.js +94 -0
  23. data/doc/js/search_index.js +1 -0
  24. data/doc/js/searcher.js +228 -0
  25. data/doc/table_of_contents.html +226 -0
  26. data/lib/babel.rb +116 -50
  27. data/lib/custom_exceptions.rb +2 -2
  28. data/lib/exodus.rb +311 -0
  29. data/lib/exodus_task_info.rb +36 -0
  30. data/lib/neptune.rb +52 -18
  31. data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
  32. data/lib/task_info.rb +155 -0
  33. data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
  34. data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
  35. data/test/test_exodus.rb +687 -0
  36. data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
  37. data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
  38. data/test/test_task_info.rb +32 -0
  39. data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
  40. metadata +30 -34
  41. data/doc/AppControllerClient.html +0 -702
  42. data/doc/AppControllerException.html +0 -145
  43. data/doc/bin/neptune.html +0 -56
  44. data/doc/js/quicksearch.js +0 -114
  45. data/doc/js/thickbox-compressed.js +0 -10
  46. data/doc/lib/app_controller_client_rb.html +0 -60
  47. data/doc/lib/babel_rb.html +0 -68
  48. data/doc/lib/common_functions_rb.html +0 -70
  49. data/doc/lib/custom_exceptions_rb.html +0 -54
  50. data/doc/lib/neptune_rb.html +0 -60
  51. data/test/integration/tc_c.rb +0 -57
  52. data/test/integration/tc_dfsp.rb +0 -37
  53. data/test/integration/tc_dwssa.rb +0 -38
  54. data/test/integration/tc_erlang.rb +0 -183
  55. data/test/integration/tc_mapreduce.rb +0 -282
  56. data/test/integration/tc_mpi.rb +0 -160
  57. data/test/integration/tc_storage.rb +0 -209
  58. data/test/integration/tc_upc.rb +0 -75
  59. data/test/integration/tc_x10.rb +0 -94
  60. data/test/integration/test_helper.rb +0 -135
  61. data/test/integration/ts_neptune.rb +0 -40
@@ -0,0 +1,226 @@
1
+ <!DOCTYPE html>
2
+
3
+ <html>
4
+ <head>
5
+ <meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
6
+
7
+ <title>Table of Contents - RDoc Documentation</title>
8
+
9
+ <link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
10
+
11
+ <script type="text/javascript">
12
+ var rdoc_rel_prefix = "./";
13
+ </script>
14
+
15
+ <script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
16
+ <script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
17
+ <script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
18
+ <script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
19
+ <script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
20
+ <script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
21
+
22
+
23
+ <body class="indexpage">
24
+ <h1>Table of Contents - RDoc Documentation</h1>
25
+
26
+
27
+ <h2 id="classes">Classes/Modules</h2>
28
+ <ul>
29
+ <li class="module">
30
+ <a href="BabelHelper.html">BabelHelper</a>
31
+ </li>
32
+ <li class="class">
33
+ <a href="BadConfigurationException.html">BadConfigurationException</a>
34
+ </li>
35
+ <li class="module">
36
+ <a href="CommonFunctions.html">CommonFunctions</a>
37
+ </li>
38
+ <li class="module">
39
+ <a href="ExodusHelper.html">ExodusHelper</a>
40
+ </li>
41
+ <li class="class">
42
+ <a href="ExodusTaskInfo.html">ExodusTaskInfo</a>
43
+ </li>
44
+ <li class="class">
45
+ <a href="FileNotFoundException.html">FileNotFoundException</a>
46
+ </li>
47
+ <li class="module">
48
+ <a href="NeptuneHelper.html">NeptuneHelper</a>
49
+ </li>
50
+ <li class="class">
51
+ <a href="NeptuneManagerClient.html">NeptuneManagerClient</a>
52
+ </li>
53
+ <li class="class">
54
+ <a href="NeptuneManagerException.html">NeptuneManagerException</a>
55
+ </li>
56
+ <li class="class">
57
+ <a href="Object.html">Object</a>
58
+ </li>
59
+ <li class="class">
60
+ <a href="TaskInfo.html">TaskInfo</a>
61
+ </li>
62
+
63
+ </ul>
64
+
65
+ <h2 id="methods">Methods</h2>
66
+ <ul>
67
+
68
+ <li class="method"><a href="ExodusHelper.html#method-c-average">::average &mdash; ExodusHelper</a>
69
+
70
+ <li class="method"><a href="BabelHelper.html#method-c-check_output_files">::check_output_files &mdash; BabelHelper</a>
71
+
72
+ <li class="method"><a href="NeptuneHelper.html#method-c-compile_code">::compile_code &mdash; NeptuneHelper</a>
73
+
74
+ <li class="method"><a href="ExodusHelper.html#method-c-convert_clouds_to_use_to_array">::convert_clouds_to_use_to_array &mdash; ExodusHelper</a>
75
+
76
+ <li class="method"><a href="BabelHelper.html#method-c-convert_from_neptune_params">::convert_from_neptune_params &mdash; BabelHelper</a>
77
+
78
+ <li class="method"><a href="BabelHelper.html#method-c-convert_to_neptune_params">::convert_to_neptune_params &mdash; BabelHelper</a>
79
+
80
+ <li class="method"><a href="NeptuneHelper.html#method-c-do_preprocessing">::do_preprocessing &mdash; NeptuneHelper</a>
81
+
82
+ <li class="method"><a href="ExodusHelper.html#method-c-ensure_all_jobs_are_hashes">::ensure_all_jobs_are_hashes &mdash; ExodusHelper</a>
83
+
84
+ <li class="method"><a href="ExodusHelper.html#method-c-ensure_all_params_are_present">::ensure_all_params_are_present &mdash; ExodusHelper</a>
85
+
86
+ <li class="method"><a href="ExodusHelper.html#method-c-ensure_credentials_are_in_correct_format">::ensure_credentials_are_in_correct_format &mdash; ExodusHelper</a>
87
+
88
+ <li class="method"><a href="BabelHelper.html#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist &mdash; BabelHelper</a>
89
+
90
+ <li class="method"><a href="ExodusHelper.html#method-c-find_optimal_cloud_for_task">::find_optimal_cloud_for_task &mdash; ExodusHelper</a>
91
+
92
+ <li class="method"><a href="ExodusHelper.html#method-c-generate_babel_tasks">::generate_babel_tasks &mdash; ExodusHelper</a>
93
+
94
+ <li class="method"><a href="BabelHelper.html#method-c-generate_output_location">::generate_output_location &mdash; BabelHelper</a>
95
+
96
+ <li class="method"><a href="BabelHelper.html#method-c-get_bucket_for_local_data">::get_bucket_for_local_data &mdash; BabelHelper</a>
97
+
98
+ <li class="method"><a href="ExodusHelper.html#method-c-get_clouds_to_run_task_on">::get_clouds_to_run_task_on &mdash; ExodusHelper</a>
99
+
100
+ <li class="method"><a href="CommonFunctions.html#method-c-get_from_yaml">::get_from_yaml &mdash; CommonFunctions</a>
101
+
102
+ <li class="method"><a href="NeptuneHelper.html#method-c-get_input">::get_input &mdash; NeptuneHelper</a>
103
+
104
+ <li class="method"><a href="NeptuneHelper.html#method-c-get_job_data">::get_job_data &mdash; NeptuneHelper</a>
105
+
106
+ <li class="method"><a href="ExodusHelper.html#method-c-get_key_from_job_data">::get_key_from_job_data &mdash; ExodusHelper</a>
107
+
108
+ <li class="method"><a href="ExodusHelper.html#method-c-get_minimum_val_in_data">::get_minimum_val_in_data &mdash; ExodusHelper</a>
109
+
110
+ <li class="method"><a href="BabelHelper.html#method-c-get_neptune_manager_client">::get_neptune_manager_client &mdash; BabelHelper</a>
111
+
112
+ <li class="method"><a href="ExodusHelper.html#method-c-get_profiling_info">::get_profiling_info &mdash; ExodusHelper</a>
113
+
114
+ <li class="method"><a href="CommonFunctions.html#method-c-get_random_alphanumeric">::get_random_alphanumeric &mdash; CommonFunctions</a>
115
+
116
+ <li class="method"><a href="CommonFunctions.html#method-c-get_secret_key">::get_secret_key &mdash; CommonFunctions</a>
117
+
118
+ <li class="method"><a href="NeptuneHelper.html#method-c-get_std_out_and_err">::get_std_out_and_err &mdash; NeptuneHelper</a>
119
+
120
+ <li class="method"><a href="ExodusTaskInfo.html#method-c-new">::new &mdash; ExodusTaskInfo</a>
121
+
122
+ <li class="method"><a href="NeptuneManagerClient.html#method-c-new">::new &mdash; NeptuneManagerClient</a>
123
+
124
+ <li class="method"><a href="TaskInfo.html#method-c-new">::new &mdash; TaskInfo</a>
125
+
126
+ <li class="method"><a href="NeptuneHelper.html#method-c-preprocess_babel">::preprocess_babel &mdash; NeptuneHelper</a>
127
+
128
+ <li class="method"><a href="NeptuneHelper.html#method-c-preprocess_compile">::preprocess_compile &mdash; NeptuneHelper</a>
129
+
130
+ <li class="method"><a href="NeptuneHelper.html#method-c-preprocess_erlang">::preprocess_erlang &mdash; NeptuneHelper</a>
131
+
132
+ <li class="method"><a href="NeptuneHelper.html#method-c-preprocess_mpi">::preprocess_mpi &mdash; NeptuneHelper</a>
133
+
134
+ <li class="method"><a href="NeptuneHelper.html#method-c-preprocess_ssa">::preprocess_ssa &mdash; NeptuneHelper</a>
135
+
136
+ <li class="method"><a href="ExodusHelper.html#method-c-propogate_credentials_from_environment">::propogate_credentials_from_environment &mdash; ExodusHelper</a>
137
+
138
+ <li class="method"><a href="BabelHelper.html#method-c-put_code">::put_code &mdash; BabelHelper</a>
139
+
140
+ <li class="method"><a href="BabelHelper.html#method-c-put_file">::put_file &mdash; BabelHelper</a>
141
+
142
+ <li class="method"><a href="BabelHelper.html#method-c-put_inputs">::put_inputs &mdash; BabelHelper</a>
143
+
144
+ <li class="method"><a href="NeptuneHelper.html#method-c-require_file_to_exist">::require_file_to_exist &mdash; NeptuneHelper</a>
145
+
146
+ <li class="method"><a href="NeptuneHelper.html#method-c-require_file_to_not_exist">::require_file_to_not_exist &mdash; NeptuneHelper</a>
147
+
148
+ <li class="method"><a href="NeptuneHelper.html#method-c-require_param">::require_param &mdash; NeptuneHelper</a>
149
+
150
+ <li class="method"><a href="BabelHelper.html#method-c-run_job">::run_job &mdash; BabelHelper</a>
151
+
152
+ <li class="method"><a href="NeptuneHelper.html#method-c-run_job">::run_job &mdash; NeptuneHelper</a>
153
+
154
+ <li class="method"><a href="ExodusHelper.html#method-c-run_job">::run_job &mdash; ExodusHelper</a>
155
+
156
+ <li class="method"><a href="CommonFunctions.html#method-c-scp_file">::scp_file &mdash; CommonFunctions</a>
157
+
158
+ <li class="method"><a href="CommonFunctions.html#method-c-scp_to_shadow">::scp_to_shadow &mdash; CommonFunctions</a>
159
+
160
+ <li class="method"><a href="CommonFunctions.html#method-c-shell">::shell &mdash; CommonFunctions</a>
161
+
162
+ <li class="method"><a href="NeptuneHelper.html#method-c-upload_app_for_cicero">::upload_app_for_cicero &mdash; NeptuneHelper</a>
163
+
164
+ <li class="method"><a href="ExodusHelper.html#method-c-validate_clouds_to_use">::validate_clouds_to_use &mdash; ExodusHelper</a>
165
+
166
+ <li class="method"><a href="ExodusHelper.html#method-c-validate_files_argv_executable">::validate_files_argv_executable &mdash; ExodusHelper</a>
167
+
168
+ <li class="method"><a href="BabelHelper.html#method-c-validate_inputs">::validate_inputs &mdash; BabelHelper</a>
169
+
170
+ <li class="method"><a href="ExodusHelper.html#method-c-validate_optimize_for_param">::validate_optimize_for_param &mdash; ExodusHelper</a>
171
+
172
+ <li class="method"><a href="NeptuneHelper.html#method-c-validate_storage_params">::validate_storage_params &mdash; NeptuneHelper</a>
173
+
174
+ <li class="method"><a href="BabelHelper.html#method-c-wait_and_get_output">::wait_and_get_output &mdash; BabelHelper</a>
175
+
176
+ <li class="method"><a href="NeptuneHelper.html#method-c-wait_for_compilation_to_finish">::wait_for_compilation_to_finish &mdash; NeptuneHelper</a>
177
+
178
+ <li class="method"><a href="Object.html#method-i-babel">#babel &mdash; Object</a>
179
+
180
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-compile_code">#compile_code &mdash; NeptuneManagerClient</a>
181
+
182
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-does_file_exist-3F">#does_file_exist? &mdash; NeptuneManagerClient</a>
183
+
184
+ <li class="method"><a href="Object.html#method-i-exodus">#exodus &mdash; Object</a>
185
+
186
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-get_acl">#get_acl &mdash; NeptuneManagerClient</a>
187
+
188
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-get_output">#get_output &mdash; NeptuneManagerClient</a>
189
+
190
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-get_profiling_info">#get_profiling_info &mdash; NeptuneManagerClient</a>
191
+
192
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-get_supported_babel_engines">#get_supported_babel_engines &mdash; NeptuneManagerClient</a>
193
+
194
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-make_call">#make_call &mdash; NeptuneManagerClient</a>
195
+
196
+ <li class="method"><a href="ExodusTaskInfo.html#method-i-method_missing">#method_missing &mdash; ExodusTaskInfo</a>
197
+
198
+ <li class="method"><a href="Object.html#method-i-neptune">#neptune &mdash; Object</a>
199
+
200
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-put_input">#put_input &mdash; NeptuneManagerClient</a>
201
+
202
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-set_acl">#set_acl &mdash; NeptuneManagerClient</a>
203
+
204
+ <li class="method"><a href="NeptuneManagerClient.html#method-i-start_neptune_job">#start_neptune_job &mdash; NeptuneManagerClient</a>
205
+
206
+ <li class="method"><a href="TaskInfo.html#method-i-stderr">#stderr &mdash; TaskInfo</a>
207
+
208
+ <li class="method"><a href="TaskInfo.html#method-i-stdout">#stdout &mdash; TaskInfo</a>
209
+
210
+ <li class="method"><a href="TaskInfo.html#method-i-success-3F">#success? &mdash; TaskInfo</a>
211
+
212
+ <li class="method"><a href="TaskInfo.html#method-i-to_json">#to_json &mdash; TaskInfo</a>
213
+
214
+ <li class="method"><a href="ExodusTaskInfo.html#method-i-to_s">#to_s &mdash; ExodusTaskInfo</a>
215
+
216
+ <li class="method"><a href="TaskInfo.html#method-i-to_s">#to_s &mdash; TaskInfo</a>
217
+
218
+ </ul>
219
+
220
+
221
+ <footer id="validator-badges">
222
+ <p><a href="http://validator.w3.org/check/referer">[Validate]</a>
223
+ <p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
224
+ <p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
225
+ </footer>
226
+
@@ -1,10 +1,11 @@
1
1
  #!/usr/bin/ruby
2
2
  # Programmer: Chris Bunch (cgb@cs.ucsb.edu)
3
3
 
4
- require 'app_controller_client'
5
4
  require 'common_functions'
6
5
  require 'custom_exceptions'
7
6
  require 'neptune'
7
+ require 'neptune_manager_client'
8
+ require 'task_info'
8
9
 
9
10
 
10
11
  # The promise gem gives us futures / promises out-of-the-box, which we need
@@ -40,39 +41,61 @@ MAX_SLEEP_TIME = 60 # seconds
40
41
  # write multiple Neptune jobs to actually run code (e.g., putting input in the
41
42
  # datastore, run the job, get the output back), Babel automatically handles
42
43
  # this.
43
- def babel(params)
44
+ def babel(jobs)
44
45
  # Since this whole function should run asynchronously, we run it as a future.
45
46
  # It automatically starts running in a new thread, and attempting to get the
46
47
  # value of what this returns causes it to block until the job completes.
47
- future {
48
- if params[:storage]
49
- params[:is_remote] = true
48
+ #future {
49
+ if jobs.class == Hash
50
+ was_one_job = true
51
+ jobs = [jobs]
50
52
  else
51
- params[:is_remote] = false
53
+ was_one_job = false
52
54
  end
53
55
 
54
- job_data = BabelHelper.convert_from_neptune_params(params)
56
+ job_data_list = []
57
+ jobs.each { |params|
58
+ job_data = BabelHelper.convert_from_neptune_params(params)
59
+ job_data['@metadata_info'] = {'time_to_store_inputs' => 0.0}
55
60
 
56
- NeptuneHelper.validate_storage_params(job_data) # adds in S3 storage params
61
+ # Add in S3 storage parameters
62
+ NeptuneHelper.validate_storage_params(job_data)
57
63
 
58
- # :code is the only required parameter - everything else can use default vals
59
- NeptuneHelper.require_param("@code", job_data)
64
+ # :code is the only required parameter
65
+ # everything else can use default vals
66
+ NeptuneHelper.require_param("@code", job_data)
67
+ BabelHelper.check_output_files(job_data)
60
68
 
61
- if job_data["@output"].nil? or job_data["@output"].empty?
62
- job_data["@output"] = BabelHelper.generate_output_location(job_data)
63
- end
64
- BabelHelper.ensure_output_does_not_exist(job_data)
69
+ if job_data["@is_remote"]
70
+ #BabelHelper.validate_inputs(job_data)
71
+ else
72
+ BabelHelper.put_code(job_data)
73
+ BabelHelper.put_inputs(job_data)
74
+ end
65
75
 
66
- if job_data["@is_remote"]
67
- BabelHelper.validate_inputs(job_data)
76
+ job_data_list << job_data
77
+ }
78
+
79
+ BabelHelper.run_job(job_data_list)
80
+
81
+ # Return an object to the user that has all the information about their
82
+ # task, including its standard out, err, debugging info, and profiling
83
+ # info. Since the job may not be done when the user asks for this info,
84
+ # its the responsibility of TaskInfo objects to block until that info
85
+ # is ready. We don't explicitly return the TaskInfo object, because it's
86
+ # in a Future block - it will automatically return whatever the last
87
+ # statement returns.
88
+ tasks = []
89
+ job_data_list.each { |job_data|
90
+ tasks << TaskInfo.new(job_data)
91
+ }
92
+
93
+ if was_one_job
94
+ tasks[0]
68
95
  else
69
- BabelHelper.put_code(job_data)
70
- BabelHelper.put_inputs(job_data)
96
+ tasks
71
97
  end
72
-
73
- BabelHelper.run_job(job_data)
74
- BabelHelper.wait_and_get_output(job_data)
75
- }
98
+ #}
76
99
  end
77
100
 
78
101
 
@@ -84,7 +107,7 @@ module BabelHelper
84
107
  # one for them, based on either the location of their code (for remotely
85
108
  # specified code), or a babel parameter (for locally specified code).
86
109
  def self.generate_output_location(job_data)
87
- if job_data["@storage"]
110
+ if job_data["@is_remote"]
88
111
  # We already know the bucket name - the same one that the user
89
112
  # has told us their code is located in.
90
113
  prefix = job_data["@code"].scan(/\/(.*?)\//)[0].to_s
@@ -99,7 +122,9 @@ module BabelHelper
99
122
  # Provides a common way for callers to get the name of the bucket that
100
123
  # should be used for Neptune jobs where the code is stored locally.
101
124
  def self.get_bucket_for_local_data(job_data)
102
- bucket_name = job_data["@bucket_name"] || ENV['BABEL_BUCKET_NAME']
125
+ bucket_name = job_data["@bucket_name"] || ENV['BABEL_BUCKET_NAME'] ||
126
+ job_data["@S3_bucket_name"] || job_data["@Walrus_bucket_name"] ||
127
+ job_data["@GStorage_bucket_name"] || job_data["@WAZ_Container_Name"]
103
128
 
104
129
  if bucket_name.nil?
105
130
  raise BadConfigurationException.new(NEEDS_BUCKET_INFO)
@@ -113,12 +138,26 @@ module BabelHelper
113
138
  return bucket_name
114
139
  end
115
140
 
141
+
142
+ # babel() callers do not have to specify a location where the standard output
143
+ # and error the task produces should be placed. If they don't, generate
144
+ # locations for them and make sure they don't exist beforehand.
145
+ def self.check_output_files(job_data)
146
+ ["@output", "@error", "@metadata"].each { |item|
147
+ if job_data[item].nil? or job_data[item].empty?
148
+ job_data[item] = BabelHelper.generate_output_location(job_data)
149
+ else
150
+ BabelHelper.ensure_output_does_not_exist(job_data, job_data[item])
151
+ end
152
+ }
153
+ end
154
+
116
155
 
117
156
  # For jobs where the code is stored remotely, this method ensures that
118
157
  # the code and any possible inputs actually do exist, before attempting to
119
158
  # use them for computation.
120
159
  def self.validate_inputs(job_data)
121
- controller = self.get_appcontroller(job_data)
160
+ controller = self.get_neptune_manager_client(job_data)
122
161
 
123
162
  # First, make sure the code exists
124
163
  NeptuneHelper.require_file_to_exist(job_data["@code"], job_data, controller)
@@ -139,20 +178,18 @@ module BabelHelper
139
178
  # To avoid accidentally overwriting outputs from previous jobs, we first
140
179
  # check to make sure an output file doesn't exist before starting a new job
141
180
  # with the given name.
142
- def self.ensure_output_does_not_exist(job_data)
143
- file = job_data["@output"]
144
- controller = self.get_appcontroller(job_data)
145
- # Kernel.puts job_data.inspect
146
- NeptuneHelper.require_file_to_not_exist(file, job_data, controller)
181
+ def self.ensure_output_does_not_exist(job_data, remote_file)
182
+ controller = self.get_neptune_manager_client(job_data)
183
+ NeptuneHelper.require_file_to_not_exist(remote_file, job_data, controller)
147
184
  end
148
185
 
149
186
 
150
- # Returns an AppControllerClient for the given job data.
151
- def self.get_appcontroller(job_data)
187
+ # Returns an NeptuneManagerClient for the given job data.
188
+ def self.get_neptune_manager_client(job_data)
152
189
  keyname = job_data["@keyname"] || "appscale"
153
190
  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
154
191
  secret = CommonFunctions.get_secret_key(keyname)
155
- return AppControllerClient.new(shadow_ip, secret)
192
+ return NeptuneManagerClient.new(shadow_ip, secret)
156
193
  end
157
194
 
158
195
 
@@ -198,7 +235,15 @@ module BabelHelper
198
235
  bucket_name = self.get_bucket_for_local_data(job_data)
199
236
  input_data[:remote] = "/#{bucket_name}/babel#{local_path}"
200
237
 
238
+ start = Time.now
201
239
  Kernel.neptune(input_data)
240
+ fin = Time.now
241
+
242
+ if job_data['@metadata_info'].nil?
243
+ job_data['@metadata_info'] = {'time_to_store_inputs' => 0.0}
244
+ end
245
+
246
+ job_data['@metadata_info']['time_to_store_inputs'] += fin - start
202
247
 
203
248
  return input_data[:remote]
204
249
  end
@@ -234,31 +279,52 @@ module BabelHelper
234
279
 
235
280
  # Constructs a Neptune job to run the user's code as a Babel job (task queue)
236
281
  # from the given parameters.
237
- def self.run_job(job_data)
238
- run_data = self.convert_to_neptune_params(job_data)
282
+ def self.run_job(job_data_list)
283
+ run_data_list = []
239
284
 
240
- # Default to babel as the job type, if the user doesn't specify one.
241
- if run_data[:type].nil? or run_data[:type].empty?
242
- run_data[:type] = "babel"
243
- end
285
+ job_data_list.each { |job_data|
286
+ run_data = self.convert_to_neptune_params(job_data)
244
287
 
245
- # TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to
246
- # exec tasks over it, instead of locally.
247
- if job_data["@run_local"].nil?
248
- run_data[:run_local] = true
249
- run_data[:engine] = "executor-sqs"
250
- end
288
+ # Default to babel as the job type, if the user doesn't specify one.
289
+ if run_data[:type].nil? or run_data[:type].empty?
290
+ run_data[:type] = "babel"
291
+ end
292
+
293
+ # TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to
294
+ # exec tasks over it, instead of locally.
295
+ if job_data["@run_local"].nil?
296
+ run_data[:run_local] = true
297
+ run_data[:engine] = "executor-sqs"
298
+ end
299
+
300
+ run_data[:failed_attempts] = 0
301
+ run_data_list << run_data
302
+ }
251
303
 
252
- return Kernel.neptune(run_data)
304
+ loop {
305
+ if run_data_list.length == 1
306
+ run_job = Kernel.neptune(run_data_list[0])
307
+ else
308
+ run_job = Kernel.neptune(run_data_list)
309
+ end
310
+
311
+ if run_job[:result] == :success
312
+ return run_job
313
+ else
314
+ run_data_list[0][:failed_attempts] += 1
315
+ Kernel.sleep(SLEEP_TIME) # TODO(cgb): this should exponentially backoff
316
+ end
317
+ }
253
318
  end
254
319
 
255
320
 
256
321
  # Constructs a Neptune job to get the output of a Babel job. If the job is not
257
322
  # yet finished, this function waits until it does, and then returns the output
258
323
  # of the job.
259
- def self.wait_and_get_output(job_data)
324
+ def self.wait_and_get_output(job_data, output_location)
260
325
  output_data = self.convert_to_neptune_params(job_data)
261
326
  output_data[:type] = "output"
327
+ output_data[:output] = output_location
262
328
 
263
329
  output = ""
264
330
  time_to_sleep = SLEEP_TIME
@@ -267,9 +333,9 @@ module BabelHelper
267
333
  if output == DOES_NOT_EXIST
268
334
  # Exponentially back off, up to a limit of MAX_SLEEP_TIME
269
335
  Kernel.sleep(time_to_sleep)
270
- if time_to_sleep < MAX_SLEEP_TIME
271
- time_to_sleep *= 2
272
- end
336
+ #if time_to_sleep < MAX_SLEEP_TIME
337
+ # time_to_sleep *= 2
338
+ #end
273
339
  else
274
340
  break
275
341
  end