neptune 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +4 -0
- data/doc/BabelHelper.html +393 -376
- data/doc/BadConfigurationException.html +121 -127
- data/doc/CommonFunctions.html +237 -265
- data/doc/ExodusHelper.html +820 -0
- data/doc/ExodusTaskInfo.html +263 -0
- data/doc/FileNotFoundException.html +121 -127
- data/doc/NeptuneHelper.html +527 -592
- data/doc/NeptuneManagerClient.html +696 -0
- data/doc/NeptuneManagerException.html +139 -0
- data/doc/Object.html +334 -236
- data/doc/TaskInfo.html +428 -0
- data/doc/created.rid +8 -5
- data/doc/images/add.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/index.html +74 -142
- data/doc/js/darkfish.js +99 -62
- data/doc/js/jquery.js +15 -29
- data/doc/js/navigation.js +142 -0
- data/doc/js/search.js +94 -0
- data/doc/js/search_index.js +1 -0
- data/doc/js/searcher.js +228 -0
- data/doc/table_of_contents.html +226 -0
- data/lib/babel.rb +116 -50
- data/lib/custom_exceptions.rb +2 -2
- data/lib/exodus.rb +311 -0
- data/lib/exodus_task_info.rb +36 -0
- data/lib/neptune.rb +52 -18
- data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
- data/lib/task_info.rb +155 -0
- data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
- data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
- data/test/test_exodus.rb +687 -0
- data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
- data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
- data/test/test_task_info.rb +32 -0
- data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
- metadata +30 -34
- data/doc/AppControllerClient.html +0 -702
- data/doc/AppControllerException.html +0 -145
- data/doc/bin/neptune.html +0 -56
- data/doc/js/quicksearch.js +0 -114
- data/doc/js/thickbox-compressed.js +0 -10
- data/doc/lib/app_controller_client_rb.html +0 -60
- data/doc/lib/babel_rb.html +0 -68
- data/doc/lib/common_functions_rb.html +0 -70
- data/doc/lib/custom_exceptions_rb.html +0 -54
- data/doc/lib/neptune_rb.html +0 -60
- data/test/integration/tc_c.rb +0 -57
- data/test/integration/tc_dfsp.rb +0 -37
- data/test/integration/tc_dwssa.rb +0 -38
- data/test/integration/tc_erlang.rb +0 -183
- data/test/integration/tc_mapreduce.rb +0 -282
- data/test/integration/tc_mpi.rb +0 -160
- data/test/integration/tc_storage.rb +0 -209
- data/test/integration/tc_upc.rb +0 -75
- data/test/integration/tc_x10.rb +0 -94
- data/test/integration/test_helper.rb +0 -135
- data/test/integration/ts_neptune.rb +0 -40
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
|
|
3
|
+
<html>
|
|
4
|
+
<head>
|
|
5
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
|
6
|
+
|
|
7
|
+
<title>Table of Contents - RDoc Documentation</title>
|
|
8
|
+
|
|
9
|
+
<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
|
|
10
|
+
|
|
11
|
+
<script type="text/javascript">
|
|
12
|
+
var rdoc_rel_prefix = "./";
|
|
13
|
+
</script>
|
|
14
|
+
|
|
15
|
+
<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
|
|
16
|
+
<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
|
|
17
|
+
<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
|
|
18
|
+
<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
|
|
19
|
+
<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
|
|
20
|
+
<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
<body class="indexpage">
|
|
24
|
+
<h1>Table of Contents - RDoc Documentation</h1>
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
<h2 id="classes">Classes/Modules</h2>
|
|
28
|
+
<ul>
|
|
29
|
+
<li class="module">
|
|
30
|
+
<a href="BabelHelper.html">BabelHelper</a>
|
|
31
|
+
</li>
|
|
32
|
+
<li class="class">
|
|
33
|
+
<a href="BadConfigurationException.html">BadConfigurationException</a>
|
|
34
|
+
</li>
|
|
35
|
+
<li class="module">
|
|
36
|
+
<a href="CommonFunctions.html">CommonFunctions</a>
|
|
37
|
+
</li>
|
|
38
|
+
<li class="module">
|
|
39
|
+
<a href="ExodusHelper.html">ExodusHelper</a>
|
|
40
|
+
</li>
|
|
41
|
+
<li class="class">
|
|
42
|
+
<a href="ExodusTaskInfo.html">ExodusTaskInfo</a>
|
|
43
|
+
</li>
|
|
44
|
+
<li class="class">
|
|
45
|
+
<a href="FileNotFoundException.html">FileNotFoundException</a>
|
|
46
|
+
</li>
|
|
47
|
+
<li class="module">
|
|
48
|
+
<a href="NeptuneHelper.html">NeptuneHelper</a>
|
|
49
|
+
</li>
|
|
50
|
+
<li class="class">
|
|
51
|
+
<a href="NeptuneManagerClient.html">NeptuneManagerClient</a>
|
|
52
|
+
</li>
|
|
53
|
+
<li class="class">
|
|
54
|
+
<a href="NeptuneManagerException.html">NeptuneManagerException</a>
|
|
55
|
+
</li>
|
|
56
|
+
<li class="class">
|
|
57
|
+
<a href="Object.html">Object</a>
|
|
58
|
+
</li>
|
|
59
|
+
<li class="class">
|
|
60
|
+
<a href="TaskInfo.html">TaskInfo</a>
|
|
61
|
+
</li>
|
|
62
|
+
|
|
63
|
+
</ul>
|
|
64
|
+
|
|
65
|
+
<h2 id="methods">Methods</h2>
|
|
66
|
+
<ul>
|
|
67
|
+
|
|
68
|
+
<li class="method"><a href="ExodusHelper.html#method-c-average">::average — ExodusHelper</a>
|
|
69
|
+
|
|
70
|
+
<li class="method"><a href="BabelHelper.html#method-c-check_output_files">::check_output_files — BabelHelper</a>
|
|
71
|
+
|
|
72
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-compile_code">::compile_code — NeptuneHelper</a>
|
|
73
|
+
|
|
74
|
+
<li class="method"><a href="ExodusHelper.html#method-c-convert_clouds_to_use_to_array">::convert_clouds_to_use_to_array — ExodusHelper</a>
|
|
75
|
+
|
|
76
|
+
<li class="method"><a href="BabelHelper.html#method-c-convert_from_neptune_params">::convert_from_neptune_params — BabelHelper</a>
|
|
77
|
+
|
|
78
|
+
<li class="method"><a href="BabelHelper.html#method-c-convert_to_neptune_params">::convert_to_neptune_params — BabelHelper</a>
|
|
79
|
+
|
|
80
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-do_preprocessing">::do_preprocessing — NeptuneHelper</a>
|
|
81
|
+
|
|
82
|
+
<li class="method"><a href="ExodusHelper.html#method-c-ensure_all_jobs_are_hashes">::ensure_all_jobs_are_hashes — ExodusHelper</a>
|
|
83
|
+
|
|
84
|
+
<li class="method"><a href="ExodusHelper.html#method-c-ensure_all_params_are_present">::ensure_all_params_are_present — ExodusHelper</a>
|
|
85
|
+
|
|
86
|
+
<li class="method"><a href="ExodusHelper.html#method-c-ensure_credentials_are_in_correct_format">::ensure_credentials_are_in_correct_format — ExodusHelper</a>
|
|
87
|
+
|
|
88
|
+
<li class="method"><a href="BabelHelper.html#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist — BabelHelper</a>
|
|
89
|
+
|
|
90
|
+
<li class="method"><a href="ExodusHelper.html#method-c-find_optimal_cloud_for_task">::find_optimal_cloud_for_task — ExodusHelper</a>
|
|
91
|
+
|
|
92
|
+
<li class="method"><a href="ExodusHelper.html#method-c-generate_babel_tasks">::generate_babel_tasks — ExodusHelper</a>
|
|
93
|
+
|
|
94
|
+
<li class="method"><a href="BabelHelper.html#method-c-generate_output_location">::generate_output_location — BabelHelper</a>
|
|
95
|
+
|
|
96
|
+
<li class="method"><a href="BabelHelper.html#method-c-get_bucket_for_local_data">::get_bucket_for_local_data — BabelHelper</a>
|
|
97
|
+
|
|
98
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_clouds_to_run_task_on">::get_clouds_to_run_task_on — ExodusHelper</a>
|
|
99
|
+
|
|
100
|
+
<li class="method"><a href="CommonFunctions.html#method-c-get_from_yaml">::get_from_yaml — CommonFunctions</a>
|
|
101
|
+
|
|
102
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-get_input">::get_input — NeptuneHelper</a>
|
|
103
|
+
|
|
104
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-get_job_data">::get_job_data — NeptuneHelper</a>
|
|
105
|
+
|
|
106
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_key_from_job_data">::get_key_from_job_data — ExodusHelper</a>
|
|
107
|
+
|
|
108
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_minimum_val_in_data">::get_minimum_val_in_data — ExodusHelper</a>
|
|
109
|
+
|
|
110
|
+
<li class="method"><a href="BabelHelper.html#method-c-get_neptune_manager_client">::get_neptune_manager_client — BabelHelper</a>
|
|
111
|
+
|
|
112
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_profiling_info">::get_profiling_info — ExodusHelper</a>
|
|
113
|
+
|
|
114
|
+
<li class="method"><a href="CommonFunctions.html#method-c-get_random_alphanumeric">::get_random_alphanumeric — CommonFunctions</a>
|
|
115
|
+
|
|
116
|
+
<li class="method"><a href="CommonFunctions.html#method-c-get_secret_key">::get_secret_key — CommonFunctions</a>
|
|
117
|
+
|
|
118
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-get_std_out_and_err">::get_std_out_and_err — NeptuneHelper</a>
|
|
119
|
+
|
|
120
|
+
<li class="method"><a href="ExodusTaskInfo.html#method-c-new">::new — ExodusTaskInfo</a>
|
|
121
|
+
|
|
122
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-c-new">::new — NeptuneManagerClient</a>
|
|
123
|
+
|
|
124
|
+
<li class="method"><a href="TaskInfo.html#method-c-new">::new — TaskInfo</a>
|
|
125
|
+
|
|
126
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_babel">::preprocess_babel — NeptuneHelper</a>
|
|
127
|
+
|
|
128
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_compile">::preprocess_compile — NeptuneHelper</a>
|
|
129
|
+
|
|
130
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_erlang">::preprocess_erlang — NeptuneHelper</a>
|
|
131
|
+
|
|
132
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_mpi">::preprocess_mpi — NeptuneHelper</a>
|
|
133
|
+
|
|
134
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_ssa">::preprocess_ssa — NeptuneHelper</a>
|
|
135
|
+
|
|
136
|
+
<li class="method"><a href="ExodusHelper.html#method-c-propogate_credentials_from_environment">::propogate_credentials_from_environment — ExodusHelper</a>
|
|
137
|
+
|
|
138
|
+
<li class="method"><a href="BabelHelper.html#method-c-put_code">::put_code — BabelHelper</a>
|
|
139
|
+
|
|
140
|
+
<li class="method"><a href="BabelHelper.html#method-c-put_file">::put_file — BabelHelper</a>
|
|
141
|
+
|
|
142
|
+
<li class="method"><a href="BabelHelper.html#method-c-put_inputs">::put_inputs — BabelHelper</a>
|
|
143
|
+
|
|
144
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-require_file_to_exist">::require_file_to_exist — NeptuneHelper</a>
|
|
145
|
+
|
|
146
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-require_file_to_not_exist">::require_file_to_not_exist — NeptuneHelper</a>
|
|
147
|
+
|
|
148
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-require_param">::require_param — NeptuneHelper</a>
|
|
149
|
+
|
|
150
|
+
<li class="method"><a href="BabelHelper.html#method-c-run_job">::run_job — BabelHelper</a>
|
|
151
|
+
|
|
152
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-run_job">::run_job — NeptuneHelper</a>
|
|
153
|
+
|
|
154
|
+
<li class="method"><a href="ExodusHelper.html#method-c-run_job">::run_job — ExodusHelper</a>
|
|
155
|
+
|
|
156
|
+
<li class="method"><a href="CommonFunctions.html#method-c-scp_file">::scp_file — CommonFunctions</a>
|
|
157
|
+
|
|
158
|
+
<li class="method"><a href="CommonFunctions.html#method-c-scp_to_shadow">::scp_to_shadow — CommonFunctions</a>
|
|
159
|
+
|
|
160
|
+
<li class="method"><a href="CommonFunctions.html#method-c-shell">::shell — CommonFunctions</a>
|
|
161
|
+
|
|
162
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-upload_app_for_cicero">::upload_app_for_cicero — NeptuneHelper</a>
|
|
163
|
+
|
|
164
|
+
<li class="method"><a href="ExodusHelper.html#method-c-validate_clouds_to_use">::validate_clouds_to_use — ExodusHelper</a>
|
|
165
|
+
|
|
166
|
+
<li class="method"><a href="ExodusHelper.html#method-c-validate_files_argv_executable">::validate_files_argv_executable — ExodusHelper</a>
|
|
167
|
+
|
|
168
|
+
<li class="method"><a href="BabelHelper.html#method-c-validate_inputs">::validate_inputs — BabelHelper</a>
|
|
169
|
+
|
|
170
|
+
<li class="method"><a href="ExodusHelper.html#method-c-validate_optimize_for_param">::validate_optimize_for_param — ExodusHelper</a>
|
|
171
|
+
|
|
172
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-validate_storage_params">::validate_storage_params — NeptuneHelper</a>
|
|
173
|
+
|
|
174
|
+
<li class="method"><a href="BabelHelper.html#method-c-wait_and_get_output">::wait_and_get_output — BabelHelper</a>
|
|
175
|
+
|
|
176
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-wait_for_compilation_to_finish">::wait_for_compilation_to_finish — NeptuneHelper</a>
|
|
177
|
+
|
|
178
|
+
<li class="method"><a href="Object.html#method-i-babel">#babel — Object</a>
|
|
179
|
+
|
|
180
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-compile_code">#compile_code — NeptuneManagerClient</a>
|
|
181
|
+
|
|
182
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-does_file_exist-3F">#does_file_exist? — NeptuneManagerClient</a>
|
|
183
|
+
|
|
184
|
+
<li class="method"><a href="Object.html#method-i-exodus">#exodus — Object</a>
|
|
185
|
+
|
|
186
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_acl">#get_acl — NeptuneManagerClient</a>
|
|
187
|
+
|
|
188
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_output">#get_output — NeptuneManagerClient</a>
|
|
189
|
+
|
|
190
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_profiling_info">#get_profiling_info — NeptuneManagerClient</a>
|
|
191
|
+
|
|
192
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_supported_babel_engines">#get_supported_babel_engines — NeptuneManagerClient</a>
|
|
193
|
+
|
|
194
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-make_call">#make_call — NeptuneManagerClient</a>
|
|
195
|
+
|
|
196
|
+
<li class="method"><a href="ExodusTaskInfo.html#method-i-method_missing">#method_missing — ExodusTaskInfo</a>
|
|
197
|
+
|
|
198
|
+
<li class="method"><a href="Object.html#method-i-neptune">#neptune — Object</a>
|
|
199
|
+
|
|
200
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-put_input">#put_input — NeptuneManagerClient</a>
|
|
201
|
+
|
|
202
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-set_acl">#set_acl — NeptuneManagerClient</a>
|
|
203
|
+
|
|
204
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-start_neptune_job">#start_neptune_job — NeptuneManagerClient</a>
|
|
205
|
+
|
|
206
|
+
<li class="method"><a href="TaskInfo.html#method-i-stderr">#stderr — TaskInfo</a>
|
|
207
|
+
|
|
208
|
+
<li class="method"><a href="TaskInfo.html#method-i-stdout">#stdout — TaskInfo</a>
|
|
209
|
+
|
|
210
|
+
<li class="method"><a href="TaskInfo.html#method-i-success-3F">#success? — TaskInfo</a>
|
|
211
|
+
|
|
212
|
+
<li class="method"><a href="TaskInfo.html#method-i-to_json">#to_json — TaskInfo</a>
|
|
213
|
+
|
|
214
|
+
<li class="method"><a href="ExodusTaskInfo.html#method-i-to_s">#to_s — ExodusTaskInfo</a>
|
|
215
|
+
|
|
216
|
+
<li class="method"><a href="TaskInfo.html#method-i-to_s">#to_s — TaskInfo</a>
|
|
217
|
+
|
|
218
|
+
</ul>
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
<footer id="validator-badges">
|
|
222
|
+
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
|
223
|
+
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
|
|
224
|
+
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
|
225
|
+
</footer>
|
|
226
|
+
|
data/lib/babel.rb
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/ruby
|
|
2
2
|
# Programmer: Chris Bunch (cgb@cs.ucsb.edu)
|
|
3
3
|
|
|
4
|
-
require 'app_controller_client'
|
|
5
4
|
require 'common_functions'
|
|
6
5
|
require 'custom_exceptions'
|
|
7
6
|
require 'neptune'
|
|
7
|
+
require 'neptune_manager_client'
|
|
8
|
+
require 'task_info'
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
# The promise gem gives us futures / promises out-of-the-box, which we need
|
|
@@ -40,39 +41,61 @@ MAX_SLEEP_TIME = 60 # seconds
|
|
|
40
41
|
# write multiple Neptune jobs to actually run code (e.g., putting input in the
|
|
41
42
|
# datastore, run the job, get the output back), Babel automatically handles
|
|
42
43
|
# this.
|
|
43
|
-
def babel(
|
|
44
|
+
def babel(jobs)
|
|
44
45
|
# Since this whole function should run asynchronously, we run it as a future.
|
|
45
46
|
# It automatically starts running in a new thread, and attempting to get the
|
|
46
47
|
# value of what this returns causes it to block until the job completes.
|
|
47
|
-
future {
|
|
48
|
-
if
|
|
49
|
-
|
|
48
|
+
#future {
|
|
49
|
+
if jobs.class == Hash
|
|
50
|
+
was_one_job = true
|
|
51
|
+
jobs = [jobs]
|
|
50
52
|
else
|
|
51
|
-
|
|
53
|
+
was_one_job = false
|
|
52
54
|
end
|
|
53
55
|
|
|
54
|
-
|
|
56
|
+
job_data_list = []
|
|
57
|
+
jobs.each { |params|
|
|
58
|
+
job_data = BabelHelper.convert_from_neptune_params(params)
|
|
59
|
+
job_data['@metadata_info'] = {'time_to_store_inputs' => 0.0}
|
|
55
60
|
|
|
56
|
-
|
|
61
|
+
# Add in S3 storage parameters
|
|
62
|
+
NeptuneHelper.validate_storage_params(job_data)
|
|
57
63
|
|
|
58
|
-
|
|
59
|
-
|
|
64
|
+
# :code is the only required parameter
|
|
65
|
+
# everything else can use default vals
|
|
66
|
+
NeptuneHelper.require_param("@code", job_data)
|
|
67
|
+
BabelHelper.check_output_files(job_data)
|
|
60
68
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
69
|
+
if job_data["@is_remote"]
|
|
70
|
+
#BabelHelper.validate_inputs(job_data)
|
|
71
|
+
else
|
|
72
|
+
BabelHelper.put_code(job_data)
|
|
73
|
+
BabelHelper.put_inputs(job_data)
|
|
74
|
+
end
|
|
65
75
|
|
|
66
|
-
|
|
67
|
-
|
|
76
|
+
job_data_list << job_data
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
BabelHelper.run_job(job_data_list)
|
|
80
|
+
|
|
81
|
+
# Return an object to the user that has all the information about their
|
|
82
|
+
# task, including its standard out, err, debugging info, and profiling
|
|
83
|
+
# info. Since the job may not be done when the user asks for this info,
|
|
84
|
+
# its the responsibility of TaskInfo objects to block until that info
|
|
85
|
+
# is ready. We don't explicitly return the TaskInfo object, because it's
|
|
86
|
+
# in a Future block - it will automatically return whatever the last
|
|
87
|
+
# statement returns.
|
|
88
|
+
tasks = []
|
|
89
|
+
job_data_list.each { |job_data|
|
|
90
|
+
tasks << TaskInfo.new(job_data)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if was_one_job
|
|
94
|
+
tasks[0]
|
|
68
95
|
else
|
|
69
|
-
|
|
70
|
-
BabelHelper.put_inputs(job_data)
|
|
96
|
+
tasks
|
|
71
97
|
end
|
|
72
|
-
|
|
73
|
-
BabelHelper.run_job(job_data)
|
|
74
|
-
BabelHelper.wait_and_get_output(job_data)
|
|
75
|
-
}
|
|
98
|
+
#}
|
|
76
99
|
end
|
|
77
100
|
|
|
78
101
|
|
|
@@ -84,7 +107,7 @@ module BabelHelper
|
|
|
84
107
|
# one for them, based on either the location of their code (for remotely
|
|
85
108
|
# specified code), or a babel parameter (for locally specified code).
|
|
86
109
|
def self.generate_output_location(job_data)
|
|
87
|
-
if job_data["@
|
|
110
|
+
if job_data["@is_remote"]
|
|
88
111
|
# We already know the bucket name - the same one that the user
|
|
89
112
|
# has told us their code is located in.
|
|
90
113
|
prefix = job_data["@code"].scan(/\/(.*?)\//)[0].to_s
|
|
@@ -99,7 +122,9 @@ module BabelHelper
|
|
|
99
122
|
# Provides a common way for callers to get the name of the bucket that
|
|
100
123
|
# should be used for Neptune jobs where the code is stored locally.
|
|
101
124
|
def self.get_bucket_for_local_data(job_data)
|
|
102
|
-
bucket_name = job_data["@bucket_name"] || ENV['BABEL_BUCKET_NAME']
|
|
125
|
+
bucket_name = job_data["@bucket_name"] || ENV['BABEL_BUCKET_NAME'] ||
|
|
126
|
+
job_data["@S3_bucket_name"] || job_data["@Walrus_bucket_name"] ||
|
|
127
|
+
job_data["@GStorage_bucket_name"] || job_data["@WAZ_Container_Name"]
|
|
103
128
|
|
|
104
129
|
if bucket_name.nil?
|
|
105
130
|
raise BadConfigurationException.new(NEEDS_BUCKET_INFO)
|
|
@@ -113,12 +138,26 @@ module BabelHelper
|
|
|
113
138
|
return bucket_name
|
|
114
139
|
end
|
|
115
140
|
|
|
141
|
+
|
|
142
|
+
# babel() callers do not have to specify a location where the standard output
|
|
143
|
+
# and error the task produces should be placed. If they don't, generate
|
|
144
|
+
# locations for them and make sure they don't exist beforehand.
|
|
145
|
+
def self.check_output_files(job_data)
|
|
146
|
+
["@output", "@error", "@metadata"].each { |item|
|
|
147
|
+
if job_data[item].nil? or job_data[item].empty?
|
|
148
|
+
job_data[item] = BabelHelper.generate_output_location(job_data)
|
|
149
|
+
else
|
|
150
|
+
BabelHelper.ensure_output_does_not_exist(job_data, job_data[item])
|
|
151
|
+
end
|
|
152
|
+
}
|
|
153
|
+
end
|
|
154
|
+
|
|
116
155
|
|
|
117
156
|
# For jobs where the code is stored remotely, this method ensures that
|
|
118
157
|
# the code and any possible inputs actually do exist, before attempting to
|
|
119
158
|
# use them for computation.
|
|
120
159
|
def self.validate_inputs(job_data)
|
|
121
|
-
controller = self.
|
|
160
|
+
controller = self.get_neptune_manager_client(job_data)
|
|
122
161
|
|
|
123
162
|
# First, make sure the code exists
|
|
124
163
|
NeptuneHelper.require_file_to_exist(job_data["@code"], job_data, controller)
|
|
@@ -139,20 +178,18 @@ module BabelHelper
|
|
|
139
178
|
# To avoid accidentally overwriting outputs from previous jobs, we first
|
|
140
179
|
# check to make sure an output file doesn't exist before starting a new job
|
|
141
180
|
# with the given name.
|
|
142
|
-
def self.ensure_output_does_not_exist(job_data)
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
# Kernel.puts job_data.inspect
|
|
146
|
-
NeptuneHelper.require_file_to_not_exist(file, job_data, controller)
|
|
181
|
+
def self.ensure_output_does_not_exist(job_data, remote_file)
|
|
182
|
+
controller = self.get_neptune_manager_client(job_data)
|
|
183
|
+
NeptuneHelper.require_file_to_not_exist(remote_file, job_data, controller)
|
|
147
184
|
end
|
|
148
185
|
|
|
149
186
|
|
|
150
|
-
# Returns an
|
|
151
|
-
def self.
|
|
187
|
+
# Returns an NeptuneManagerClient for the given job data.
|
|
188
|
+
def self.get_neptune_manager_client(job_data)
|
|
152
189
|
keyname = job_data["@keyname"] || "appscale"
|
|
153
190
|
shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
|
|
154
191
|
secret = CommonFunctions.get_secret_key(keyname)
|
|
155
|
-
return
|
|
192
|
+
return NeptuneManagerClient.new(shadow_ip, secret)
|
|
156
193
|
end
|
|
157
194
|
|
|
158
195
|
|
|
@@ -198,7 +235,15 @@ module BabelHelper
|
|
|
198
235
|
bucket_name = self.get_bucket_for_local_data(job_data)
|
|
199
236
|
input_data[:remote] = "/#{bucket_name}/babel#{local_path}"
|
|
200
237
|
|
|
238
|
+
start = Time.now
|
|
201
239
|
Kernel.neptune(input_data)
|
|
240
|
+
fin = Time.now
|
|
241
|
+
|
|
242
|
+
if job_data['@metadata_info'].nil?
|
|
243
|
+
job_data['@metadata_info'] = {'time_to_store_inputs' => 0.0}
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
job_data['@metadata_info']['time_to_store_inputs'] += fin - start
|
|
202
247
|
|
|
203
248
|
return input_data[:remote]
|
|
204
249
|
end
|
|
@@ -234,31 +279,52 @@ module BabelHelper
|
|
|
234
279
|
|
|
235
280
|
# Constructs a Neptune job to run the user's code as a Babel job (task queue)
|
|
236
281
|
# from the given parameters.
|
|
237
|
-
def self.run_job(
|
|
238
|
-
|
|
282
|
+
def self.run_job(job_data_list)
|
|
283
|
+
run_data_list = []
|
|
239
284
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
run_data[:type] = "babel"
|
|
243
|
-
end
|
|
285
|
+
job_data_list.each { |job_data|
|
|
286
|
+
run_data = self.convert_to_neptune_params(job_data)
|
|
244
287
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
288
|
+
# Default to babel as the job type, if the user doesn't specify one.
|
|
289
|
+
if run_data[:type].nil? or run_data[:type].empty?
|
|
290
|
+
run_data[:type] = "babel"
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to
|
|
294
|
+
# exec tasks over it, instead of locally.
|
|
295
|
+
if job_data["@run_local"].nil?
|
|
296
|
+
run_data[:run_local] = true
|
|
297
|
+
run_data[:engine] = "executor-sqs"
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
run_data[:failed_attempts] = 0
|
|
301
|
+
run_data_list << run_data
|
|
302
|
+
}
|
|
251
303
|
|
|
252
|
-
|
|
304
|
+
loop {
|
|
305
|
+
if run_data_list.length == 1
|
|
306
|
+
run_job = Kernel.neptune(run_data_list[0])
|
|
307
|
+
else
|
|
308
|
+
run_job = Kernel.neptune(run_data_list)
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
if run_job[:result] == :success
|
|
312
|
+
return run_job
|
|
313
|
+
else
|
|
314
|
+
run_data_list[0][:failed_attempts] += 1
|
|
315
|
+
Kernel.sleep(SLEEP_TIME) # TODO(cgb): this should exponentially backoff
|
|
316
|
+
end
|
|
317
|
+
}
|
|
253
318
|
end
|
|
254
319
|
|
|
255
320
|
|
|
256
321
|
# Constructs a Neptune job to get the output of a Babel job. If the job is not
|
|
257
322
|
# yet finished, this function waits until it does, and then returns the output
|
|
258
323
|
# of the job.
|
|
259
|
-
def self.wait_and_get_output(job_data)
|
|
324
|
+
def self.wait_and_get_output(job_data, output_location)
|
|
260
325
|
output_data = self.convert_to_neptune_params(job_data)
|
|
261
326
|
output_data[:type] = "output"
|
|
327
|
+
output_data[:output] = output_location
|
|
262
328
|
|
|
263
329
|
output = ""
|
|
264
330
|
time_to_sleep = SLEEP_TIME
|
|
@@ -267,9 +333,9 @@ module BabelHelper
|
|
|
267
333
|
if output == DOES_NOT_EXIST
|
|
268
334
|
# Exponentially back off, up to a limit of MAX_SLEEP_TIME
|
|
269
335
|
Kernel.sleep(time_to_sleep)
|
|
270
|
-
if time_to_sleep < MAX_SLEEP_TIME
|
|
271
|
-
|
|
272
|
-
end
|
|
336
|
+
#if time_to_sleep < MAX_SLEEP_TIME
|
|
337
|
+
# time_to_sleep *= 2
|
|
338
|
+
#end
|
|
273
339
|
else
|
|
274
340
|
break
|
|
275
341
|
end
|