neptune 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README +4 -0
- data/doc/BabelHelper.html +393 -376
- data/doc/BadConfigurationException.html +121 -127
- data/doc/CommonFunctions.html +237 -265
- data/doc/ExodusHelper.html +820 -0
- data/doc/ExodusTaskInfo.html +263 -0
- data/doc/FileNotFoundException.html +121 -127
- data/doc/NeptuneHelper.html +527 -592
- data/doc/NeptuneManagerClient.html +696 -0
- data/doc/NeptuneManagerException.html +139 -0
- data/doc/Object.html +334 -236
- data/doc/TaskInfo.html +428 -0
- data/doc/created.rid +8 -5
- data/doc/images/add.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/index.html +74 -142
- data/doc/js/darkfish.js +99 -62
- data/doc/js/jquery.js +15 -29
- data/doc/js/navigation.js +142 -0
- data/doc/js/search.js +94 -0
- data/doc/js/search_index.js +1 -0
- data/doc/js/searcher.js +228 -0
- data/doc/table_of_contents.html +226 -0
- data/lib/babel.rb +116 -50
- data/lib/custom_exceptions.rb +2 -2
- data/lib/exodus.rb +311 -0
- data/lib/exodus_task_info.rb +36 -0
- data/lib/neptune.rb +52 -18
- data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
- data/lib/task_info.rb +155 -0
- data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
- data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
- data/test/test_exodus.rb +687 -0
- data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
- data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
- data/test/test_task_info.rb +32 -0
- data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
- metadata +30 -34
- data/doc/AppControllerClient.html +0 -702
- data/doc/AppControllerException.html +0 -145
- data/doc/bin/neptune.html +0 -56
- data/doc/js/quicksearch.js +0 -114
- data/doc/js/thickbox-compressed.js +0 -10
- data/doc/lib/app_controller_client_rb.html +0 -60
- data/doc/lib/babel_rb.html +0 -68
- data/doc/lib/common_functions_rb.html +0 -70
- data/doc/lib/custom_exceptions_rb.html +0 -54
- data/doc/lib/neptune_rb.html +0 -60
- data/test/integration/tc_c.rb +0 -57
- data/test/integration/tc_dfsp.rb +0 -37
- data/test/integration/tc_dwssa.rb +0 -38
- data/test/integration/tc_erlang.rb +0 -183
- data/test/integration/tc_mapreduce.rb +0 -282
- data/test/integration/tc_mpi.rb +0 -160
- data/test/integration/tc_storage.rb +0 -209
- data/test/integration/tc_upc.rb +0 -75
- data/test/integration/tc_x10.rb +0 -94
- data/test/integration/test_helper.rb +0 -135
- data/test/integration/ts_neptune.rb +0 -40
@@ -0,0 +1,226 @@
|
|
1
|
+
<!DOCTYPE html>
|
2
|
+
|
3
|
+
<html>
|
4
|
+
<head>
|
5
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
6
|
+
|
7
|
+
<title>Table of Contents - RDoc Documentation</title>
|
8
|
+
|
9
|
+
<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
|
10
|
+
|
11
|
+
<script type="text/javascript">
|
12
|
+
var rdoc_rel_prefix = "./";
|
13
|
+
</script>
|
14
|
+
|
15
|
+
<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
|
16
|
+
<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
|
17
|
+
<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
|
18
|
+
<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
|
19
|
+
<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
|
20
|
+
<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
|
21
|
+
|
22
|
+
|
23
|
+
<body class="indexpage">
|
24
|
+
<h1>Table of Contents - RDoc Documentation</h1>
|
25
|
+
|
26
|
+
|
27
|
+
<h2 id="classes">Classes/Modules</h2>
|
28
|
+
<ul>
|
29
|
+
<li class="module">
|
30
|
+
<a href="BabelHelper.html">BabelHelper</a>
|
31
|
+
</li>
|
32
|
+
<li class="class">
|
33
|
+
<a href="BadConfigurationException.html">BadConfigurationException</a>
|
34
|
+
</li>
|
35
|
+
<li class="module">
|
36
|
+
<a href="CommonFunctions.html">CommonFunctions</a>
|
37
|
+
</li>
|
38
|
+
<li class="module">
|
39
|
+
<a href="ExodusHelper.html">ExodusHelper</a>
|
40
|
+
</li>
|
41
|
+
<li class="class">
|
42
|
+
<a href="ExodusTaskInfo.html">ExodusTaskInfo</a>
|
43
|
+
</li>
|
44
|
+
<li class="class">
|
45
|
+
<a href="FileNotFoundException.html">FileNotFoundException</a>
|
46
|
+
</li>
|
47
|
+
<li class="module">
|
48
|
+
<a href="NeptuneHelper.html">NeptuneHelper</a>
|
49
|
+
</li>
|
50
|
+
<li class="class">
|
51
|
+
<a href="NeptuneManagerClient.html">NeptuneManagerClient</a>
|
52
|
+
</li>
|
53
|
+
<li class="class">
|
54
|
+
<a href="NeptuneManagerException.html">NeptuneManagerException</a>
|
55
|
+
</li>
|
56
|
+
<li class="class">
|
57
|
+
<a href="Object.html">Object</a>
|
58
|
+
</li>
|
59
|
+
<li class="class">
|
60
|
+
<a href="TaskInfo.html">TaskInfo</a>
|
61
|
+
</li>
|
62
|
+
|
63
|
+
</ul>
|
64
|
+
|
65
|
+
<h2 id="methods">Methods</h2>
|
66
|
+
<ul>
|
67
|
+
|
68
|
+
<li class="method"><a href="ExodusHelper.html#method-c-average">::average — ExodusHelper</a>
|
69
|
+
|
70
|
+
<li class="method"><a href="BabelHelper.html#method-c-check_output_files">::check_output_files — BabelHelper</a>
|
71
|
+
|
72
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-compile_code">::compile_code — NeptuneHelper</a>
|
73
|
+
|
74
|
+
<li class="method"><a href="ExodusHelper.html#method-c-convert_clouds_to_use_to_array">::convert_clouds_to_use_to_array — ExodusHelper</a>
|
75
|
+
|
76
|
+
<li class="method"><a href="BabelHelper.html#method-c-convert_from_neptune_params">::convert_from_neptune_params — BabelHelper</a>
|
77
|
+
|
78
|
+
<li class="method"><a href="BabelHelper.html#method-c-convert_to_neptune_params">::convert_to_neptune_params — BabelHelper</a>
|
79
|
+
|
80
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-do_preprocessing">::do_preprocessing — NeptuneHelper</a>
|
81
|
+
|
82
|
+
<li class="method"><a href="ExodusHelper.html#method-c-ensure_all_jobs_are_hashes">::ensure_all_jobs_are_hashes — ExodusHelper</a>
|
83
|
+
|
84
|
+
<li class="method"><a href="ExodusHelper.html#method-c-ensure_all_params_are_present">::ensure_all_params_are_present — ExodusHelper</a>
|
85
|
+
|
86
|
+
<li class="method"><a href="ExodusHelper.html#method-c-ensure_credentials_are_in_correct_format">::ensure_credentials_are_in_correct_format — ExodusHelper</a>
|
87
|
+
|
88
|
+
<li class="method"><a href="BabelHelper.html#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist — BabelHelper</a>
|
89
|
+
|
90
|
+
<li class="method"><a href="ExodusHelper.html#method-c-find_optimal_cloud_for_task">::find_optimal_cloud_for_task — ExodusHelper</a>
|
91
|
+
|
92
|
+
<li class="method"><a href="ExodusHelper.html#method-c-generate_babel_tasks">::generate_babel_tasks — ExodusHelper</a>
|
93
|
+
|
94
|
+
<li class="method"><a href="BabelHelper.html#method-c-generate_output_location">::generate_output_location — BabelHelper</a>
|
95
|
+
|
96
|
+
<li class="method"><a href="BabelHelper.html#method-c-get_bucket_for_local_data">::get_bucket_for_local_data — BabelHelper</a>
|
97
|
+
|
98
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_clouds_to_run_task_on">::get_clouds_to_run_task_on — ExodusHelper</a>
|
99
|
+
|
100
|
+
<li class="method"><a href="CommonFunctions.html#method-c-get_from_yaml">::get_from_yaml — CommonFunctions</a>
|
101
|
+
|
102
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-get_input">::get_input — NeptuneHelper</a>
|
103
|
+
|
104
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-get_job_data">::get_job_data — NeptuneHelper</a>
|
105
|
+
|
106
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_key_from_job_data">::get_key_from_job_data — ExodusHelper</a>
|
107
|
+
|
108
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_minimum_val_in_data">::get_minimum_val_in_data — ExodusHelper</a>
|
109
|
+
|
110
|
+
<li class="method"><a href="BabelHelper.html#method-c-get_neptune_manager_client">::get_neptune_manager_client — BabelHelper</a>
|
111
|
+
|
112
|
+
<li class="method"><a href="ExodusHelper.html#method-c-get_profiling_info">::get_profiling_info — ExodusHelper</a>
|
113
|
+
|
114
|
+
<li class="method"><a href="CommonFunctions.html#method-c-get_random_alphanumeric">::get_random_alphanumeric — CommonFunctions</a>
|
115
|
+
|
116
|
+
<li class="method"><a href="CommonFunctions.html#method-c-get_secret_key">::get_secret_key — CommonFunctions</a>
|
117
|
+
|
118
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-get_std_out_and_err">::get_std_out_and_err — NeptuneHelper</a>
|
119
|
+
|
120
|
+
<li class="method"><a href="ExodusTaskInfo.html#method-c-new">::new — ExodusTaskInfo</a>
|
121
|
+
|
122
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-c-new">::new — NeptuneManagerClient</a>
|
123
|
+
|
124
|
+
<li class="method"><a href="TaskInfo.html#method-c-new">::new — TaskInfo</a>
|
125
|
+
|
126
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_babel">::preprocess_babel — NeptuneHelper</a>
|
127
|
+
|
128
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_compile">::preprocess_compile — NeptuneHelper</a>
|
129
|
+
|
130
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_erlang">::preprocess_erlang — NeptuneHelper</a>
|
131
|
+
|
132
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_mpi">::preprocess_mpi — NeptuneHelper</a>
|
133
|
+
|
134
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-preprocess_ssa">::preprocess_ssa — NeptuneHelper</a>
|
135
|
+
|
136
|
+
<li class="method"><a href="ExodusHelper.html#method-c-propogate_credentials_from_environment">::propogate_credentials_from_environment — ExodusHelper</a>
|
137
|
+
|
138
|
+
<li class="method"><a href="BabelHelper.html#method-c-put_code">::put_code — BabelHelper</a>
|
139
|
+
|
140
|
+
<li class="method"><a href="BabelHelper.html#method-c-put_file">::put_file — BabelHelper</a>
|
141
|
+
|
142
|
+
<li class="method"><a href="BabelHelper.html#method-c-put_inputs">::put_inputs — BabelHelper</a>
|
143
|
+
|
144
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-require_file_to_exist">::require_file_to_exist — NeptuneHelper</a>
|
145
|
+
|
146
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-require_file_to_not_exist">::require_file_to_not_exist — NeptuneHelper</a>
|
147
|
+
|
148
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-require_param">::require_param — NeptuneHelper</a>
|
149
|
+
|
150
|
+
<li class="method"><a href="BabelHelper.html#method-c-run_job">::run_job — BabelHelper</a>
|
151
|
+
|
152
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-run_job">::run_job — NeptuneHelper</a>
|
153
|
+
|
154
|
+
<li class="method"><a href="ExodusHelper.html#method-c-run_job">::run_job — ExodusHelper</a>
|
155
|
+
|
156
|
+
<li class="method"><a href="CommonFunctions.html#method-c-scp_file">::scp_file — CommonFunctions</a>
|
157
|
+
|
158
|
+
<li class="method"><a href="CommonFunctions.html#method-c-scp_to_shadow">::scp_to_shadow — CommonFunctions</a>
|
159
|
+
|
160
|
+
<li class="method"><a href="CommonFunctions.html#method-c-shell">::shell — CommonFunctions</a>
|
161
|
+
|
162
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-upload_app_for_cicero">::upload_app_for_cicero — NeptuneHelper</a>
|
163
|
+
|
164
|
+
<li class="method"><a href="ExodusHelper.html#method-c-validate_clouds_to_use">::validate_clouds_to_use — ExodusHelper</a>
|
165
|
+
|
166
|
+
<li class="method"><a href="ExodusHelper.html#method-c-validate_files_argv_executable">::validate_files_argv_executable — ExodusHelper</a>
|
167
|
+
|
168
|
+
<li class="method"><a href="BabelHelper.html#method-c-validate_inputs">::validate_inputs — BabelHelper</a>
|
169
|
+
|
170
|
+
<li class="method"><a href="ExodusHelper.html#method-c-validate_optimize_for_param">::validate_optimize_for_param — ExodusHelper</a>
|
171
|
+
|
172
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-validate_storage_params">::validate_storage_params — NeptuneHelper</a>
|
173
|
+
|
174
|
+
<li class="method"><a href="BabelHelper.html#method-c-wait_and_get_output">::wait_and_get_output — BabelHelper</a>
|
175
|
+
|
176
|
+
<li class="method"><a href="NeptuneHelper.html#method-c-wait_for_compilation_to_finish">::wait_for_compilation_to_finish — NeptuneHelper</a>
|
177
|
+
|
178
|
+
<li class="method"><a href="Object.html#method-i-babel">#babel — Object</a>
|
179
|
+
|
180
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-compile_code">#compile_code — NeptuneManagerClient</a>
|
181
|
+
|
182
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-does_file_exist-3F">#does_file_exist? — NeptuneManagerClient</a>
|
183
|
+
|
184
|
+
<li class="method"><a href="Object.html#method-i-exodus">#exodus — Object</a>
|
185
|
+
|
186
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_acl">#get_acl — NeptuneManagerClient</a>
|
187
|
+
|
188
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_output">#get_output — NeptuneManagerClient</a>
|
189
|
+
|
190
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_profiling_info">#get_profiling_info — NeptuneManagerClient</a>
|
191
|
+
|
192
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-get_supported_babel_engines">#get_supported_babel_engines — NeptuneManagerClient</a>
|
193
|
+
|
194
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-make_call">#make_call — NeptuneManagerClient</a>
|
195
|
+
|
196
|
+
<li class="method"><a href="ExodusTaskInfo.html#method-i-method_missing">#method_missing — ExodusTaskInfo</a>
|
197
|
+
|
198
|
+
<li class="method"><a href="Object.html#method-i-neptune">#neptune — Object</a>
|
199
|
+
|
200
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-put_input">#put_input — NeptuneManagerClient</a>
|
201
|
+
|
202
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-set_acl">#set_acl — NeptuneManagerClient</a>
|
203
|
+
|
204
|
+
<li class="method"><a href="NeptuneManagerClient.html#method-i-start_neptune_job">#start_neptune_job — NeptuneManagerClient</a>
|
205
|
+
|
206
|
+
<li class="method"><a href="TaskInfo.html#method-i-stderr">#stderr — TaskInfo</a>
|
207
|
+
|
208
|
+
<li class="method"><a href="TaskInfo.html#method-i-stdout">#stdout — TaskInfo</a>
|
209
|
+
|
210
|
+
<li class="method"><a href="TaskInfo.html#method-i-success-3F">#success? — TaskInfo</a>
|
211
|
+
|
212
|
+
<li class="method"><a href="TaskInfo.html#method-i-to_json">#to_json — TaskInfo</a>
|
213
|
+
|
214
|
+
<li class="method"><a href="ExodusTaskInfo.html#method-i-to_s">#to_s — ExodusTaskInfo</a>
|
215
|
+
|
216
|
+
<li class="method"><a href="TaskInfo.html#method-i-to_s">#to_s — TaskInfo</a>
|
217
|
+
|
218
|
+
</ul>
|
219
|
+
|
220
|
+
|
221
|
+
<footer id="validator-badges">
|
222
|
+
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
223
|
+
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
|
224
|
+
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
225
|
+
</footer>
|
226
|
+
|
data/lib/babel.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
#!/usr/bin/ruby
|
2
2
|
# Programmer: Chris Bunch (cgb@cs.ucsb.edu)
|
3
3
|
|
4
|
-
require 'app_controller_client'
|
5
4
|
require 'common_functions'
|
6
5
|
require 'custom_exceptions'
|
7
6
|
require 'neptune'
|
7
|
+
require 'neptune_manager_client'
|
8
|
+
require 'task_info'
|
8
9
|
|
9
10
|
|
10
11
|
# The promise gem gives us futures / promises out-of-the-box, which we need
|
@@ -40,39 +41,61 @@ MAX_SLEEP_TIME = 60 # seconds
|
|
40
41
|
# write multiple Neptune jobs to actually run code (e.g., putting input in the
|
41
42
|
# datastore, run the job, get the output back), Babel automatically handles
|
42
43
|
# this.
|
43
|
-
def babel(
|
44
|
+
def babel(jobs)
|
44
45
|
# Since this whole function should run asynchronously, we run it as a future.
|
45
46
|
# It automatically starts running in a new thread, and attempting to get the
|
46
47
|
# value of what this returns causes it to block until the job completes.
|
47
|
-
future {
|
48
|
-
if
|
49
|
-
|
48
|
+
#future {
|
49
|
+
if jobs.class == Hash
|
50
|
+
was_one_job = true
|
51
|
+
jobs = [jobs]
|
50
52
|
else
|
51
|
-
|
53
|
+
was_one_job = false
|
52
54
|
end
|
53
55
|
|
54
|
-
|
56
|
+
job_data_list = []
|
57
|
+
jobs.each { |params|
|
58
|
+
job_data = BabelHelper.convert_from_neptune_params(params)
|
59
|
+
job_data['@metadata_info'] = {'time_to_store_inputs' => 0.0}
|
55
60
|
|
56
|
-
|
61
|
+
# Add in S3 storage parameters
|
62
|
+
NeptuneHelper.validate_storage_params(job_data)
|
57
63
|
|
58
|
-
|
59
|
-
|
64
|
+
# :code is the only required parameter
|
65
|
+
# everything else can use default vals
|
66
|
+
NeptuneHelper.require_param("@code", job_data)
|
67
|
+
BabelHelper.check_output_files(job_data)
|
60
68
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
69
|
+
if job_data["@is_remote"]
|
70
|
+
#BabelHelper.validate_inputs(job_data)
|
71
|
+
else
|
72
|
+
BabelHelper.put_code(job_data)
|
73
|
+
BabelHelper.put_inputs(job_data)
|
74
|
+
end
|
65
75
|
|
66
|
-
|
67
|
-
|
76
|
+
job_data_list << job_data
|
77
|
+
}
|
78
|
+
|
79
|
+
BabelHelper.run_job(job_data_list)
|
80
|
+
|
81
|
+
# Return an object to the user that has all the information about their
|
82
|
+
# task, including its standard out, err, debugging info, and profiling
|
83
|
+
# info. Since the job may not be done when the user asks for this info,
|
84
|
+
# its the responsibility of TaskInfo objects to block until that info
|
85
|
+
# is ready. We don't explicitly return the TaskInfo object, because it's
|
86
|
+
# in a Future block - it will automatically return whatever the last
|
87
|
+
# statement returns.
|
88
|
+
tasks = []
|
89
|
+
job_data_list.each { |job_data|
|
90
|
+
tasks << TaskInfo.new(job_data)
|
91
|
+
}
|
92
|
+
|
93
|
+
if was_one_job
|
94
|
+
tasks[0]
|
68
95
|
else
|
69
|
-
|
70
|
-
BabelHelper.put_inputs(job_data)
|
96
|
+
tasks
|
71
97
|
end
|
72
|
-
|
73
|
-
BabelHelper.run_job(job_data)
|
74
|
-
BabelHelper.wait_and_get_output(job_data)
|
75
|
-
}
|
98
|
+
#}
|
76
99
|
end
|
77
100
|
|
78
101
|
|
@@ -84,7 +107,7 @@ module BabelHelper
|
|
84
107
|
# one for them, based on either the location of their code (for remotely
|
85
108
|
# specified code), or a babel parameter (for locally specified code).
|
86
109
|
def self.generate_output_location(job_data)
|
87
|
-
if job_data["@
|
110
|
+
if job_data["@is_remote"]
|
88
111
|
# We already know the bucket name - the same one that the user
|
89
112
|
# has told us their code is located in.
|
90
113
|
prefix = job_data["@code"].scan(/\/(.*?)\//)[0].to_s
|
@@ -99,7 +122,9 @@ module BabelHelper
|
|
99
122
|
# Provides a common way for callers to get the name of the bucket that
|
100
123
|
# should be used for Neptune jobs where the code is stored locally.
|
101
124
|
def self.get_bucket_for_local_data(job_data)
|
102
|
-
bucket_name = job_data["@bucket_name"] || ENV['BABEL_BUCKET_NAME']
|
125
|
+
bucket_name = job_data["@bucket_name"] || ENV['BABEL_BUCKET_NAME'] ||
|
126
|
+
job_data["@S3_bucket_name"] || job_data["@Walrus_bucket_name"] ||
|
127
|
+
job_data["@GStorage_bucket_name"] || job_data["@WAZ_Container_Name"]
|
103
128
|
|
104
129
|
if bucket_name.nil?
|
105
130
|
raise BadConfigurationException.new(NEEDS_BUCKET_INFO)
|
@@ -113,12 +138,26 @@ module BabelHelper
|
|
113
138
|
return bucket_name
|
114
139
|
end
|
115
140
|
|
141
|
+
|
142
|
+
# babel() callers do not have to specify a location where the standard output
|
143
|
+
# and error the task produces should be placed. If they don't, generate
|
144
|
+
# locations for them and make sure they don't exist beforehand.
|
145
|
+
def self.check_output_files(job_data)
|
146
|
+
["@output", "@error", "@metadata"].each { |item|
|
147
|
+
if job_data[item].nil? or job_data[item].empty?
|
148
|
+
job_data[item] = BabelHelper.generate_output_location(job_data)
|
149
|
+
else
|
150
|
+
BabelHelper.ensure_output_does_not_exist(job_data, job_data[item])
|
151
|
+
end
|
152
|
+
}
|
153
|
+
end
|
154
|
+
|
116
155
|
|
117
156
|
# For jobs where the code is stored remotely, this method ensures that
|
118
157
|
# the code and any possible inputs actually do exist, before attempting to
|
119
158
|
# use them for computation.
|
120
159
|
def self.validate_inputs(job_data)
|
121
|
-
controller = self.
|
160
|
+
controller = self.get_neptune_manager_client(job_data)
|
122
161
|
|
123
162
|
# First, make sure the code exists
|
124
163
|
NeptuneHelper.require_file_to_exist(job_data["@code"], job_data, controller)
|
@@ -139,20 +178,18 @@ module BabelHelper
|
|
139
178
|
# To avoid accidentally overwriting outputs from previous jobs, we first
|
140
179
|
# check to make sure an output file doesn't exist before starting a new job
|
141
180
|
# with the given name.
|
142
|
-
def self.ensure_output_does_not_exist(job_data)
|
143
|
-
|
144
|
-
|
145
|
-
# Kernel.puts job_data.inspect
|
146
|
-
NeptuneHelper.require_file_to_not_exist(file, job_data, controller)
|
181
|
+
def self.ensure_output_does_not_exist(job_data, remote_file)
|
182
|
+
controller = self.get_neptune_manager_client(job_data)
|
183
|
+
NeptuneHelper.require_file_to_not_exist(remote_file, job_data, controller)
|
147
184
|
end
|
148
185
|
|
149
186
|
|
150
|
-
# Returns an
|
151
|
-
def self.
|
187
|
+
# Returns an NeptuneManagerClient for the given job data.
|
188
|
+
def self.get_neptune_manager_client(job_data)
|
152
189
|
keyname = job_data["@keyname"] || "appscale"
|
153
190
|
shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
|
154
191
|
secret = CommonFunctions.get_secret_key(keyname)
|
155
|
-
return
|
192
|
+
return NeptuneManagerClient.new(shadow_ip, secret)
|
156
193
|
end
|
157
194
|
|
158
195
|
|
@@ -198,7 +235,15 @@ module BabelHelper
|
|
198
235
|
bucket_name = self.get_bucket_for_local_data(job_data)
|
199
236
|
input_data[:remote] = "/#{bucket_name}/babel#{local_path}"
|
200
237
|
|
238
|
+
start = Time.now
|
201
239
|
Kernel.neptune(input_data)
|
240
|
+
fin = Time.now
|
241
|
+
|
242
|
+
if job_data['@metadata_info'].nil?
|
243
|
+
job_data['@metadata_info'] = {'time_to_store_inputs' => 0.0}
|
244
|
+
end
|
245
|
+
|
246
|
+
job_data['@metadata_info']['time_to_store_inputs'] += fin - start
|
202
247
|
|
203
248
|
return input_data[:remote]
|
204
249
|
end
|
@@ -234,31 +279,52 @@ module BabelHelper
|
|
234
279
|
|
235
280
|
# Constructs a Neptune job to run the user's code as a Babel job (task queue)
|
236
281
|
# from the given parameters.
|
237
|
-
def self.run_job(
|
238
|
-
|
282
|
+
def self.run_job(job_data_list)
|
283
|
+
run_data_list = []
|
239
284
|
|
240
|
-
|
241
|
-
|
242
|
-
run_data[:type] = "babel"
|
243
|
-
end
|
285
|
+
job_data_list.each { |job_data|
|
286
|
+
run_data = self.convert_to_neptune_params(job_data)
|
244
287
|
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
288
|
+
# Default to babel as the job type, if the user doesn't specify one.
|
289
|
+
if run_data[:type].nil? or run_data[:type].empty?
|
290
|
+
run_data[:type] = "babel"
|
291
|
+
end
|
292
|
+
|
293
|
+
# TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to
|
294
|
+
# exec tasks over it, instead of locally.
|
295
|
+
if job_data["@run_local"].nil?
|
296
|
+
run_data[:run_local] = true
|
297
|
+
run_data[:engine] = "executor-sqs"
|
298
|
+
end
|
299
|
+
|
300
|
+
run_data[:failed_attempts] = 0
|
301
|
+
run_data_list << run_data
|
302
|
+
}
|
251
303
|
|
252
|
-
|
304
|
+
loop {
|
305
|
+
if run_data_list.length == 1
|
306
|
+
run_job = Kernel.neptune(run_data_list[0])
|
307
|
+
else
|
308
|
+
run_job = Kernel.neptune(run_data_list)
|
309
|
+
end
|
310
|
+
|
311
|
+
if run_job[:result] == :success
|
312
|
+
return run_job
|
313
|
+
else
|
314
|
+
run_data_list[0][:failed_attempts] += 1
|
315
|
+
Kernel.sleep(SLEEP_TIME) # TODO(cgb): this should exponentially backoff
|
316
|
+
end
|
317
|
+
}
|
253
318
|
end
|
254
319
|
|
255
320
|
|
256
321
|
# Constructs a Neptune job to get the output of a Babel job. If the job is not
|
257
322
|
# yet finished, this function waits until it does, and then returns the output
|
258
323
|
# of the job.
|
259
|
-
def self.wait_and_get_output(job_data)
|
324
|
+
def self.wait_and_get_output(job_data, output_location)
|
260
325
|
output_data = self.convert_to_neptune_params(job_data)
|
261
326
|
output_data[:type] = "output"
|
327
|
+
output_data[:output] = output_location
|
262
328
|
|
263
329
|
output = ""
|
264
330
|
time_to_sleep = SLEEP_TIME
|
@@ -267,9 +333,9 @@ module BabelHelper
|
|
267
333
|
if output == DOES_NOT_EXIST
|
268
334
|
# Exponentially back off, up to a limit of MAX_SLEEP_TIME
|
269
335
|
Kernel.sleep(time_to_sleep)
|
270
|
-
if time_to_sleep < MAX_SLEEP_TIME
|
271
|
-
|
272
|
-
end
|
336
|
+
#if time_to_sleep < MAX_SLEEP_TIME
|
337
|
+
# time_to_sleep *= 2
|
338
|
+
#end
|
273
339
|
else
|
274
340
|
break
|
275
341
|
end
|