neptune 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +4 -0
- data/bin/neptune +7 -0
- data/doc/AppControllerClient.html +113 -22
- data/doc/{Kernel.html → AppControllerException.html} +23 -55
- data/doc/BabelHelper.html +707 -0
- data/doc/BadConfigurationException.html +142 -0
- data/doc/CommonFunctions.html +121 -33
- data/doc/FileNotFoundException.html +142 -0
- data/doc/NeptuneHelper.html +1102 -0
- data/doc/Object.html +94 -749
- data/doc/bin/neptune.html +3 -1
- data/doc/created.rid +7 -5
- data/doc/index.html +73 -25
- data/doc/lib/app_controller_client_rb.html +1 -1
- data/doc/lib/babel_rb.html +68 -0
- data/doc/lib/common_functions_rb.html +3 -1
- data/doc/lib/custom_exceptions_rb.html +54 -0
- data/doc/lib/neptune_rb.html +3 -1
- data/lib/app_controller_client.rb +28 -10
- data/lib/babel.rb +260 -0
- data/lib/common_functions.rb +42 -28
- data/lib/custom_exceptions.rb +10 -0
- data/lib/neptune.rb +371 -304
- data/test/unit/test_app_controller_client.rb +9 -9
- data/test/unit/test_babel.rb +154 -0
- data/test/unit/test_common_functions.rb +39 -75
- data/test/unit/test_neptune.rb +168 -76
- data/test/unit/ts_all.rb +5 -0
- metadata +32 -8
data/doc/Object.html
CHANGED
|
@@ -41,6 +41,9 @@
|
|
|
41
41
|
<li><a href="./lib/app_controller_client_rb.html?TB_iframe=true&height=550&width=785"
|
|
42
42
|
class="thickbox" title="lib/app_controller_client.rb">lib/app_controller_client.rb</a></li>
|
|
43
43
|
|
|
44
|
+
<li><a href="./lib/babel_rb.html?TB_iframe=true&height=550&width=785"
|
|
45
|
+
class="thickbox" title="lib/babel.rb">lib/babel.rb</a></li>
|
|
46
|
+
|
|
44
47
|
<li><a href="./lib/neptune_rb.html?TB_iframe=true&height=550&width=785"
|
|
45
48
|
class="thickbox" title="lib/neptune.rb">lib/neptune.rb</a></li>
|
|
46
49
|
|
|
@@ -72,34 +75,10 @@
|
|
|
72
75
|
<h3 class="section-header">Methods</h3>
|
|
73
76
|
<ul class="link-list">
|
|
74
77
|
|
|
75
|
-
<li><a href="#method-i-
|
|
76
|
-
|
|
77
|
-
<li><a href="#method-i-do_preprocessing">#do_preprocessing</a></li>
|
|
78
|
-
|
|
79
|
-
<li><a href="#method-i-get_input">#get_input</a></li>
|
|
80
|
-
|
|
81
|
-
<li><a href="#method-i-get_job_data">#get_job_data</a></li>
|
|
82
|
-
|
|
83
|
-
<li><a href="#method-i-get_std_out_and_err">#get_std_out_and_err</a></li>
|
|
78
|
+
<li><a href="#method-i-babel">#babel</a></li>
|
|
84
79
|
|
|
85
80
|
<li><a href="#method-i-neptune">#neptune</a></li>
|
|
86
81
|
|
|
87
|
-
<li><a href="#method-i-preprocess_compile">#preprocess_compile</a></li>
|
|
88
|
-
|
|
89
|
-
<li><a href="#method-i-preprocess_erlang">#preprocess_erlang</a></li>
|
|
90
|
-
|
|
91
|
-
<li><a href="#method-i-preprocess_mpi">#preprocess_mpi</a></li>
|
|
92
|
-
|
|
93
|
-
<li><a href="#method-i-preprocess_ssa">#preprocess_ssa</a></li>
|
|
94
|
-
|
|
95
|
-
<li><a href="#method-i-run_job">#run_job</a></li>
|
|
96
|
-
|
|
97
|
-
<li><a href="#method-i-upload_app_for_cicero">#upload_app_for_cicero</a></li>
|
|
98
|
-
|
|
99
|
-
<li><a href="#method-i-validate_storage_params">#validate_storage_params</a></li>
|
|
100
|
-
|
|
101
|
-
<li><a href="#method-i-wait_for_compilation_to_finish">#wait_for_compilation_to_finish</a></li>
|
|
102
|
-
|
|
103
82
|
</ul>
|
|
104
83
|
</div>
|
|
105
84
|
|
|
@@ -129,9 +108,17 @@
|
|
|
129
108
|
|
|
130
109
|
<li><a href="./AppControllerClient.html">AppControllerClient</a></li>
|
|
131
110
|
|
|
111
|
+
<li><a href="./AppControllerException.html">AppControllerException</a></li>
|
|
112
|
+
|
|
113
|
+
<li><a href="./BabelHelper.html">BabelHelper</a></li>
|
|
114
|
+
|
|
115
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
|
|
116
|
+
|
|
132
117
|
<li><a href="./CommonFunctions.html">CommonFunctions</a></li>
|
|
133
118
|
|
|
134
|
-
<li><a href="./
|
|
119
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
|
|
120
|
+
|
|
121
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
|
|
135
122
|
|
|
136
123
|
<li><a href="./Object.html">Object</a></li>
|
|
137
124
|
|
|
@@ -173,6 +160,41 @@ timeout. The next version should replace this and properly timeout and not
|
|
|
173
160
|
use long calls unless necessary.</p></dd>
|
|
174
161
|
|
|
175
162
|
|
|
163
|
+
<dt><a name="NEEDS_BUCKET_INFO">NEEDS_BUCKET_INFO</a></dt>
|
|
164
|
+
|
|
165
|
+
<dd class="description"><p>If the user doesn’t give us enough info to infer what bucket we should
|
|
166
|
+
place their code in, this message is displayed and execution aborts.</p></dd>
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
<dt><a name="DOES_NOT_EXIST">DOES_NOT_EXIST</a></dt>
|
|
170
|
+
|
|
171
|
+
<dd class="description"><p>The constant string that a Neptune output job returns if the output does
|
|
172
|
+
not yet exist.</p></dd>
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
<dt><a name="SLEEP_TIME">SLEEP_TIME</a></dt>
|
|
176
|
+
|
|
177
|
+
<dd class="description"><p>The initial amount of time, in seconds, to sleep between output job
|
|
178
|
+
requests. An exponential backoff is used with this value as the starting
|
|
179
|
+
sleep time.</p></dd>
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
<dt><a name="MAX_SLEEP_TIME">MAX_SLEEP_TIME</a></dt>
|
|
183
|
+
|
|
184
|
+
<dd class="description"><p>The maximum amount of time that we should sleep to, when waiting for output
|
|
185
|
+
job requests.</p></dd>
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
<dt><a name="ALLOWED_JOB_TYPES">ALLOWED_JOB_TYPES</a></dt>
|
|
189
|
+
|
|
190
|
+
<dd class="description"><p>A list of all the Neptune job types that we support</p></dd>
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
<dt><a name="JOB_TYPE_NOT_ALLOWED">JOB_TYPE_NOT_ALLOWED</a></dt>
|
|
194
|
+
|
|
195
|
+
<dd class="description"><p>The string to display for disallowed job types.</p></dd>
|
|
196
|
+
|
|
197
|
+
|
|
176
198
|
<dt><a name="NO_NODES_NEEDED">NO_NODES_NEEDED</a></dt>
|
|
177
199
|
|
|
178
200
|
<dd class="description"><p>A list of Neptune jobs that do not require nodes to be spawned up for
|
|
@@ -210,176 +232,12 @@ computation can be performed.</p></dd>
|
|
|
210
232
|
<h3 class="section-header">Public Instance Methods</h3>
|
|
211
233
|
|
|
212
234
|
|
|
213
|
-
<div id="
|
|
214
|
-
<a name="method-i-
|
|
235
|
+
<div id="babel-method" class="method-detail ">
|
|
236
|
+
<a name="method-i-babel"></a>
|
|
215
237
|
|
|
216
238
|
|
|
217
239
|
<div class="method-heading">
|
|
218
|
-
<span class="method-name">
|
|
219
|
-
class="method-args">(job_data, ssh_args, shadow_ip, shell=Kernel.method(:`))</span>
|
|
220
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
221
|
-
</div>
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
<div class="method-description">
|
|
225
|
-
|
|
226
|
-
<p>This method sends out a request to compile code, waits for it to finish,
|
|
227
|
-
and gets the standard out and error returned from the compilation. This
|
|
228
|
-
method returns a hash containing the standard out, error, and a result that
|
|
229
|
-
indicates whether or not the compilation was successful.</p>
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
<div class="method-source-code"
|
|
234
|
-
id="compile_code-source">
|
|
235
|
-
<pre>
|
|
236
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 301</span>
|
|
237
|
-
def compile_code(job_data, ssh_args, shadow_ip, shell=<span class="ruby-constant">Kernel</span>.method(:`))
|
|
238
|
-
compiled_location = controller.compile_code(job_data)
|
|
239
|
-
|
|
240
|
-
copy_to = job_data[<span class="ruby-string">"@copy_to"</span>]
|
|
241
|
-
|
|
242
|
-
wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
|
|
243
|
-
|
|
244
|
-
<span class="ruby-constant">FileUtils</span>.rm_rf(copy_to)
|
|
245
|
-
|
|
246
|
-
scp_command = "scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2>&1"
|
|
247
|
-
puts scp_command
|
|
248
|
-
shell.call(scp_command)
|
|
249
|
-
|
|
250
|
-
code = job_data[<span class="ruby-string">"@code"</span>]
|
|
251
|
-
dirs = code.split(<span class="ruby-regexp">/\//</span>)
|
|
252
|
-
remote_dir = <span class="ruby-string">"/tmp/"</span> + dirs[-1]
|
|
253
|
-
|
|
254
|
-
[remote_dir, compiled_location].each { |remote_files|
|
|
255
|
-
ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_files}' 2>&1"
|
|
256
|
-
puts ssh_command
|
|
257
|
-
shell.call(ssh_command)
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
return get_std_out_and_err(copy_to)
|
|
261
|
-
end</pre>
|
|
262
|
-
</div>
|
|
263
|
-
|
|
264
|
-
</div>
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
</div>
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
<div id="do_preprocessing-method" class="method-detail ">
|
|
273
|
-
<a name="method-i-do_preprocessing"></a>
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
<div class="method-heading">
|
|
277
|
-
<span class="method-name">do_preprocessing</span><span
|
|
278
|
-
class="method-args">(job_data)</span>
|
|
279
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
280
|
-
</div>
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
<div class="method-description">
|
|
284
|
-
|
|
285
|
-
<p>Certain types of jobs need steps to be taken before they can be started
|
|
286
|
-
(e.g., copying input data or code over). This method dispatches the right
|
|
287
|
-
method to use based on the type of the job that the user has asked to run.</p>
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
<div class="method-source-code"
|
|
292
|
-
id="do_preprocessing-source">
|
|
293
|
-
<pre>
|
|
294
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 52</span>
|
|
295
|
-
def do_preprocessing(job_data)
|
|
296
|
-
job_type = job_data[<span class="ruby-string">"@type"</span>]
|
|
297
|
-
if !<span class="ruby-constant">NEED_PREPROCESSING</span>.include?(job_type)
|
|
298
|
-
return
|
|
299
|
-
end
|
|
300
|
-
|
|
301
|
-
preprocess = "preprocess_#{job_type}".to_sym
|
|
302
|
-
send(preprocess, job_data)
|
|
303
|
-
end</pre>
|
|
304
|
-
</div>
|
|
305
|
-
|
|
306
|
-
</div>
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
</div>
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
<div id="get_input-method" class="method-detail ">
|
|
315
|
-
<a name="method-i-get_input"></a>
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
<div class="method-heading">
|
|
319
|
-
<span class="method-name">get_input</span><span
|
|
320
|
-
class="method-args">(job_data, ssh_args, shadow_ip, controller, file=File, shell=Kernel.method(:`))</span>
|
|
321
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
322
|
-
</div>
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
<div class="method-description">
|
|
326
|
-
|
|
327
|
-
<p>This method takes a file on the local user’s computer and stores it
|
|
328
|
-
remotely via AppScale. It returns a hash map indicating whether or not the
|
|
329
|
-
job succeeded and if it failed, the reason for it.</p>
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
<div class="method-source-code"
|
|
334
|
-
id="get_input-source">
|
|
335
|
-
<pre>
|
|
336
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 247</span>
|
|
337
|
-
def get_input(job_data, ssh_args, shadow_ip, controller, file=<span class="ruby-constant">File</span>,
|
|
338
|
-
shell=<span class="ruby-constant">Kernel</span>.method(:`))
|
|
339
|
-
result = {:result => :success}
|
|
340
|
-
|
|
341
|
-
if !job_data[<span class="ruby-string">"@local"</span>]
|
|
342
|
-
abort(<span class="ruby-string">"You failed to specify a file to copy over via the :local flag."</span>)
|
|
343
|
-
end
|
|
344
|
-
|
|
345
|
-
local_file = file.expand_path(job_data[<span class="ruby-string">"@local"</span>])
|
|
346
|
-
if !file.exists?(local_file)
|
|
347
|
-
reason = "the file you specified to copy, #{local_file}, doesn't exist." +
|
|
348
|
-
<span class="ruby-string">" Please specify a file that exists and try again."</span>
|
|
349
|
-
return {:result => :failure, :reason => reason}
|
|
350
|
-
end
|
|
351
|
-
|
|
352
|
-
remote = "/tmp/neptune-input-#{rand(100000)}"
|
|
353
|
-
scp_cmd = "scp -r #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
|
|
354
|
-
puts scp_cmd
|
|
355
|
-
shell.call(scp_cmd)
|
|
356
|
-
|
|
357
|
-
job_data[<span class="ruby-string">"@local"</span>] = remote
|
|
358
|
-
puts "job data = #{job_data.inspect}"
|
|
359
|
-
response = controller.put_input(job_data)
|
|
360
|
-
if response
|
|
361
|
-
return {:result => :success}
|
|
362
|
-
else
|
|
363
|
-
<span class="ruby-comment"># TODO - expand this to include the reason why it failed</span>
|
|
364
|
-
return {:result => :failure}
|
|
365
|
-
end
|
|
366
|
-
end</pre>
|
|
367
|
-
</div>
|
|
368
|
-
|
|
369
|
-
</div>
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
</div>
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
<div id="get_job_data-method" class="method-detail ">
|
|
378
|
-
<a name="method-i-get_job_data"></a>
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
<div class="method-heading">
|
|
382
|
-
<span class="method-name">get_job_data</span><span
|
|
240
|
+
<span class="method-name">babel</span><span
|
|
383
241
|
class="method-args">(params)</span>
|
|
384
242
|
<span class="method-click-advice">click to toggle source</span>
|
|
385
243
|
</div>
|
|
@@ -387,106 +245,48 @@ end</pre>
|
|
|
387
245
|
|
|
388
246
|
<div class="method-description">
|
|
389
247
|
|
|
390
|
-
|
|
248
|
+
<p>Babel provides a nice wrapper around Neptune jobs. Instead of making users
|
|
249
|
+
write multiple Neptune jobs to actually run code (e.g., putting input in
|
|
250
|
+
the datastore, run the job, get the output back), Babel automatically
|
|
251
|
+
handles this.</p>
|
|
391
252
|
|
|
392
253
|
|
|
393
254
|
|
|
394
255
|
<div class="method-source-code"
|
|
395
|
-
id="
|
|
256
|
+
id="babel-source">
|
|
396
257
|
<pre>
|
|
397
|
-
<span class="ruby-comment"># File lib/
|
|
398
|
-
def
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
if type == <span class="ruby-string">"upc"</span> or type == <span class="ruby-string">"x10"</span>
|
|
412
|
-
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"mpi"</span>
|
|
413
|
-
type = <span class="ruby-string">"mpi"</span>
|
|
414
|
-
end
|
|
415
|
-
|
|
416
|
-
<span class="ruby-comment"># kdt jobs also run as mpi jobs, but need to pass along an executable</span>
|
|
417
|
-
<span class="ruby-comment"># parameter to let mpiexec know to use python to exec it</span>
|
|
418
|
-
if type == <span class="ruby-string">"kdt"</span>
|
|
419
|
-
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"mpi"</span>
|
|
420
|
-
type = <span class="ruby-string">"mpi"</span>
|
|
421
|
-
|
|
422
|
-
job_data[<span class="ruby-string">"@executable"</span>] = <span class="ruby-string">"python"</span>
|
|
423
|
-
end
|
|
424
|
-
|
|
425
|
-
if job_data[<span class="ruby-string">"@nodes_to_use"</span>].class == <span class="ruby-constant">Hash</span>
|
|
426
|
-
job_data[<span class="ruby-string">"@nodes_to_use"</span>] = job_data[<span class="ruby-string">"@nodes_to_use"</span>].to_a.flatten
|
|
427
|
-
end
|
|
428
|
-
|
|
429
|
-
if !<span class="ruby-constant">NO_OUTPUT_NEEDED</span>.include?(type)
|
|
430
|
-
if (job_data[<span class="ruby-string">"@output"</span>].nil? or job_data[<span class="ruby-string">"@output"</span>] == <span class="ruby-string">""</span>)
|
|
431
|
-
abort(<span class="ruby-string">"Job output must be specified"</span>)
|
|
258
|
+
<span class="ruby-comment"># File lib/babel.rb, line 42</span>
|
|
259
|
+
def babel(params)
|
|
260
|
+
<span class="ruby-comment"># Since this whole function should run asynchronously, we run it as a future.</span>
|
|
261
|
+
<span class="ruby-comment"># It automatically starts running in a new thread, and attempting to get the</span>
|
|
262
|
+
<span class="ruby-comment"># value of what this returns causes it to block until the job completes.</span>
|
|
263
|
+
future {
|
|
264
|
+
job_data = <span class="ruby-constant">BabelHelper</span>.convert_from_neptune_params(params)
|
|
265
|
+
<span class="ruby-constant">NeptuneHelper</span>.validate_storage_params(job_data) <span class="ruby-comment"># adds in S3 storage params</span>
|
|
266
|
+
|
|
267
|
+
<span class="ruby-comment"># :code is the only required parameter - everything else can use default vals</span>
|
|
268
|
+
<span class="ruby-constant">NeptuneHelper</span>.require_param(<span class="ruby-string">"@code"</span>, job_data)
|
|
269
|
+
|
|
270
|
+
if job_data[<span class="ruby-string">"@output"</span>].nil? or job_data[<span class="ruby-string">"@output"</span>].empty?
|
|
271
|
+
job_data[<span class="ruby-string">"@output"</span>] = <span class="ruby-constant">BabelHelper</span>.generate_output_location(job_data)
|
|
432
272
|
end
|
|
273
|
+
<span class="ruby-constant">BabelHelper</span>.ensure_output_does_not_exist(job_data)
|
|
433
274
|
|
|
434
|
-
if job_data[<span class="ruby-string">"@
|
|
435
|
-
|
|
275
|
+
if job_data[<span class="ruby-string">"@is_remote"</span>]
|
|
276
|
+
<span class="ruby-constant">BabelHelper</span>.validate_inputs(job_data)
|
|
277
|
+
else
|
|
278
|
+
<span class="ruby-constant">BabelHelper</span>.put_code(job_data)
|
|
279
|
+
<span class="ruby-constant">BabelHelper</span>.put_inputs(job_data)
|
|
436
280
|
end
|
|
437
|
-
end
|
|
438
|
-
|
|
439
|
-
return job_data
|
|
440
|
-
end</pre>
|
|
441
|
-
</div>
|
|
442
|
-
|
|
443
|
-
</div>
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
</div>
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
<div id="get_std_out_and_err-method" class="method-detail ">
|
|
452
|
-
<a name="method-i-get_std_out_and_err"></a>
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
<div class="method-heading">
|
|
456
|
-
<span class="method-name">get_std_out_and_err</span><span
|
|
457
|
-
class="method-args">(location)</span>
|
|
458
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
459
|
-
</div>
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
<div class="method-description">
|
|
463
|
-
|
|
464
|
-
<p>This method returns a hash containing the standard out and standard error
|
|
465
|
-
from a completed job, as well as a result field that indicates whether or
|
|
466
|
-
not the job completed successfully (success = no errors).</p>
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
<div class="method-source-code"
|
|
471
|
-
id="get_std_out_and_err-source">
|
|
472
|
-
<pre>
|
|
473
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 330</span>
|
|
474
|
-
def get_std_out_and_err(location)
|
|
475
|
-
result = {}
|
|
476
281
|
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
else
|
|
486
|
-
result[:result] = :success
|
|
487
|
-
end
|
|
488
|
-
|
|
489
|
-
return result
|
|
282
|
+
<span class="ruby-constant">BabelHelper</span>.run_job(job_data)
|
|
283
|
+
<span class="ruby-comment"># So actually retrieving the job's output is done via a promise, so only if</span>
|
|
284
|
+
<span class="ruby-comment"># the user actually uses the value do we actually go and poll for output.</span>
|
|
285
|
+
<span class="ruby-comment"># The running of the job is done above, outside of the promise, so</span>
|
|
286
|
+
<span class="ruby-comment"># the job is always run, regardless of whether or not we get its output.</span>
|
|
287
|
+
<span class="ruby-constant">BabelHelper</span>.wait_and_get_output(job_data)
|
|
288
|
+
<span class="ruby-comment"># promise { BabelHelper.wait_and_get_output(job_data) }</span>
|
|
289
|
+
}
|
|
490
290
|
end</pre>
|
|
491
291
|
</div>
|
|
492
292
|
|
|
@@ -512,7 +312,7 @@ end</pre>
|
|
|
512
312
|
<div class="method-description">
|
|
513
313
|
|
|
514
314
|
<p>This method is the heart of Neptune - here, we take blocks of code that the
|
|
515
|
-
user has written and convert them into HPC job requests. At a high level,
|
|
315
|
+
user has written and convert them into HPC job requests. At a high level,
|
|
516
316
|
the user can request to run a job, retrieve a job’s output, or modify the
|
|
517
317
|
access policy (ACL) for the output of a job. By default, job data is
|
|
518
318
|
private, but a Neptune job can be used to set it to public later (and
|
|
@@ -523,15 +323,14 @@ vice-versa).</p>
|
|
|
523
323
|
<div class="method-source-code"
|
|
524
324
|
id="neptune-source">
|
|
525
325
|
<pre>
|
|
526
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
|
326
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 485</span>
|
|
527
327
|
def neptune(params)
|
|
528
|
-
puts <span class="ruby-string">"Received a request to run a job."</span>
|
|
529
|
-
puts params[:type]
|
|
328
|
+
<span class="ruby-constant">Kernel</span>.puts <span class="ruby-string">"Received a request to run a job."</span>
|
|
329
|
+
<span class="ruby-constant">Kernel</span>.puts params[:type]
|
|
530
330
|
|
|
531
|
-
job_data = get_job_data(params)
|
|
532
|
-
validate_storage_params(job_data)
|
|
533
|
-
puts "job data = #{job_data.inspect}"
|
|
534
|
-
do_preprocessing(job_data)
|
|
331
|
+
job_data = <span class="ruby-constant">NeptuneHelper</span>.get_job_data(params)
|
|
332
|
+
<span class="ruby-constant">NeptuneHelper</span>.validate_storage_params(job_data)
|
|
333
|
+
<span class="ruby-constant">Kernel</span>.puts "job data = #{job_data.inspect}"
|
|
535
334
|
keyname = job_data[<span class="ruby-string">"@keyname"</span>]
|
|
536
335
|
|
|
537
336
|
shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
|
|
@@ -539,463 +338,9 @@ def neptune(params)
|
|
|
539
338
|
ssh_key = <span class="ruby-constant">File</span>.expand_path("~/.appscale/#{keyname}.key")
|
|
540
339
|
ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "
|
|
541
340
|
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
</div>
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
</div>
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
<div id="preprocess_compile-method" class="method-detail ">
|
|
555
|
-
<a name="method-i-preprocess_compile"></a>
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
<div class="method-heading">
|
|
559
|
-
<span class="method-name">preprocess_compile</span><span
|
|
560
|
-
class="method-args">(job_data, shell=Kernel.method(:`))</span>
|
|
561
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
562
|
-
</div>
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
<div class="method-description">
|
|
566
|
-
|
|
567
|
-
<p>This preprocessing method copies over the user’s code to the Shadow node
|
|
568
|
-
so that it can be compiled there. A future version of this method may also
|
|
569
|
-
copy over libraries as well.</p>
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
<div class="method-source-code"
|
|
574
|
-
id="preprocess_compile-source">
|
|
575
|
-
<pre>
|
|
576
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 65</span>
|
|
577
|
-
def preprocess_compile(job_data, shell=<span class="ruby-constant">Kernel</span>.method(:`))
|
|
578
|
-
code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@code"</span>])
|
|
579
|
-
if !<span class="ruby-constant">File</span>.exists?(code)
|
|
580
|
-
abort("The source file #{code} does not exist.")
|
|
581
|
-
end
|
|
582
|
-
|
|
583
|
-
suffix = code.split(<span class="ruby-string">'/'</span>)[-1]
|
|
584
|
-
dest = "/tmp/#{suffix}"
|
|
585
|
-
keyname = job_data[<span class="ruby-string">"@keyname"</span>]
|
|
586
|
-
shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
|
|
587
|
-
|
|
588
|
-
ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}"
|
|
589
|
-
remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}' 2>&1"
|
|
590
|
-
puts remove_dir
|
|
591
|
-
shell.call(remove_dir)
|
|
592
|
-
|
|
593
|
-
<span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(code, dest, keyname, is_dir=true)
|
|
594
|
-
|
|
595
|
-
job_data[<span class="ruby-string">"@code"</span>] = dest
|
|
596
|
-
end</pre>
|
|
597
|
-
</div>
|
|
598
|
-
|
|
599
|
-
</div>
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
</div>
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
<div id="preprocess_erlang-method" class="method-detail ">
|
|
608
|
-
<a name="method-i-preprocess_erlang"></a>
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
<div class="method-heading">
|
|
612
|
-
<span class="method-name">preprocess_erlang</span><span
|
|
613
|
-
class="method-args">(job_data, file=File, common_functions=CommonFunctions)</span>
|
|
614
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
615
|
-
</div>
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
<div class="method-description">
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
<div class="method-source-code"
|
|
625
|
-
id="preprocess_erlang-source">
|
|
626
|
-
<pre>
|
|
627
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 86</span>
|
|
628
|
-
def preprocess_erlang(job_data, file=<span class="ruby-constant">File</span>, common_functions=<span class="ruby-constant">CommonFunctions</span>)
|
|
629
|
-
if !job_data[<span class="ruby-string">"@code"</span>]
|
|
630
|
-
abort(<span class="ruby-string">"When running Erlang jobs, :code must be specified."</span>)
|
|
631
|
-
end
|
|
632
|
-
|
|
633
|
-
source_code = file.expand_path(job_data[<span class="ruby-string">"@code"</span>])
|
|
634
|
-
if !file.exists?(source_code)
|
|
635
|
-
abort("The specified code, #{job_data['@code']}," +
|
|
636
|
-
<span class="ruby-string">" didn't exist. Please specify one that exists and try again"</span>)
|
|
637
|
-
end
|
|
638
|
-
dest_code = <span class="ruby-string">"/tmp/"</span>
|
|
639
|
-
|
|
640
|
-
keyname = job_data[<span class="ruby-string">"@keyname"</span>]
|
|
641
|
-
common_functions.scp_to_shadow(source_code, dest_code, keyname)
|
|
642
|
-
end</pre>
|
|
643
|
-
</div>
|
|
644
|
-
|
|
645
|
-
</div>
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
</div>
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
<div id="preprocess_mpi-method" class="method-detail ">
|
|
654
|
-
<a name="method-i-preprocess_mpi"></a>
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
<div class="method-heading">
|
|
658
|
-
<span class="method-name">preprocess_mpi</span><span
|
|
659
|
-
class="method-args">(job_data)</span>
|
|
660
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
661
|
-
</div>
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
<div class="method-description">
|
|
665
|
-
|
|
666
|
-
<p>This preprocessing method verifies that the user specified the number of
|
|
667
|
-
nodes to use. If they also specified the number of processes to use, we
|
|
668
|
-
also verify that this value is at least as many as the number of nodes
|
|
669
|
-
(that is, nodes can’t be underprovisioned in MPI).</p>
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
<div class="method-source-code"
|
|
674
|
-
id="preprocess_mpi-source">
|
|
675
|
-
<pre>
|
|
676
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 106</span>
|
|
677
|
-
def preprocess_mpi(job_data)
|
|
678
|
-
if !job_data[<span class="ruby-string">"@nodes_to_use"</span>]
|
|
679
|
-
abort(<span class="ruby-string">"When running MPI jobs, :nodes_to_use must be specified."</span>)
|
|
680
|
-
end
|
|
681
|
-
|
|
682
|
-
if !job_data[<span class="ruby-string">"@procs_to_use"</span>]
|
|
683
|
-
abort(<span class="ruby-string">"When running MPI jobs, :procs_to_use must be specified."</span>)
|
|
684
|
-
end
|
|
685
|
-
|
|
686
|
-
if job_data[<span class="ruby-string">"@procs_to_use"</span>]
|
|
687
|
-
p = job_data[<span class="ruby-string">"@procs_to_use"</span>]
|
|
688
|
-
n = job_data[<span class="ruby-string">"@nodes_to_use"</span>]
|
|
689
|
-
if p < n
|
|
690
|
-
abort(<span class="ruby-string">"When specifying both :procs_to_use and :nodes_to_use"</span> +
|
|
691
|
-
<span class="ruby-string">", :procs_to_use must be at least as large as :nodes_to_use. Please "</span> +
|
|
692
|
-
"change this and try again. You specified :procs_to_use = #{p} and" +
|
|
693
|
-
":nodes_to_use = #{n}.")
|
|
694
|
-
end
|
|
695
|
-
end
|
|
696
|
-
|
|
697
|
-
if job_data[<span class="ruby-string">"@argv"</span>]
|
|
698
|
-
argv = job_data[<span class="ruby-string">"@argv"</span>]
|
|
699
|
-
if argv.class != <span class="ruby-constant">String</span> and argv.class != <span class="ruby-constant">Array</span>
|
|
700
|
-
abort(<span class="ruby-string">"The value specified for :argv must be either a String or Array"</span>)
|
|
701
|
-
end
|
|
702
|
-
|
|
703
|
-
if argv.class == <span class="ruby-constant">Array</span>
|
|
704
|
-
job_data[<span class="ruby-string">"@argv"</span>] = argv.join(<span class="ruby-string">' '</span>)
|
|
705
|
-
end
|
|
706
|
-
end
|
|
707
|
-
|
|
708
|
-
return job_data
|
|
709
|
-
end</pre>
|
|
710
|
-
</div>
|
|
711
|
-
|
|
712
|
-
</div>
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
</div>
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
<div id="preprocess_ssa-method" class="method-detail ">
|
|
721
|
-
<a name="method-i-preprocess_ssa"></a>
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
<div class="method-heading">
|
|
725
|
-
<span class="method-name">preprocess_ssa</span><span
|
|
726
|
-
class="method-args">(job_data)</span>
|
|
727
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
728
|
-
</div>
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
<div class="method-description">
|
|
732
|
-
|
|
733
|
-
<p>This preprocessing method verifies that the user specified the number of
|
|
734
|
-
trajectories to run, via either :trajectories or :simulations. Both should
|
|
735
|
-
not be specified - only one or the other, and regardless of which they
|
|
736
|
-
specify, convert it to be :trajectories.</p>
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
<div class="method-source-code"
|
|
741
|
-
id="preprocess_ssa-source">
|
|
742
|
-
<pre>
|
|
743
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 144</span>
|
|
744
|
-
def preprocess_ssa(job_data)
|
|
745
|
-
if job_data[<span class="ruby-string">"@simulations"</span>] and job_data[<span class="ruby-string">"@trajectories"</span>]
|
|
746
|
-
abort(<span class="ruby-string">"Both :simulations and :trajectories cannot be specified - use one"</span> +
|
|
747
|
-
<span class="ruby-string">" or the other."</span>)
|
|
748
|
-
end
|
|
749
|
-
|
|
750
|
-
if job_data[<span class="ruby-string">"@simulations"</span>]
|
|
751
|
-
job_data[<span class="ruby-string">"@trajectories"</span>] = job_data[<span class="ruby-string">"@simulations"</span>]
|
|
752
|
-
job_data.delete(<span class="ruby-string">"@simulations"</span>)
|
|
753
|
-
end
|
|
754
|
-
|
|
755
|
-
if !job_data[<span class="ruby-string">"@trajectories"</span>]
|
|
756
|
-
abort(<span class="ruby-string">":trajectories needs to be specified when running ssa jobs"</span>)
|
|
757
|
-
end
|
|
758
|
-
|
|
759
|
-
return job_data
|
|
760
|
-
end</pre>
|
|
761
|
-
</div>
|
|
762
|
-
|
|
763
|
-
</div>
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
</div>
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
<div id="run_job-method" class="method-detail ">
|
|
772
|
-
<a name="method-i-run_job"></a>
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
<div class="method-heading">
|
|
776
|
-
<span class="method-name">run_job</span><span
|
|
777
|
-
class="method-args">(job_data, ssh_args, shadow_ip, secret, controller=AppControllerClient, file=File)</span>
|
|
778
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
779
|
-
</div>
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
<div class="method-description">
|
|
783
|
-
|
|
784
|
-
<p>This method actually runs the Neptune job, given information about the job
|
|
785
|
-
as well as information about the node to send the request to.</p>
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
<div class="method-source-code"
|
|
790
|
-
id="run_job-source">
|
|
791
|
-
<pre>
|
|
792
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 376</span>
|
|
793
|
-
def run_job(job_data, ssh_args, shadow_ip, secret,
|
|
794
|
-
controller=<span class="ruby-constant">AppControllerClient</span>, file=<span class="ruby-constant">File</span>)
|
|
795
|
-
controller = controller.new(shadow_ip, secret)
|
|
796
|
-
|
|
797
|
-
<span class="ruby-comment"># TODO - right now the job is assumed to succeed in many cases</span>
|
|
798
|
-
<span class="ruby-comment"># need to investigate the various failure scenarios</span>
|
|
799
|
-
result = { :result => :success }
|
|
800
|
-
|
|
801
|
-
case job_data[<span class="ruby-string">"@type"</span>]
|
|
802
|
-
when <span class="ruby-string">"input"</span>
|
|
803
|
-
result = get_input(job_data, ssh_args, shadow_ip, controller, file)
|
|
804
|
-
when <span class="ruby-string">"output"</span>
|
|
805
|
-
result[:output] = controller.get_output(job_data)
|
|
806
|
-
when <span class="ruby-string">"get-acl"</span>
|
|
807
|
-
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"acl"</span>
|
|
808
|
-
result[:acl] = controller.get_acl(job_data)
|
|
809
|
-
when <span class="ruby-string">"set-acl"</span>
|
|
810
|
-
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"acl"</span>
|
|
811
|
-
result[:acl] = controller.set_acl(job_data)
|
|
812
|
-
when <span class="ruby-string">"compile"</span>
|
|
813
|
-
result = compile_code(job_data, ssh_args, shadow_ip)
|
|
814
|
-
when <span class="ruby-string">"cicero"</span>
|
|
815
|
-
upload_app_for_cicero(job_data)
|
|
816
|
-
msg = controller.start_neptune_job(job_data)
|
|
817
|
-
result[:msg] = msg
|
|
818
|
-
result[:result] = :failure if result[:msg] !~ <span class="ruby-regexp">/job is now running\Z/</span>
|
|
819
|
-
else
|
|
820
|
-
msg = controller.start_neptune_job(job_data)
|
|
821
|
-
result[:msg] = msg
|
|
822
|
-
result[:result] = :failure if result[:msg] !~ <span class="ruby-regexp">/job is now running\Z/</span>
|
|
823
|
-
end
|
|
824
|
-
|
|
825
|
-
return result
|
|
826
|
-
end</pre>
|
|
827
|
-
</div>
|
|
828
|
-
|
|
829
|
-
</div>
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
</div>
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
<div id="upload_app_for_cicero-method" class="method-detail ">
|
|
838
|
-
<a name="method-i-upload_app_for_cicero"></a>
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
<div class="method-heading">
|
|
842
|
-
<span class="method-name">upload_app_for_cicero</span><span
|
|
843
|
-
class="method-args">(job_data)</span>
|
|
844
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
845
|
-
</div>
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
<div class="method-description">
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
<div class="method-source-code"
|
|
855
|
-
id="upload_app_for_cicero-source">
|
|
856
|
-
<pre>
|
|
857
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 348</span>
|
|
858
|
-
def upload_app_for_cicero(job_data)
|
|
859
|
-
if !job_data[<span class="ruby-string">"@app"</span>]
|
|
860
|
-
puts <span class="ruby-string">"No app specified, not uploading..."</span>
|
|
861
|
-
return
|
|
862
|
-
end
|
|
863
|
-
|
|
864
|
-
app_location = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@app"</span>])
|
|
865
|
-
if !<span class="ruby-constant">File</span>.exists?(app_location)
|
|
866
|
-
abort("The app you specified, #{app_location}, does not exist." +
|
|
867
|
-
<span class="ruby-string">"Please specify one that does and try again."</span>)
|
|
868
|
-
end
|
|
869
|
-
|
|
870
|
-
keyname = job_data[<span class="ruby-string">"@keyname"</span>] || <span class="ruby-string">"appscale"</span>
|
|
871
|
-
if job_data[<span class="ruby-string">"@appscale_tools"</span>]
|
|
872
|
-
upload_app = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@appscale_tools"</span>]) +
|
|
873
|
-
<span class="ruby-constant">File</span>::<span class="ruby-constant">SEPARATOR</span> + <span class="ruby-string">"bin"</span> + <span class="ruby-constant">File</span>::<span class="ruby-constant">SEPARATOR</span> + <span class="ruby-string">"appscale-upload-app"</span>
|
|
874
|
-
else
|
|
875
|
-
upload_app = <span class="ruby-string">"appscale-upload-app"</span>
|
|
876
|
-
end
|
|
877
|
-
|
|
878
|
-
puts "Uploading AppEngine app at #{app_location}"
|
|
879
|
-
upload_command = "#{upload_app} --file #{app_location} --test --keyname #{keyname}"
|
|
880
|
-
puts upload_command
|
|
881
|
-
puts `#{upload_command}`
|
|
882
|
-
end</pre>
|
|
883
|
-
</div>
|
|
884
|
-
|
|
885
|
-
</div>
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
</div>
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
<div id="validate_storage_params-method" class="method-detail ">
|
|
894
|
-
<a name="method-i-validate_storage_params"></a>
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
<div class="method-heading">
|
|
898
|
-
<span class="method-name">validate_storage_params</span><span
|
|
899
|
-
class="method-args">(job_data)</span>
|
|
900
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
901
|
-
</div>
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
<div class="method-description">
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
<div class="method-source-code"
|
|
911
|
-
id="validate_storage_params-source">
|
|
912
|
-
<pre>
|
|
913
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 206</span>
|
|
914
|
-
def validate_storage_params(job_data)
|
|
915
|
-
if !job_data[<span class="ruby-string">"@storage"</span>]
|
|
916
|
-
job_data[<span class="ruby-string">"@storage"</span>] = <span class="ruby-string">"appdb"</span>
|
|
917
|
-
end
|
|
918
|
-
|
|
919
|
-
storage = job_data[<span class="ruby-string">"@storage"</span>]
|
|
920
|
-
if !<span class="ruby-constant">ALLOWED_STORAGE_TYPES</span>.include?(storage)
|
|
921
|
-
abort("Supported storage types are #{ALLOWED_STORAGE_TYPES.join(', ')}" +
|
|
922
|
-
" - we do not support #{storage}.")
|
|
923
|
-
end
|
|
924
|
-
|
|
925
|
-
<span class="ruby-comment"># Our implementation for storing / retrieving via Google Storage</span>
|
|
926
|
-
<span class="ruby-comment"># and Walrus uses</span>
|
|
927
|
-
<span class="ruby-comment"># the same library as we do for S3 - so just tell it that it's S3</span>
|
|
928
|
-
if storage == <span class="ruby-string">"gstorage"</span> or storage == <span class="ruby-string">"walrus"</span>
|
|
929
|
-
storage = <span class="ruby-string">"s3"</span>
|
|
930
|
-
job_data[<span class="ruby-string">"@storage"</span>] = <span class="ruby-string">"s3"</span>
|
|
931
|
-
end
|
|
932
|
-
|
|
933
|
-
if storage == <span class="ruby-string">"s3"</span>
|
|
934
|
-
[<span class="ruby-string">"EC2_ACCESS_KEY"</span>, <span class="ruby-string">"EC2_SECRET_KEY"</span>, <span class="ruby-string">"S3_URL"</span>].each { |item|
|
|
935
|
-
if job_data["@#{item}"]
|
|
936
|
-
puts "Using specified #{item}"
|
|
937
|
-
else
|
|
938
|
-
if <span class="ruby-constant">ENV</span>[item]
|
|
939
|
-
puts "Using #{item} from environment"
|
|
940
|
-
job_data["@#{item}"] = <span class="ruby-constant">ENV</span>[item]
|
|
941
|
-
else
|
|
942
|
-
abort("When storing data to S3, #{item} must be specified or be in " +
|
|
943
|
-
<span class="ruby-string">"your environment. Please do so and try again."</span>)
|
|
944
|
-
end
|
|
945
|
-
end
|
|
946
|
-
}
|
|
947
|
-
end
|
|
948
|
-
|
|
949
|
-
return job_data
|
|
950
|
-
end</pre>
|
|
951
|
-
</div>
|
|
952
|
-
|
|
953
|
-
</div>
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
</div>
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
<div id="wait_for_compilation_to_finish-method" class="method-detail ">
|
|
962
|
-
<a name="method-i-wait_for_compilation_to_finish"></a>
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
<div class="method-heading">
|
|
966
|
-
<span class="method-name">wait_for_compilation_to_finish</span><span
|
|
967
|
-
class="method-args">(ssh_args, shadow_ip, compiled_location, shell=Kernel.method(:`))</span>
|
|
968
|
-
<span class="method-click-advice">click to toggle source</span>
|
|
969
|
-
</div>
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
<div class="method-description">
|
|
973
|
-
|
|
974
|
-
<p>This method waits for AppScale to finish compiling the user’s code,
|
|
975
|
-
indicated by AppScale copying the finished code to a pre-determined
|
|
976
|
-
location.</p>
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
<div class="method-source-code"
|
|
981
|
-
id="wait_for_compilation_to_finish-source">
|
|
982
|
-
<pre>
|
|
983
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 280</span>
|
|
984
|
-
def wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location,
|
|
985
|
-
shell=<span class="ruby-constant">Kernel</span>.method(:`))
|
|
986
|
-
loop {
|
|
987
|
-
ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2>&1"
|
|
988
|
-
puts ssh_command
|
|
989
|
-
ssh_result = shell.call(ssh_command)
|
|
990
|
-
puts "result was [#{ssh_result}]"
|
|
991
|
-
if ssh_result =~ <span class="ruby-regexp">/No such file or directory/</span>
|
|
992
|
-
puts <span class="ruby-string">"Still waiting for code to be compiled..."</span>
|
|
993
|
-
else
|
|
994
|
-
puts "compilation complete! Copying compiled code to #{copy_to}"
|
|
995
|
-
return
|
|
996
|
-
end
|
|
997
|
-
sleep(5)
|
|
998
|
-
}
|
|
341
|
+
controller = <span class="ruby-constant">AppControllerClient</span>.new(shadow_ip, secret)
|
|
342
|
+
<span class="ruby-constant">NeptuneHelper</span>.do_preprocessing(job_data, controller)
|
|
343
|
+
return <span class="ruby-constant">NeptuneHelper</span>.run_job(job_data, ssh_args, shadow_ip, secret)
|
|
999
344
|
end</pre>
|
|
1000
345
|
</div>
|
|
1001
346
|
|