neptune 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +6 -0
- data/bin/neptune +0 -1
- data/doc/AppControllerClient.html +23 -16
- data/doc/AppControllerException.html +2 -1
- data/doc/BabelHelper.html +19 -15
- data/doc/BadConfigurationException.html +3 -0
- data/doc/CommonFunctions.html +12 -16
- data/doc/FileNotFoundException.html +3 -0
- data/doc/NeptuneHelper.html +55 -39
- data/doc/Object.html +14 -17
- data/doc/bin/neptune.html +1 -1
- data/doc/created.rid +7 -7
- data/doc/lib/app_controller_client_rb.html +1 -1
- data/doc/lib/babel_rb.html +1 -1
- data/doc/lib/common_functions_rb.html +1 -1
- data/doc/lib/custom_exceptions_rb.html +1 -1
- data/doc/lib/neptune_rb.html +1 -1
- data/lib/app_controller_client.rb +33 -1
- data/lib/babel.rb +29 -9
- data/lib/common_functions.rb +15 -11
- data/lib/custom_exceptions.rb +8 -0
- data/lib/neptune.rb +91 -58
- data/test/unit/test_babel.rb +82 -2
- metadata +4 -4
data/README
CHANGED
@@ -67,6 +67,12 @@ http://www.neptune-lang.org/2011/6/Neptune-Picks-up-Best-Paper-at-ScienceCloud-2
|
|
67
67
|
|
68
68
|
Version History:
|
69
69
|
|
70
|
+
February 12, 2012 - 0.2.1 released, adding support for the babel() method on
|
71
|
+
non-Babel job types (e.g., MPI, MapReduce), and removed the is_remote
|
72
|
+
parameter for babel() calls (since it can always be inferred from :storage).
|
73
|
+
Added more unit tests accordingly for babel() calls, and completed documentation
|
74
|
+
per rdoc standards.
|
75
|
+
|
70
76
|
February 11, 2012 - 0.2.0 released, adding support for Babel jobs (code that
|
71
77
|
runs over multiple clouds, using their queues automatically) and the babel()
|
72
78
|
method, to automatically upload code, run it, and retrieve the output.
|
data/bin/neptune
CHANGED
@@ -172,7 +172,7 @@ AppScale Tools, but with non-Neptune SOAP calls removed.</p>
|
|
172
172
|
|
173
173
|
<div class="method-description">
|
174
174
|
|
175
|
-
|
175
|
+
<p>The SOAP client that we use to communicate with the AppController.</p>
|
176
176
|
|
177
177
|
</div>
|
178
178
|
</div>
|
@@ -189,7 +189,7 @@ AppScale Tools, but with non-Neptune SOAP calls removed.</p>
|
|
189
189
|
|
190
190
|
<div class="method-description">
|
191
191
|
|
192
|
-
|
192
|
+
<p>The IP address of the AppController that we will be connecting to.</p>
|
193
193
|
|
194
194
|
</div>
|
195
195
|
</div>
|
@@ -206,7 +206,9 @@ AppScale Tools, but with non-Neptune SOAP calls removed.</p>
|
|
206
206
|
|
207
207
|
<div class="method-description">
|
208
208
|
|
209
|
-
|
209
|
+
<p>The secret string that is used to authenticate this client with
|
210
|
+
AppControllers. It is initially generated by appscale-run-instances and can
|
211
|
+
be found on the machine that ran that tool, or on any AppScale machine.</p>
|
210
212
|
|
211
213
|
</div>
|
212
214
|
</div>
|
@@ -245,7 +247,7 @@ for us.</p>
|
|
245
247
|
<div class="method-source-code"
|
246
248
|
id="new-source">
|
247
249
|
<pre>
|
248
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
250
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 41</span>
|
249
251
|
def initialize(ip, secret)
|
250
252
|
<span class="ruby-ivar">@ip</span> = ip
|
251
253
|
<span class="ruby-ivar">@secret</span> = secret
|
@@ -289,14 +291,15 @@ end</pre>
|
|
289
291
|
|
290
292
|
<div class="method-description">
|
291
293
|
|
292
|
-
|
294
|
+
<p>Instructs the AppController to fetch the code specified and compile it. The
|
295
|
+
result should then be placed in a location specified in the job data.</p>
|
293
296
|
|
294
297
|
|
295
298
|
|
296
299
|
<div class="method-source-code"
|
297
300
|
id="compile_code-source">
|
298
301
|
<pre>
|
299
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
302
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 178</span>
|
300
303
|
def compile_code(job_data)
|
301
304
|
result = <span class="ruby-string">""</span>
|
302
305
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -328,14 +331,16 @@ end</pre>
|
|
328
331
|
|
329
332
|
<div class="method-description">
|
330
333
|
|
331
|
-
|
334
|
+
<p>Asks the AppController to see if the given file exists in the remote
|
335
|
+
datastore. If extra credentials are needed for this operation, they are
|
336
|
+
searched for within the job data.</p>
|
332
337
|
|
333
338
|
|
334
339
|
|
335
340
|
<div class="method-source-code"
|
336
341
|
id="does_file_exist-3F-source">
|
337
342
|
<pre>
|
338
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
343
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 203</span>
|
339
344
|
def does_file_exist?(file, job_data)
|
340
345
|
result = false
|
341
346
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -376,7 +381,7 @@ exceptions mirror that of start_neptune_job.</p>
|
|
376
381
|
<div class="method-source-code"
|
377
382
|
id="get_acl-source">
|
378
383
|
<pre>
|
379
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
384
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 151</span>
|
380
385
|
def get_acl(job_data)
|
381
386
|
result = <span class="ruby-string">""</span>
|
382
387
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -425,7 +430,7 @@ well as the input format used for job_data.</p>
|
|
425
430
|
<div class="method-source-code"
|
426
431
|
id="get_output-source">
|
427
432
|
<pre>
|
428
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
433
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 136</span>
|
429
434
|
def get_output(job_data)
|
430
435
|
result = <span class="ruby-string">""</span>
|
431
436
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -457,14 +462,16 @@ end</pre>
|
|
457
462
|
|
458
463
|
<div class="method-description">
|
459
464
|
|
460
|
-
|
465
|
+
<p>Asks the AppController for a list of all the Babel engines (each of which
|
466
|
+
is a queue to store jobs and something that executes tasks) that are
|
467
|
+
supported for the given credentials.</p>
|
461
468
|
|
462
469
|
|
463
470
|
|
464
471
|
<div class="method-source-code"
|
465
472
|
id="get_supported_babel_engines-source">
|
466
473
|
<pre>
|
467
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
474
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 191</span>
|
468
475
|
def get_supported_babel_engines(job_data)
|
469
476
|
result = []
|
470
477
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -513,7 +520,7 @@ The result of the block is returned to the caller.</p>
|
|
513
520
|
<div class="method-source-code"
|
514
521
|
id="make_call-source">
|
515
522
|
<pre>
|
516
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
523
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 70</span>
|
517
524
|
def make_call(time, retry_on_except)
|
518
525
|
begin
|
519
526
|
<span class="ruby-constant">Timeout</span>::timeout(time) {
|
@@ -569,7 +576,7 @@ parameter.</p>
|
|
569
576
|
<div class="method-source-code"
|
570
577
|
id="put_input-source">
|
571
578
|
<pre>
|
572
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
579
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 116</span>
|
573
580
|
def put_input(job_data)
|
574
581
|
result = <span class="ruby-string">""</span>
|
575
582
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -613,7 +620,7 @@ that of start_neptune_job.</p>
|
|
613
620
|
<div class="method-source-code"
|
614
621
|
id="set_acl-source">
|
615
622
|
<pre>
|
616
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
623
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 166</span>
|
617
624
|
def set_acl(job_data)
|
618
625
|
result = <span class="ruby-string">""</span>
|
619
626
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -660,7 +667,7 @@ AppController.</p>
|
|
660
667
|
<div class="method-source-code"
|
661
668
|
id="start_neptune_job-source">
|
662
669
|
<pre>
|
663
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
670
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 101</span>
|
664
671
|
def start_neptune_job(job_data)
|
665
672
|
result = <span class="ruby-string">""</span>
|
666
673
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
data/doc/BabelHelper.html
CHANGED
@@ -181,7 +181,7 @@ just re-converts it - how can we remove it?</p>
|
|
181
181
|
<div class="method-source-code"
|
182
182
|
id="convert_from_neptune_params-source">
|
183
183
|
<pre>
|
184
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
184
|
+
<span class="ruby-comment"># File lib/babel.rb, line 211</span>
|
185
185
|
def self.convert_from_neptune_params(params)
|
186
186
|
job_data = {}
|
187
187
|
params.each { |k, v|
|
@@ -221,7 +221,7 @@ converted them to the form @name, this function reverses that conversion.</p>
|
|
221
221
|
<div class="method-source-code"
|
222
222
|
id="convert_to_neptune_params-source">
|
223
223
|
<pre>
|
224
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
224
|
+
<span class="ruby-comment"># File lib/babel.rb, line 223</span>
|
225
225
|
def self.convert_to_neptune_params(job_data)
|
226
226
|
neptune_params = {}
|
227
227
|
|
@@ -264,11 +264,11 @@ with the given name.</p>
|
|
264
264
|
<div class="method-source-code"
|
265
265
|
id="ensure_output_does_not_exist-source">
|
266
266
|
<pre>
|
267
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
267
|
+
<span class="ruby-comment"># File lib/babel.rb, line 142</span>
|
268
268
|
def self.ensure_output_does_not_exist(job_data)
|
269
269
|
file = job_data[<span class="ruby-string">"@output"</span>]
|
270
270
|
controller = self.get_appcontroller(job_data)
|
271
|
-
puts job_data.inspect
|
271
|
+
<span class="ruby-comment"># Kernel.puts job_data.inspect</span>
|
272
272
|
<span class="ruby-constant">NeptuneHelper</span>.require_file_to_not_exist(file, job_data, controller)
|
273
273
|
end</pre>
|
274
274
|
</div>
|
@@ -304,9 +304,9 @@ code).</p>
|
|
304
304
|
<div class="method-source-code"
|
305
305
|
id="generate_output_location-source">
|
306
306
|
<pre>
|
307
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
307
|
+
<span class="ruby-comment"># File lib/babel.rb, line 86</span>
|
308
308
|
def self.generate_output_location(job_data)
|
309
|
-
if job_data[<span class="ruby-string">"@
|
309
|
+
if job_data[<span class="ruby-string">"@storage"</span>]
|
310
310
|
<span class="ruby-comment"># We already know the bucket name - the same one that the user</span>
|
311
311
|
<span class="ruby-comment"># has told us their code is located in.</span>
|
312
312
|
prefix = job_data[<span class="ruby-string">"@code"</span>].scan(<span class="ruby-regexp">/\/(.*?)\//</span>)[0].to_s
|
@@ -347,7 +347,7 @@ the given job data.</p>
|
|
347
347
|
<div class="method-source-code"
|
348
348
|
id="get_appcontroller-source">
|
349
349
|
<pre>
|
350
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
350
|
+
<span class="ruby-comment"># File lib/babel.rb, line 151</span>
|
351
351
|
def self.get_appcontroller(job_data)
|
352
352
|
keyname = job_data[<span class="ruby-string">"@keyname"</span>] || <span class="ruby-string">"appscale"</span>
|
353
353
|
shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
|
@@ -385,7 +385,7 @@ be used for Neptune jobs where the code is stored locally.</p>
|
|
385
385
|
<div class="method-source-code"
|
386
386
|
id="get_bucket_for_local_data-source">
|
387
387
|
<pre>
|
388
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
388
|
+
<span class="ruby-comment"># File lib/babel.rb, line 101</span>
|
389
389
|
def self.get_bucket_for_local_data(job_data)
|
390
390
|
bucket_name = job_data[<span class="ruby-string">"@bucket_name"</span>] || <span class="ruby-constant">ENV</span>[<span class="ruby-string">'BABEL_BUCKET_NAME'</span>]
|
391
391
|
|
@@ -432,7 +432,7 @@ used) in the remote datastore.</p>
|
|
432
432
|
<div class="method-source-code"
|
433
433
|
id="put_code-source">
|
434
434
|
<pre>
|
435
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
435
|
+
<span class="ruby-comment"># File lib/babel.rb, line 162</span>
|
436
436
|
def self.put_code(job_data)
|
437
437
|
code_dir = <span class="ruby-constant">File</span>.dirname(job_data[<span class="ruby-string">"@code"</span>])
|
438
438
|
code = <span class="ruby-constant">File</span>.basename(job_data[<span class="ruby-string">"@code"</span>])
|
@@ -471,7 +471,7 @@ Neptune ‘input’ job to store the data remotely.</p>
|
|
471
471
|
<div class="method-source-code"
|
472
472
|
id="put_file-source">
|
473
473
|
<pre>
|
474
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
474
|
+
<span class="ruby-comment"># File lib/babel.rb, line 193</span>
|
475
475
|
def self.put_file(local_path, job_data)
|
476
476
|
input_data = self.convert_to_neptune_params(job_data)
|
477
477
|
input_data[:type] = <span class="ruby-string">"input"</span>
|
@@ -517,7 +517,7 @@ remote location of these files.</p>
|
|
517
517
|
<div class="method-source-code"
|
518
518
|
id="put_inputs-source">
|
519
519
|
<pre>
|
520
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
520
|
+
<span class="ruby-comment"># File lib/babel.rb, line 175</span>
|
521
521
|
def self.put_inputs(job_data)
|
522
522
|
if job_data[<span class="ruby-string">"@argv"</span>].nil? or job_data[<span class="ruby-string">"@argv"</span>].empty?
|
523
523
|
return job_data
|
@@ -563,10 +563,14 @@ queue) from the given parameters.</p>
|
|
563
563
|
<div class="method-source-code"
|
564
564
|
id="run_job-source">
|
565
565
|
<pre>
|
566
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
566
|
+
<span class="ruby-comment"># File lib/babel.rb, line 237</span>
|
567
567
|
def self.run_job(job_data)
|
568
568
|
run_data = self.convert_to_neptune_params(job_data)
|
569
|
-
|
569
|
+
|
570
|
+
<span class="ruby-comment"># Default to babel as the job type, if the user doesn't specify one.</span>
|
571
|
+
if run_data[:type].nil? or run_data[:type].empty?
|
572
|
+
run_data[:type] = <span class="ruby-string">"babel"</span>
|
573
|
+
end
|
570
574
|
|
571
575
|
<span class="ruby-comment"># TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to</span>
|
572
576
|
<span class="ruby-comment"># exec tasks over it, instead of locally.</span>
|
@@ -609,7 +613,7 @@ them for computation.</p>
|
|
609
613
|
<div class="method-source-code"
|
610
614
|
id="validate_inputs-source">
|
611
615
|
<pre>
|
612
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
616
|
+
<span class="ruby-comment"># File lib/babel.rb, line 120</span>
|
613
617
|
def self.validate_inputs(job_data)
|
614
618
|
controller = self.get_appcontroller(job_data)
|
615
619
|
|
@@ -659,7 +663,7 @@ output of the job.</p>
|
|
659
663
|
<div class="method-source-code"
|
660
664
|
id="wait_and_get_output-source">
|
661
665
|
<pre>
|
662
|
-
<span class="ruby-comment"># File lib/babel.rb, line
|
666
|
+
<span class="ruby-comment"># File lib/babel.rb, line 259</span>
|
663
667
|
def self.wait_and_get_output(job_data)
|
664
668
|
output_data = self.convert_to_neptune_params(job_data)
|
665
669
|
output_data[:type] = <span class="ruby-string">"output"</span>
|
data/doc/CommonFunctions.html
CHANGED
@@ -175,7 +175,7 @@ instead.</p>
|
|
175
175
|
<div class="method-source-code"
|
176
176
|
id="get_from_yaml-source">
|
177
177
|
<pre>
|
178
|
-
<span class="ruby-comment"># File lib/common_functions.rb, line
|
178
|
+
<span class="ruby-comment"># File lib/common_functions.rb, line 108</span>
|
179
179
|
def self.get_from_yaml(keyname, tag, required=true)
|
180
180
|
location_file = <span class="ruby-constant">File</span>.expand_path("~/.appscale/locations-#{keyname}.yaml")
|
181
181
|
|
@@ -236,14 +236,14 @@ user requests.</p>
|
|
236
236
|
<div class="method-source-code"
|
237
237
|
id="get_random_alphanumeric-source">
|
238
238
|
<pre>
|
239
|
-
<span class="ruby-comment"># File lib/common_functions.rb, line
|
239
|
+
<span class="ruby-comment"># File lib/common_functions.rb, line 33</span>
|
240
240
|
def self.get_random_alphanumeric(length=10)
|
241
241
|
random = <span class="ruby-string">""</span>
|
242
242
|
possible = <span class="ruby-string">"0123456789abcdefghijklmnopqrstuvxwyzABCDEFGHIJKLMNOPQRSTUVWXYZ"</span>
|
243
243
|
possibleLength = possible.length
|
244
244
|
|
245
245
|
length.times { |index|
|
246
|
-
random << possible[rand(possibleLength)]
|
246
|
+
random << possible[<span class="ruby-constant">Kernel</span>.rand(possibleLength)]
|
247
247
|
}
|
248
248
|
|
249
249
|
return random
|
@@ -281,7 +281,7 @@ function, as the secret is stored in a YAML file.</p>
|
|
281
281
|
<div class="method-source-code"
|
282
282
|
id="get_secret_key-source">
|
283
283
|
<pre>
|
284
|
-
<span class="ruby-comment"># File lib/common_functions.rb, line
|
284
|
+
<span class="ruby-comment"># File lib/common_functions.rb, line 142</span>
|
285
285
|
def self.get_secret_key(keyname, required=true)
|
286
286
|
return <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :secret, required)
|
287
287
|
end</pre>
|
@@ -321,12 +321,11 @@ a directory, we append the -r flag to scp as well.</p>
|
|
321
321
|
<div class="method-source-code"
|
322
322
|
id="scp_file-source">
|
323
323
|
<pre>
|
324
|
-
<span class="ruby-comment"># File lib/common_functions.rb, line
|
324
|
+
<span class="ruby-comment"># File lib/common_functions.rb, line 66</span>
|
325
325
|
def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc,
|
326
326
|
is_dir=false)
|
327
|
-
cmd = <span class="ruby-string">""</span>
|
328
|
-
local_file_loc = <span class="ruby-constant">File</span>.expand_path(local_file_loc)
|
329
327
|
|
328
|
+
local_file_loc = <span class="ruby-constant">File</span>.expand_path(local_file_loc)
|
330
329
|
ssh_args = <span class="ruby-string">"-o StrictHostkeyChecking=no 2>&1"</span>
|
331
330
|
ssh_args << <span class="ruby-string">" -r "</span> if is_dir
|
332
331
|
|
@@ -347,7 +346,7 @@ def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc,
|
|
347
346
|
|
348
347
|
loop {
|
349
348
|
break if <span class="ruby-constant">File</span>.exists?(retval_loc)
|
350
|
-
sleep(5)
|
349
|
+
<span class="ruby-constant">Kernel</span>.sleep(5)
|
351
350
|
}
|
352
351
|
|
353
352
|
retval = (<span class="ruby-constant">File</span>.open(retval_loc) { |f| f.read }).chomp
|
@@ -390,15 +389,12 @@ by the Neptune job given, but defaults to ”appscale” if not provided.</p>
|
|
390
389
|
<div class="method-source-code"
|
391
390
|
id="scp_to_shadow-source">
|
392
391
|
<pre>
|
393
|
-
<span class="ruby-comment"># File lib/common_functions.rb, line
|
394
|
-
def self.scp_to_shadow(local_file_loc,
|
395
|
-
remote_file_loc,
|
396
|
-
keyname,
|
397
|
-
is_dir=false)
|
398
|
-
|
392
|
+
<span class="ruby-comment"># File lib/common_functions.rb, line 52</span>
|
393
|
+
def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname, is_dir=false)
|
399
394
|
shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
|
400
395
|
ssh_key = <span class="ruby-constant">File</span>.expand_path("~/.appscale/#{keyname}.key")
|
401
|
-
<span class="ruby-constant">CommonFunctions</span>.scp_file(local_file_loc, remote_file_loc, shadow_ip,
|
396
|
+
<span class="ruby-constant">CommonFunctions</span>.scp_file(local_file_loc, remote_file_loc, shadow_ip,
|
397
|
+
ssh_key, is_dir)
|
402
398
|
end</pre>
|
403
399
|
</div>
|
404
400
|
|
@@ -432,7 +428,7 @@ method).</p>
|
|
432
428
|
<div class="method-source-code"
|
433
429
|
id="shell-source">
|
434
430
|
<pre>
|
435
|
-
<span class="ruby-comment"># File lib/common_functions.rb, line
|
431
|
+
<span class="ruby-comment"># File lib/common_functions.rb, line 26</span>
|
436
432
|
def self.shell(cmd)
|
437
433
|
return `#{cmd}`
|
438
434
|
end</pre>
|
data/doc/NeptuneHelper.html
CHANGED
@@ -152,6 +152,10 @@
|
|
152
152
|
|
153
153
|
<div id="description">
|
154
154
|
|
155
|
+
<p><a href="NeptuneHelper.html">NeptuneHelper</a> provides methods that are
|
156
|
+
used by neptune() and babel() to validate parameters and run the user’s
|
157
|
+
job.</p>
|
158
|
+
|
155
159
|
</div>
|
156
160
|
|
157
161
|
<!-- Constants -->
|
@@ -189,7 +193,7 @@ indicates whether or not the compilation was successful.</p>
|
|
189
193
|
<div class="method-source-code"
|
190
194
|
id="compile_code-source">
|
191
195
|
<pre>
|
192
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
196
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 424</span>
|
193
197
|
def self.compile_code(job_data, ssh_args, shadow_ip)
|
194
198
|
compiled_location = controller.compile_code(job_data)
|
195
199
|
copy_to = job_data[<span class="ruby-string">"@copy_to"</span>]
|
@@ -198,7 +202,7 @@ def self.compile_code(job_data, ssh_args, shadow_ip)
|
|
198
202
|
<span class="ruby-constant">FileUtils</span>.rm_rf(copy_to)
|
199
203
|
|
200
204
|
scp_command = "scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2>&1"
|
201
|
-
<span class="ruby-
|
205
|
+
<span class="ruby-comment"># Kernel.puts scp_command</span>
|
202
206
|
<span class="ruby-constant">CommonFunctions</span>.shell(scp_command)
|
203
207
|
|
204
208
|
code = job_data[<span class="ruby-string">"@code"</span>]
|
@@ -207,7 +211,7 @@ def self.compile_code(job_data, ssh_args, shadow_ip)
|
|
207
211
|
|
208
212
|
[remote_dir, compiled_location].each { |remote_files|
|
209
213
|
ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_files}' 2>&1"
|
210
|
-
<span class="ruby-
|
214
|
+
<span class="ruby-comment"># Kernel.puts ssh_command</span>
|
211
215
|
<span class="ruby-constant">CommonFunctions</span>.shell(ssh_command)
|
212
216
|
}
|
213
217
|
|
@@ -245,7 +249,7 @@ method to use based on the type of the job that the user has asked to run.</p>
|
|
245
249
|
<div class="method-source-code"
|
246
250
|
id="do_preprocessing-source">
|
247
251
|
<pre>
|
248
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
252
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 87</span>
|
249
253
|
def self.do_preprocessing(job_data, controller)
|
250
254
|
job_type = job_data[<span class="ruby-string">"@type"</span>]
|
251
255
|
if !<span class="ruby-constant">NEED_PREPROCESSING</span>.include?(job_type)
|
@@ -289,7 +293,7 @@ job succeeded and if it failed, the reason for it.</p>
|
|
289
293
|
<div class="method-source-code"
|
290
294
|
id="get_input-source">
|
291
295
|
<pre>
|
292
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
296
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 372</span>
|
293
297
|
def self.get_input(job_data, ssh_args, shadow_ip, controller)
|
294
298
|
result = {:result => :success}
|
295
299
|
|
@@ -304,11 +308,11 @@ def self.get_input(job_data, ssh_args, shadow_ip, controller)
|
|
304
308
|
|
305
309
|
remote = "/tmp/neptune-input-#{rand(100000)}"
|
306
310
|
scp_cmd = "scp -r #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
|
307
|
-
<span class="ruby-
|
311
|
+
<span class="ruby-comment"># Kernel.puts scp_cmd</span>
|
308
312
|
<span class="ruby-constant">CommonFunctions</span>.shell(scp_cmd)
|
309
313
|
|
310
314
|
job_data[<span class="ruby-string">"@local"</span>] = remote
|
311
|
-
<span class="ruby-
|
315
|
+
<span class="ruby-comment"># Kernel.puts "job data = #{job_data.inspect}"</span>
|
312
316
|
response = controller.put_input(job_data)
|
313
317
|
if response
|
314
318
|
return {:result => :success}
|
@@ -340,14 +344,17 @@ end</pre>
|
|
340
344
|
|
341
345
|
<div class="method-description">
|
342
346
|
|
343
|
-
|
347
|
+
<p>This method takes in a hash in the format that users write neptune/babel
|
348
|
+
jobs in {:a => “b”} and converts it to the legacy format that
|
349
|
+
Neptune used to use {“@a” => “b”}, and is understood by the
|
350
|
+
AppController.</p>
|
344
351
|
|
345
352
|
|
346
353
|
|
347
354
|
<div class="method-source-code"
|
348
355
|
id="get_job_data-source">
|
349
356
|
<pre>
|
350
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
357
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 279</span>
|
351
358
|
def self.get_job_data(params)
|
352
359
|
job_data = {}
|
353
360
|
params.each { |k, v|
|
@@ -427,7 +434,7 @@ not the job completed successfully (success = no errors).</p>
|
|
427
434
|
<div class="method-source-code"
|
428
435
|
id="get_std_out_and_err-source">
|
429
436
|
<pre>
|
430
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
437
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 452</span>
|
431
438
|
def self.get_std_out_and_err(location)
|
432
439
|
result = {}
|
433
440
|
|
@@ -478,7 +485,7 @@ engines can be found by contacting an AppScale node.</p>
|
|
478
485
|
<div class="method-source-code"
|
479
486
|
id="preprocess_babel-source">
|
480
487
|
<pre>
|
481
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
488
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 231</span>
|
482
489
|
def self.preprocess_babel(job_data, controller)
|
483
490
|
self.require_param(<span class="ruby-string">"@code"</span>, job_data)
|
484
491
|
self.require_param(<span class="ruby-string">"@engine"</span>, job_data)
|
@@ -554,7 +561,7 @@ copy over libraries as well.</p>
|
|
554
561
|
<div class="method-source-code"
|
555
562
|
id="preprocess_compile-source">
|
556
563
|
<pre>
|
557
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
564
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 103</span>
|
558
565
|
def self.preprocess_compile(job_data, controller)
|
559
566
|
code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@code"</span>])
|
560
567
|
if !<span class="ruby-constant">File</span>.exists?(code)
|
@@ -568,7 +575,7 @@ def self.preprocess_compile(job_data, controller)
|
|
568
575
|
|
569
576
|
ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}"
|
570
577
|
remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}' 2>&1"
|
571
|
-
<span class="ruby-
|
578
|
+
<span class="ruby-comment"># Kernel.puts remove_dir</span>
|
572
579
|
<span class="ruby-constant">CommonFunctions</span>.shell(remove_dir)
|
573
580
|
<span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(code, dest, keyname, is_dir=true)
|
574
581
|
|
@@ -597,14 +604,15 @@ end</pre>
|
|
597
604
|
|
598
605
|
<div class="method-description">
|
599
606
|
|
600
|
-
|
607
|
+
<p>This preprocessing method makes sure that the user’s Erlang code exists
|
608
|
+
and copies it over to the AppScale Shadow node.</p>
|
601
609
|
|
602
610
|
|
603
611
|
|
604
612
|
<div class="method-source-code"
|
605
613
|
id="preprocess_erlang-source">
|
606
614
|
<pre>
|
607
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
615
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 126</span>
|
608
616
|
def self.preprocess_erlang(job_data, controller)
|
609
617
|
self.require_param(<span class="ruby-string">"@code"</span>, job_data)
|
610
618
|
|
@@ -651,7 +659,7 @@ also verify that this value is at least as many as the number of nodes
|
|
651
659
|
<div class="method-source-code"
|
652
660
|
id="preprocess_mpi-source">
|
653
661
|
<pre>
|
654
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
662
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 145</span>
|
655
663
|
def self.preprocess_mpi(job_data, controller)
|
656
664
|
self.require_param(<span class="ruby-string">"@nodes_to_use"</span>, job_data)
|
657
665
|
self.require_param(<span class="ruby-string">"@procs_to_use"</span>, job_data)
|
@@ -712,7 +720,7 @@ specify, convert it to be :trajectories.</p>
|
|
712
720
|
<div class="method-source-code"
|
713
721
|
id="preprocess_ssa-source">
|
714
722
|
<pre>
|
715
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
723
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 178</span>
|
716
724
|
def self.preprocess_ssa(job_data, controller)
|
717
725
|
if job_data[<span class="ruby-string">"@simulations"</span>] and job_data[<span class="ruby-string">"@trajectories"</span>]
|
718
726
|
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">":simulations and :trajectories "</span> +
|
@@ -750,14 +758,15 @@ end</pre>
|
|
750
758
|
|
751
759
|
<div class="method-description">
|
752
760
|
|
753
|
-
|
761
|
+
<p>This helper method asks the AppController if the named file exists, and if
|
762
|
+
it does not, throws an exception.</p>
|
754
763
|
|
755
764
|
|
756
765
|
|
757
766
|
<div class="method-source-code"
|
758
767
|
id="require_file_to_exist-source">
|
759
768
|
<pre>
|
760
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
769
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 205</span>
|
761
770
|
def self.require_file_to_exist(file, job_data, controller)
|
762
771
|
if controller.does_file_exist?(file, job_data)
|
763
772
|
return
|
@@ -788,14 +797,16 @@ end</pre>
|
|
788
797
|
|
789
798
|
<div class="method-description">
|
790
799
|
|
791
|
-
|
800
|
+
<p>This helper method performs the opposite function of <a
|
801
|
+
href="NeptuneHelper.html#method-c-require_file_to_exist">require_file_to_exist</a>,
|
802
|
+
raising an exception if the named file does exist.</p>
|
792
803
|
|
793
804
|
|
794
805
|
|
795
806
|
<div class="method-source-code"
|
796
807
|
id="require_file_to_not_exist-source">
|
797
808
|
<pre>
|
798
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
809
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 216</span>
|
799
810
|
def self.require_file_to_not_exist(file, job_data, controller)
|
800
811
|
begin
|
801
812
|
self.require_file_to_exist(file, job_data, controller)
|
@@ -828,14 +839,15 @@ end</pre>
|
|
828
839
|
|
829
840
|
<div class="method-description">
|
830
841
|
|
831
|
-
|
842
|
+
<p>This helper method aborts if the given parameter is not present in the job
|
843
|
+
data provided.</p>
|
832
844
|
|
833
845
|
|
834
846
|
|
835
847
|
<div class="method-source-code"
|
836
848
|
id="require_param-source">
|
837
849
|
<pre>
|
838
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
850
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 196</span>
|
839
851
|
def self.require_param(param, job_data)
|
840
852
|
if !job_data[param]
|
841
853
|
raise <span class="ruby-constant">BadConfigurationException</span>.new("#{param} must be specified")
|
@@ -872,7 +884,7 @@ as well as information about the node to send the request to.</p>
|
|
872
884
|
<div class="method-source-code"
|
873
885
|
id="run_job-source">
|
874
886
|
<pre>
|
875
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
887
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 502</span>
|
876
888
|
def self.run_job(job_data, ssh_args, shadow_ip, secret)
|
877
889
|
controller = <span class="ruby-constant">AppControllerClient</span>.new(shadow_ip, secret)
|
878
890
|
|
@@ -929,17 +941,18 @@ end</pre>
|
|
929
941
|
|
930
942
|
<div class="method-description">
|
931
943
|
|
932
|
-
|
944
|
+
<p>This method uploads a Google App Engine application into AppScale, for use
|
945
|
+
with Cicero jobs. It requires the AppScale tools to be installed.</p>
|
933
946
|
|
934
947
|
|
935
948
|
|
936
949
|
<div class="method-source-code"
|
937
950
|
id="upload_app_for_cicero-source">
|
938
951
|
<pre>
|
939
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
952
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 473</span>
|
940
953
|
def self.upload_app_for_cicero(job_data)
|
941
954
|
if !job_data[<span class="ruby-string">"@app"</span>]
|
942
|
-
<span class="ruby-
|
955
|
+
<span class="ruby-comment"># Kernel.puts "No app specified, not uploading..." </span>
|
943
956
|
return
|
944
957
|
end
|
945
958
|
|
@@ -957,10 +970,10 @@ def self.upload_app_for_cicero(job_data)
|
|
957
970
|
upload_app = <span class="ruby-string">"appscale-upload-app"</span>
|
958
971
|
end
|
959
972
|
|
960
|
-
<span class="ruby-
|
973
|
+
<span class="ruby-comment"># Kernel.puts "Uploading AppEngine app at #{app_location}"</span>
|
961
974
|
upload_command = "#{upload_app} --file #{app_location} --test --keyname #{keyname}"
|
962
|
-
<span class="ruby-
|
963
|
-
<span class="ruby-
|
975
|
+
<span class="ruby-comment"># Kernel.puts upload_command</span>
|
976
|
+
<span class="ruby-comment"># Kernel.puts `#{upload_command}`</span>
|
964
977
|
end</pre>
|
965
978
|
</div>
|
966
979
|
|
@@ -985,14 +998,17 @@ end</pre>
|
|
985
998
|
|
986
999
|
<div class="method-description">
|
987
1000
|
|
988
|
-
|
1001
|
+
<p>This method looks through the given job data and makes sure that the
|
1002
|
+
correct parameters are present for the storage mechanism specified. It
|
1003
|
+
throws an exception if there are errors in the job data or if a needed
|
1004
|
+
parameter is missing.</p>
|
989
1005
|
|
990
1006
|
|
991
1007
|
|
992
1008
|
<div class="method-source-code"
|
993
1009
|
id="validate_storage_params-source">
|
994
1010
|
<pre>
|
995
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
1011
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 332</span>
|
996
1012
|
def self.validate_storage_params(job_data)
|
997
1013
|
job_data[<span class="ruby-string">"@storage"</span>] ||= <span class="ruby-string">"appdb"</span>
|
998
1014
|
|
@@ -1013,10 +1029,10 @@ def self.validate_storage_params(job_data)
|
|
1013
1029
|
if storage == <span class="ruby-string">"s3"</span>
|
1014
1030
|
[<span class="ruby-string">"EC2_ACCESS_KEY"</span>, <span class="ruby-string">"EC2_SECRET_KEY"</span>, <span class="ruby-string">"S3_URL"</span>].each { |item|
|
1015
1031
|
if job_data["@#{item}"]
|
1016
|
-
<span class="ruby-
|
1032
|
+
<span class="ruby-comment"># Kernel.puts "Using specified #{item}"</span>
|
1017
1033
|
else
|
1018
1034
|
if <span class="ruby-constant">ENV</span>[item]
|
1019
|
-
<span class="ruby-
|
1035
|
+
<span class="ruby-comment"># Kernel.puts "Using #{item} from environment"</span>
|
1020
1036
|
job_data["@#{item}"] = <span class="ruby-constant">ENV</span>[item]
|
1021
1037
|
else
|
1022
1038
|
raise <span class="ruby-constant">BadConfigurationException</span>.new("When storing data to S3, #{item} must be specified or be in " +
|
@@ -1060,17 +1076,17 @@ location.</p>
|
|
1060
1076
|
<div class="method-source-code"
|
1061
1077
|
id="wait_for_compilation_to_finish-source">
|
1062
1078
|
<pre>
|
1063
|
-
<span class="ruby-comment"># File lib/neptune.rb, line
|
1079
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 403</span>
|
1064
1080
|
def self.wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
|
1065
1081
|
loop {
|
1066
1082
|
ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2>&1"
|
1067
|
-
<span class="ruby-
|
1083
|
+
<span class="ruby-comment"># Kernel.puts ssh_command</span>
|
1068
1084
|
ssh_result = <span class="ruby-constant">CommonFunctions</span>.shell(ssh_command)
|
1069
|
-
<span class="ruby-
|
1085
|
+
<span class="ruby-comment"># Kernel.puts "result was [#{ssh_result}]"</span>
|
1070
1086
|
if ssh_result =~ <span class="ruby-regexp">/No such file or directory/</span>
|
1071
|
-
<span class="ruby-
|
1087
|
+
<span class="ruby-comment"># Kernel.puts "Still waiting for code to be compiled..."</span>
|
1072
1088
|
else
|
1073
|
-
<span class="ruby-
|
1089
|
+
<span class="ruby-comment"># Kernel.puts "compilation complete! Copying compiled code to #{copy_to}"</span>
|
1074
1090
|
return
|
1075
1091
|
end
|
1076
1092
|
sleep(5)
|