neptune 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +4 -0
- data/bin/neptune +7 -0
- data/doc/AppControllerClient.html +113 -22
- data/doc/{Kernel.html → AppControllerException.html} +23 -55
- data/doc/BabelHelper.html +707 -0
- data/doc/BadConfigurationException.html +142 -0
- data/doc/CommonFunctions.html +121 -33
- data/doc/FileNotFoundException.html +142 -0
- data/doc/NeptuneHelper.html +1102 -0
- data/doc/Object.html +94 -749
- data/doc/bin/neptune.html +3 -1
- data/doc/created.rid +7 -5
- data/doc/index.html +73 -25
- data/doc/lib/app_controller_client_rb.html +1 -1
- data/doc/lib/babel_rb.html +68 -0
- data/doc/lib/common_functions_rb.html +3 -1
- data/doc/lib/custom_exceptions_rb.html +54 -0
- data/doc/lib/neptune_rb.html +3 -1
- data/lib/app_controller_client.rb +28 -10
- data/lib/babel.rb +260 -0
- data/lib/common_functions.rb +42 -28
- data/lib/custom_exceptions.rb +10 -0
- data/lib/neptune.rb +371 -304
- data/test/unit/test_app_controller_client.rb +9 -9
- data/test/unit/test_babel.rb +154 -0
- data/test/unit/test_common_functions.rb +39 -75
- data/test/unit/test_neptune.rb +168 -76
- data/test/unit/ts_all.rb +5 -0
- metadata +32 -8
data/README
CHANGED
@@ -67,6 +67,10 @@ http://www.neptune-lang.org/2011/6/Neptune-Picks-up-Best-Paper-at-ScienceCloud-2
|
|
67
67
|
|
68
68
|
Version History:
|
69
69
|
|
70
|
+
February 11, 2012 - 0.2.0 released, adding support for Babel jobs (code that
|
71
|
+
runs over multiple clouds, using their queues automatically) and the babel()
|
72
|
+
method, to automatically upload code, run it, and retrieve the output.
|
73
|
+
|
70
74
|
December 6, 2011 - 0.1.4 released, allowing MPI jobs (and by extension,
|
71
75
|
UPC/X10/KDT jobs) to specify command-line arguments via :argv that should be
|
72
76
|
passed to the executable.
|
data/bin/neptune
CHANGED
@@ -3,8 +3,15 @@
|
|
3
3
|
$VERBOSE = nil # to surpress excessive SSL cert warnings
|
4
4
|
|
5
5
|
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
6
|
+
require 'babel'
|
6
7
|
require 'neptune'
|
7
8
|
|
9
|
+
|
10
|
+
# If the user gives us only one argument, then we assume it's the location of a
|
11
|
+
# Neptune script and execute it accordingly. In all other conditions, we assume
|
12
|
+
# the user is trying to generate a Neptune script on-the-fly, and (if an even
|
13
|
+
# number of params are given) we generate a hash from the params and exec it
|
14
|
+
# as a Neptune job.
|
8
15
|
if ARGV.length == 1
|
9
16
|
file = ARGV[0]
|
10
17
|
unless File.exists?(file)
|
@@ -73,10 +73,14 @@
|
|
73
73
|
|
74
74
|
<li><a href="#method-i-compile_code">#compile_code</a></li>
|
75
75
|
|
76
|
+
<li><a href="#method-i-does_file_exist-3F">#does_file_exist?</a></li>
|
77
|
+
|
76
78
|
<li><a href="#method-i-get_acl">#get_acl</a></li>
|
77
79
|
|
78
80
|
<li><a href="#method-i-get_output">#get_output</a></li>
|
79
81
|
|
82
|
+
<li><a href="#method-i-get_supported_babel_engines">#get_supported_babel_engines</a></li>
|
83
|
+
|
80
84
|
<li><a href="#method-i-make_call">#make_call</a></li>
|
81
85
|
|
82
86
|
<li><a href="#method-i-put_input">#put_input</a></li>
|
@@ -114,9 +118,17 @@
|
|
114
118
|
|
115
119
|
<li><a href="./AppControllerClient.html">AppControllerClient</a></li>
|
116
120
|
|
121
|
+
<li><a href="./AppControllerException.html">AppControllerException</a></li>
|
122
|
+
|
123
|
+
<li><a href="./BabelHelper.html">BabelHelper</a></li>
|
124
|
+
|
125
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
|
126
|
+
|
117
127
|
<li><a href="./CommonFunctions.html">CommonFunctions</a></li>
|
118
128
|
|
119
|
-
<li><a href="./
|
129
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
|
130
|
+
|
131
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
|
120
132
|
|
121
133
|
<li><a href="./Object.html">Object</a></li>
|
122
134
|
|
@@ -245,6 +257,8 @@ def initialize(ip, secret)
|
|
245
257
|
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_get_acl"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
246
258
|
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_set_acl"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
247
259
|
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_compile_code"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
260
|
+
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_get_supported_babel_engines"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
261
|
+
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_does_file_exist"</span>, <span class="ruby-string">"file"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
248
262
|
end</pre>
|
249
263
|
</div>
|
250
264
|
|
@@ -282,13 +296,51 @@ end</pre>
|
|
282
296
|
<div class="method-source-code"
|
283
297
|
id="compile_code-source">
|
284
298
|
<pre>
|
285
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
299
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 154</span>
|
286
300
|
def compile_code(job_data)
|
287
301
|
result = <span class="ruby-string">""</span>
|
288
302
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
289
303
|
result = conn.neptune_compile_code(job_data, <span class="ruby-ivar">@secret</span>)
|
290
304
|
}
|
291
|
-
|
305
|
+
raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
|
306
|
+
return result
|
307
|
+
end</pre>
|
308
|
+
</div>
|
309
|
+
|
310
|
+
</div>
|
311
|
+
|
312
|
+
|
313
|
+
|
314
|
+
|
315
|
+
</div>
|
316
|
+
|
317
|
+
|
318
|
+
<div id="does_file_exist-3F-method" class="method-detail ">
|
319
|
+
<a name="method-i-does_file_exist-3F"></a>
|
320
|
+
|
321
|
+
|
322
|
+
<div class="method-heading">
|
323
|
+
<span class="method-name">does_file_exist?</span><span
|
324
|
+
class="method-args">(file, job_data)</span>
|
325
|
+
<span class="method-click-advice">click to toggle source</span>
|
326
|
+
</div>
|
327
|
+
|
328
|
+
|
329
|
+
<div class="method-description">
|
330
|
+
|
331
|
+
|
332
|
+
|
333
|
+
|
334
|
+
|
335
|
+
<div class="method-source-code"
|
336
|
+
id="does_file_exist-3F-source">
|
337
|
+
<pre>
|
338
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 171</span>
|
339
|
+
def does_file_exist?(file, job_data)
|
340
|
+
result = false
|
341
|
+
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
342
|
+
result = conn.neptune_does_file_exist(file, job_data, <span class="ruby-ivar">@secret</span>)
|
343
|
+
}
|
292
344
|
return result
|
293
345
|
end</pre>
|
294
346
|
</div>
|
@@ -324,13 +376,13 @@ exceptions mirror that of start_neptune_job.</p>
|
|
324
376
|
<div class="method-source-code"
|
325
377
|
id="get_acl-source">
|
326
378
|
<pre>
|
327
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
379
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 131</span>
|
328
380
|
def get_acl(job_data)
|
329
381
|
result = <span class="ruby-string">""</span>
|
330
382
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
331
383
|
result = conn.neptune_get_acl(job_data, <span class="ruby-ivar">@secret</span>)
|
332
384
|
}
|
333
|
-
|
385
|
+
raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
|
334
386
|
return result
|
335
387
|
end</pre>
|
336
388
|
</div>
|
@@ -365,21 +417,59 @@ string to the caller, but as this may be inefficient for non-trivial output
|
|
365
417
|
jobs, the next version of Neptune will add an additional call to directly
|
366
418
|
copy the output to a file on the local filesystem. See <a
|
367
419
|
href="AppControllerClient.html#method-i-start_neptune_job">start_neptune_job</a>
|
368
|
-
for conditions by which this method can
|
369
|
-
used for job_data.</p>
|
420
|
+
for conditions by which this method can raise AppControllerException.new as
|
421
|
+
well as the input format used for job_data.</p>
|
370
422
|
|
371
423
|
|
372
424
|
|
373
425
|
<div class="method-source-code"
|
374
426
|
id="get_output-source">
|
375
427
|
<pre>
|
376
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
428
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 117</span>
|
377
429
|
def get_output(job_data)
|
378
430
|
result = <span class="ruby-string">""</span>
|
379
431
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
380
432
|
result = conn.neptune_get_output(job_data, <span class="ruby-ivar">@secret</span>)
|
381
433
|
}
|
382
|
-
|
434
|
+
raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
|
435
|
+
return result
|
436
|
+
end</pre>
|
437
|
+
</div>
|
438
|
+
|
439
|
+
</div>
|
440
|
+
|
441
|
+
|
442
|
+
|
443
|
+
|
444
|
+
</div>
|
445
|
+
|
446
|
+
|
447
|
+
<div id="get_supported_babel_engines-method" class="method-detail ">
|
448
|
+
<a name="method-i-get_supported_babel_engines"></a>
|
449
|
+
|
450
|
+
|
451
|
+
<div class="method-heading">
|
452
|
+
<span class="method-name">get_supported_babel_engines</span><span
|
453
|
+
class="method-args">(job_data)</span>
|
454
|
+
<span class="method-click-advice">click to toggle source</span>
|
455
|
+
</div>
|
456
|
+
|
457
|
+
|
458
|
+
<div class="method-description">
|
459
|
+
|
460
|
+
|
461
|
+
|
462
|
+
|
463
|
+
|
464
|
+
<div class="method-source-code"
|
465
|
+
id="get_supported_babel_engines-source">
|
466
|
+
<pre>
|
467
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 163</span>
|
468
|
+
def get_supported_babel_engines(job_data)
|
469
|
+
result = []
|
470
|
+
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
471
|
+
result = conn.neptune_get_supported_babel_engines(job_data, <span class="ruby-ivar">@secret</span>)
|
472
|
+
}
|
383
473
|
return result
|
384
474
|
end</pre>
|
385
475
|
</div>
|
@@ -423,7 +513,7 @@ The result of the block is returned to the caller.</p>
|
|
423
513
|
<div class="method-source-code"
|
424
514
|
id="make_call-source">
|
425
515
|
<pre>
|
426
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
516
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 54</span>
|
427
517
|
def make_call(time, retry_on_except)
|
428
518
|
begin
|
429
519
|
<span class="ruby-constant">Timeout</span>::timeout(time) {
|
@@ -433,7 +523,7 @@ def make_call(time, retry_on_except)
|
|
433
523
|
if retry_on_except
|
434
524
|
retry
|
435
525
|
else
|
436
|
-
|
526
|
+
raise <span class="ruby-constant">AppControllerException</span>.new(<span class="ruby-string">"Connection was refused. Is the AppController running?"</span>)
|
437
527
|
end
|
438
528
|
rescue <span class="ruby-constant">OpenSSL</span>::<span class="ruby-constant">SSL</span>::<span class="ruby-constant">SSLError</span>, <span class="ruby-constant">NotImplementedError</span>, <span class="ruby-constant">Timeout</span>::<span class="ruby-constant">Error</span>
|
439
529
|
retry
|
@@ -441,7 +531,7 @@ def make_call(time, retry_on_except)
|
|
441
531
|
if retry_on_except
|
442
532
|
retry
|
443
533
|
else
|
444
|
-
|
534
|
+
raise <span class="ruby-constant">AppControllerException</span>.new("We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.")
|
445
535
|
end
|
446
536
|
end
|
447
537
|
end</pre>
|
@@ -479,13 +569,13 @@ parameter.</p>
|
|
479
569
|
<div class="method-source-code"
|
480
570
|
id="put_input-source">
|
481
571
|
<pre>
|
482
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
572
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 98</span>
|
483
573
|
def put_input(job_data)
|
484
574
|
result = <span class="ruby-string">""</span>
|
485
575
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
486
576
|
result = conn.neptune_put_input(job_data, <span class="ruby-ivar">@secret</span>)
|
487
577
|
}
|
488
|
-
|
578
|
+
raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
|
489
579
|
return result
|
490
580
|
end</pre>
|
491
581
|
</div>
|
@@ -523,13 +613,13 @@ that of start_neptune_job.</p>
|
|
523
613
|
<div class="method-source-code"
|
524
614
|
id="set_acl-source">
|
525
615
|
<pre>
|
526
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
616
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 145</span>
|
527
617
|
def set_acl(job_data)
|
528
618
|
result = <span class="ruby-string">""</span>
|
529
619
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
530
620
|
result = conn.neptune_set_acl(job_data, <span class="ruby-ivar">@secret</span>)
|
531
621
|
}
|
532
|
-
|
622
|
+
raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
|
533
623
|
return result
|
534
624
|
end</pre>
|
535
625
|
</div>
|
@@ -559,23 +649,24 @@ end</pre>
|
|
559
649
|
MapReduce), or a scaling job (e.g., for AppScale itself). This method
|
560
650
|
should not be used for retrieving the output of a job or getting / setting
|
561
651
|
output ACLs, but just for starting new HPC / scaling jobs. This method
|
562
|
-
takes a hash containing the parameters of the job to run, and can
|
563
|
-
the AppController it calls returns an error
|
564
|
-
or the machine isn’t running). Otherwise,
|
565
|
-
is the result returned from the
|
652
|
+
takes a hash containing the parameters of the job to run, and can raise
|
653
|
+
AppControllerException.new if the AppController it calls returns an error
|
654
|
+
(e.g., if a bad secret is used or the machine isn’t running). Otherwise,
|
655
|
+
the return value of this method is the result returned from the
|
656
|
+
AppController.</p>
|
566
657
|
|
567
658
|
|
568
659
|
|
569
660
|
<div class="method-source-code"
|
570
661
|
id="start_neptune_job-source">
|
571
662
|
<pre>
|
572
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
663
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 84</span>
|
573
664
|
def start_neptune_job(job_data)
|
574
665
|
result = <span class="ruby-string">""</span>
|
575
666
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
576
667
|
result = conn.neptune_start_job(job_data, <span class="ruby-ivar">@secret</span>)
|
577
668
|
}
|
578
|
-
|
669
|
+
raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
|
579
670
|
return result
|
580
671
|
end</pre>
|
581
672
|
</div>
|
@@ -5,7 +5,7 @@
|
|
5
5
|
<head>
|
6
6
|
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
|
7
7
|
|
8
|
-
<title>
|
8
|
+
<title>Class: AppControllerException</title>
|
9
9
|
|
10
10
|
<link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
|
11
11
|
|
@@ -19,7 +19,7 @@
|
|
19
19
|
charset="utf-8"></script>
|
20
20
|
|
21
21
|
</head>
|
22
|
-
<body class="
|
22
|
+
<body class="class">
|
23
23
|
|
24
24
|
<div id="metadata">
|
25
25
|
<div id="home-metadata">
|
@@ -38,8 +38,8 @@
|
|
38
38
|
<div class="section-body">
|
39
39
|
<ul>
|
40
40
|
|
41
|
-
<li><a href="./lib/
|
42
|
-
class="thickbox" title="lib/
|
41
|
+
<li><a href="./lib/custom_exceptions_rb.html?TB_iframe=true&height=550&width=785"
|
42
|
+
class="thickbox" title="lib/custom_exceptions.rb">lib/custom_exceptions.rb</a></li>
|
43
43
|
|
44
44
|
</ul>
|
45
45
|
</div>
|
@@ -52,21 +52,19 @@
|
|
52
52
|
|
53
53
|
<!-- Parent Class -->
|
54
54
|
|
55
|
+
<div id="parent-class-section" class="section">
|
56
|
+
<h3 class="section-header">Parent</h3>
|
57
|
+
|
58
|
+
<p class="link">Exception</p>
|
59
|
+
|
60
|
+
</div>
|
61
|
+
|
55
62
|
|
56
63
|
<!-- Namespace Contents -->
|
57
64
|
|
58
65
|
|
59
66
|
<!-- Method Quickref -->
|
60
67
|
|
61
|
-
<div id="method-list-section" class="section">
|
62
|
-
<h3 class="section-header">Methods</h3>
|
63
|
-
<ul class="link-list">
|
64
|
-
|
65
|
-
<li><a href="#method-i-shell">#shell</a></li>
|
66
|
-
|
67
|
-
</ul>
|
68
|
-
</div>
|
69
|
-
|
70
68
|
|
71
69
|
<!-- Included Modules -->
|
72
70
|
|
@@ -93,9 +91,17 @@
|
|
93
91
|
|
94
92
|
<li><a href="./AppControllerClient.html">AppControllerClient</a></li>
|
95
93
|
|
94
|
+
<li><a href="./AppControllerException.html">AppControllerException</a></li>
|
95
|
+
|
96
|
+
<li><a href="./BabelHelper.html">BabelHelper</a></li>
|
97
|
+
|
98
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
|
99
|
+
|
96
100
|
<li><a href="./CommonFunctions.html">CommonFunctions</a></li>
|
97
101
|
|
98
|
-
<li><a href="./
|
102
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
|
103
|
+
|
104
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
|
99
105
|
|
100
106
|
<li><a href="./Object.html">Object</a></li>
|
101
107
|
|
@@ -108,10 +114,12 @@
|
|
108
114
|
</div>
|
109
115
|
|
110
116
|
<div id="documentation">
|
111
|
-
<h1 class="
|
117
|
+
<h1 class="class">AppControllerException</h1>
|
112
118
|
|
113
119
|
<div id="description">
|
114
120
|
|
121
|
+
<p>Programmer: Chris Bunch</p>
|
122
|
+
|
115
123
|
</div>
|
116
124
|
|
117
125
|
<!-- Constants -->
|
@@ -122,46 +130,6 @@
|
|
122
130
|
|
123
131
|
<!-- Methods -->
|
124
132
|
|
125
|
-
<div id="public-instance-method-details" class="method-section section">
|
126
|
-
<h3 class="section-header">Public Instance Methods</h3>
|
127
|
-
|
128
|
-
|
129
|
-
<div id="shell-method" class="method-detail ">
|
130
|
-
<a name="method-i-shell"></a>
|
131
|
-
|
132
|
-
|
133
|
-
<div class="method-heading">
|
134
|
-
<span class="method-name">shell</span><span
|
135
|
-
class="method-args">(command)</span>
|
136
|
-
<span class="method-click-advice">click to toggle source</span>
|
137
|
-
</div>
|
138
|
-
|
139
|
-
|
140
|
-
<div class="method-description">
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
<div class="method-source-code"
|
147
|
-
id="shell-source">
|
148
|
-
<pre>
|
149
|
-
<span class="ruby-comment"># File lib/common_functions.rb, line 13</span>
|
150
|
-
def shell(command)
|
151
|
-
return `#{command}`
|
152
|
-
end</pre>
|
153
|
-
</div>
|
154
|
-
|
155
|
-
</div>
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
</div>
|
161
|
-
|
162
|
-
|
163
|
-
</div>
|
164
|
-
|
165
133
|
|
166
134
|
</div>
|
167
135
|
|
@@ -0,0 +1,707 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
3
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
4
|
+
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
5
|
+
<head>
|
6
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
|
7
|
+
|
8
|
+
<title>Module: BabelHelper</title>
|
9
|
+
|
10
|
+
<link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
|
11
|
+
|
12
|
+
<script src="./js/jquery.js" type="text/javascript"
|
13
|
+
charset="utf-8"></script>
|
14
|
+
<script src="./js/thickbox-compressed.js" type="text/javascript"
|
15
|
+
charset="utf-8"></script>
|
16
|
+
<script src="./js/quicksearch.js" type="text/javascript"
|
17
|
+
charset="utf-8"></script>
|
18
|
+
<script src="./js/darkfish.js" type="text/javascript"
|
19
|
+
charset="utf-8"></script>
|
20
|
+
|
21
|
+
</head>
|
22
|
+
<body class="module">
|
23
|
+
|
24
|
+
<div id="metadata">
|
25
|
+
<div id="home-metadata">
|
26
|
+
<div id="home-section" class="section">
|
27
|
+
<h3 class="section-header">
|
28
|
+
<a href="./index.html">Home</a>
|
29
|
+
<a href="./index.html#classes">Classes</a>
|
30
|
+
<a href="./index.html#methods">Methods</a>
|
31
|
+
</h3>
|
32
|
+
</div>
|
33
|
+
</div>
|
34
|
+
|
35
|
+
<div id="file-metadata">
|
36
|
+
<div id="file-list-section" class="section">
|
37
|
+
<h3 class="section-header">In Files</h3>
|
38
|
+
<div class="section-body">
|
39
|
+
<ul>
|
40
|
+
|
41
|
+
<li><a href="./lib/babel_rb.html?TB_iframe=true&height=550&width=785"
|
42
|
+
class="thickbox" title="lib/babel.rb">lib/babel.rb</a></li>
|
43
|
+
|
44
|
+
</ul>
|
45
|
+
</div>
|
46
|
+
</div>
|
47
|
+
|
48
|
+
|
49
|
+
</div>
|
50
|
+
|
51
|
+
<div id="class-metadata">
|
52
|
+
|
53
|
+
<!-- Parent Class -->
|
54
|
+
|
55
|
+
|
56
|
+
<!-- Namespace Contents -->
|
57
|
+
|
58
|
+
|
59
|
+
<!-- Method Quickref -->
|
60
|
+
|
61
|
+
<div id="method-list-section" class="section">
|
62
|
+
<h3 class="section-header">Methods</h3>
|
63
|
+
<ul class="link-list">
|
64
|
+
|
65
|
+
<li><a href="#method-c-convert_from_neptune_params">::convert_from_neptune_params</a></li>
|
66
|
+
|
67
|
+
<li><a href="#method-c-convert_to_neptune_params">::convert_to_neptune_params</a></li>
|
68
|
+
|
69
|
+
<li><a href="#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist</a></li>
|
70
|
+
|
71
|
+
<li><a href="#method-c-generate_output_location">::generate_output_location</a></li>
|
72
|
+
|
73
|
+
<li><a href="#method-c-get_appcontroller">::get_appcontroller</a></li>
|
74
|
+
|
75
|
+
<li><a href="#method-c-get_bucket_for_local_data">::get_bucket_for_local_data</a></li>
|
76
|
+
|
77
|
+
<li><a href="#method-c-put_code">::put_code</a></li>
|
78
|
+
|
79
|
+
<li><a href="#method-c-put_file">::put_file</a></li>
|
80
|
+
|
81
|
+
<li><a href="#method-c-put_inputs">::put_inputs</a></li>
|
82
|
+
|
83
|
+
<li><a href="#method-c-run_job">::run_job</a></li>
|
84
|
+
|
85
|
+
<li><a href="#method-c-validate_inputs">::validate_inputs</a></li>
|
86
|
+
|
87
|
+
<li><a href="#method-c-wait_and_get_output">::wait_and_get_output</a></li>
|
88
|
+
|
89
|
+
</ul>
|
90
|
+
</div>
|
91
|
+
|
92
|
+
|
93
|
+
<!-- Included Modules -->
|
94
|
+
|
95
|
+
</div>
|
96
|
+
|
97
|
+
<div id="project-metadata">
|
98
|
+
|
99
|
+
|
100
|
+
|
101
|
+
<div id="classindex-section" class="section project-section">
|
102
|
+
<h3 class="section-header">Class/Module Index
|
103
|
+
<span class="search-toggle"><img src="./images/find.png"
|
104
|
+
height="16" width="16" alt="[+]"
|
105
|
+
title="show/hide quicksearch" /></span></h3>
|
106
|
+
<form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
|
107
|
+
<fieldset>
|
108
|
+
<legend>Quicksearch</legend>
|
109
|
+
<input type="text" name="quicksearch" value=""
|
110
|
+
class="quicksearch-field" />
|
111
|
+
</fieldset>
|
112
|
+
</form>
|
113
|
+
|
114
|
+
<ul class="link-list">
|
115
|
+
|
116
|
+
<li><a href="./AppControllerClient.html">AppControllerClient</a></li>
|
117
|
+
|
118
|
+
<li><a href="./AppControllerException.html">AppControllerException</a></li>
|
119
|
+
|
120
|
+
<li><a href="./BabelHelper.html">BabelHelper</a></li>
|
121
|
+
|
122
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
|
123
|
+
|
124
|
+
<li><a href="./CommonFunctions.html">CommonFunctions</a></li>
|
125
|
+
|
126
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
|
127
|
+
|
128
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
|
129
|
+
|
130
|
+
<li><a href="./Object.html">Object</a></li>
|
131
|
+
|
132
|
+
</ul>
|
133
|
+
<div id="no-class-search-results" style="display: none;">No matching classes.</div>
|
134
|
+
</div>
|
135
|
+
|
136
|
+
|
137
|
+
</div>
|
138
|
+
</div>
|
139
|
+
|
140
|
+
<div id="documentation">
|
141
|
+
<h1 class="module">BabelHelper</h1>
|
142
|
+
|
143
|
+
<div id="description">
|
144
|
+
|
145
|
+
<p>This module provides convenience functions for babel().</p>
|
146
|
+
|
147
|
+
</div>
|
148
|
+
|
149
|
+
<!-- Constants -->
|
150
|
+
|
151
|
+
|
152
|
+
<!-- Attributes -->
|
153
|
+
|
154
|
+
|
155
|
+
<!-- Methods -->
|
156
|
+
|
157
|
+
<div id="public-class-method-details" class="method-section section">
|
158
|
+
<h3 class="section-header">Public Class Methods</h3>
|
159
|
+
|
160
|
+
|
161
|
+
<div id="convert_from_neptune_params-method" class="method-detail ">
|
162
|
+
<a name="method-c-convert_from_neptune_params"></a>
|
163
|
+
|
164
|
+
|
165
|
+
<div class="method-heading">
|
166
|
+
<span class="method-name">convert_from_neptune_params</span><span
|
167
|
+
class="method-args">(params)</span>
|
168
|
+
<span class="method-click-advice">click to toggle source</span>
|
169
|
+
</div>
|
170
|
+
|
171
|
+
|
172
|
+
<div class="method-description">
|
173
|
+
|
174
|
+
<p>Neptune internally uses job_data with keys of the form @name, but since the
|
175
|
+
user has given them to us in the form :name, we convert it here. TODO(cgb):
|
176
|
+
It looks like this conversion to/from may be unnecessary since neptune()
|
177
|
+
just re-converts it - how can we remove it?</p>
|
178
|
+
|
179
|
+
|
180
|
+
|
181
|
+
<div class="method-source-code"
|
182
|
+
id="convert_from_neptune_params-source">
|
183
|
+
<pre>
|
184
|
+
<span class="ruby-comment"># File lib/babel.rb, line 198</span>
|
185
|
+
def self.convert_from_neptune_params(params)
|
186
|
+
job_data = {}
|
187
|
+
params.each { |k, v|
|
188
|
+
key = "@#{k}"
|
189
|
+
job_data[key] = v
|
190
|
+
}
|
191
|
+
return job_data
|
192
|
+
end</pre>
|
193
|
+
</div>
|
194
|
+
|
195
|
+
</div>
|
196
|
+
|
197
|
+
|
198
|
+
|
199
|
+
|
200
|
+
</div>
|
201
|
+
|
202
|
+
|
203
|
+
<div id="convert_to_neptune_params-method" class="method-detail ">
|
204
|
+
<a name="method-c-convert_to_neptune_params"></a>
|
205
|
+
|
206
|
+
|
207
|
+
<div class="method-heading">
|
208
|
+
<span class="method-name">convert_to_neptune_params</span><span
|
209
|
+
class="method-args">(job_data)</span>
|
210
|
+
<span class="method-click-advice">click to toggle source</span>
|
211
|
+
</div>
|
212
|
+
|
213
|
+
|
214
|
+
<div class="method-description">
|
215
|
+
|
216
|
+
<p>Neptune input jobs expect keys of the form :name, but since we’ve already
|
217
|
+
converted them to the form @name, this function reverses that conversion.</p>
|
218
|
+
|
219
|
+
|
220
|
+
|
221
|
+
<div class="method-source-code"
|
222
|
+
id="convert_to_neptune_params-source">
|
223
|
+
<pre>
|
224
|
+
<span class="ruby-comment"># File lib/babel.rb, line 209</span>
|
225
|
+
def self.convert_to_neptune_params(job_data)
|
226
|
+
neptune_params = {}
|
227
|
+
|
228
|
+
job_data.each { |k, v|
|
229
|
+
key = k.delete(<span class="ruby-string">"@"</span>).to_sym
|
230
|
+
neptune_params[key] = v
|
231
|
+
}
|
232
|
+
|
233
|
+
return neptune_params
|
234
|
+
end</pre>
|
235
|
+
</div>
|
236
|
+
|
237
|
+
</div>
|
238
|
+
|
239
|
+
|
240
|
+
|
241
|
+
|
242
|
+
</div>
|
243
|
+
|
244
|
+
|
245
|
+
<div id="ensure_output_does_not_exist-method" class="method-detail ">
|
246
|
+
<a name="method-c-ensure_output_does_not_exist"></a>
|
247
|
+
|
248
|
+
|
249
|
+
<div class="method-heading">
|
250
|
+
<span class="method-name">ensure_output_does_not_exist</span><span
|
251
|
+
class="method-args">(job_data)</span>
|
252
|
+
<span class="method-click-advice">click to toggle source</span>
|
253
|
+
</div>
|
254
|
+
|
255
|
+
|
256
|
+
<div class="method-description">
|
257
|
+
|
258
|
+
<p>To avoid accidentally overwriting outputs from previous jobs, we first
|
259
|
+
check to make sure an output file doesn’t exist before starting a new job
|
260
|
+
with the given name.</p>
|
261
|
+
|
262
|
+
|
263
|
+
|
264
|
+
<div class="method-source-code"
|
265
|
+
id="ensure_output_does_not_exist-source">
|
266
|
+
<pre>
|
267
|
+
<span class="ruby-comment"># File lib/babel.rb, line 134</span>
|
268
|
+
def self.ensure_output_does_not_exist(job_data)
|
269
|
+
file = job_data[<span class="ruby-string">"@output"</span>]
|
270
|
+
controller = self.get_appcontroller(job_data)
|
271
|
+
puts job_data.inspect
|
272
|
+
<span class="ruby-constant">NeptuneHelper</span>.require_file_to_not_exist(file, job_data, controller)
|
273
|
+
end</pre>
|
274
|
+
</div>
|
275
|
+
|
276
|
+
</div>
|
277
|
+
|
278
|
+
|
279
|
+
|
280
|
+
|
281
|
+
</div>
|
282
|
+
|
283
|
+
|
284
|
+
<div id="generate_output_location-method" class="method-detail ">
|
285
|
+
<a name="method-c-generate_output_location"></a>
|
286
|
+
|
287
|
+
|
288
|
+
<div class="method-heading">
|
289
|
+
<span class="method-name">generate_output_location</span><span
|
290
|
+
class="method-args">(job_data)</span>
|
291
|
+
<span class="method-click-advice">click to toggle source</span>
|
292
|
+
</div>
|
293
|
+
|
294
|
+
|
295
|
+
<div class="method-description">
|
296
|
+
|
297
|
+
<p>If the user fails to give us an output location, this function will
|
298
|
+
generate one for them, based on either the location of their code (for
|
299
|
+
remotely specified code), or a babel parameter (for locally specified
|
300
|
+
code).</p>
|
301
|
+
|
302
|
+
|
303
|
+
|
304
|
+
<div class="method-source-code"
|
305
|
+
id="generate_output_location-source">
|
306
|
+
<pre>
|
307
|
+
<span class="ruby-comment"># File lib/babel.rb, line 81</span>
|
308
|
+
def self.generate_output_location(job_data)
|
309
|
+
if job_data[<span class="ruby-string">"@is_remote"</span>]
|
310
|
+
<span class="ruby-comment"># We already know the bucket name - the same one that the user</span>
|
311
|
+
<span class="ruby-comment"># has told us their code is located in.</span>
|
312
|
+
prefix = job_data[<span class="ruby-string">"@code"</span>].scan(<span class="ruby-regexp">/\/(.*?)\//</span>)[0].to_s
|
313
|
+
else
|
314
|
+
prefix = self.get_bucket_for_local_data(job_data)
|
315
|
+
end
|
316
|
+
|
317
|
+
return "/#{prefix}/babel/temp-#{CommonFunctions.get_random_alphanumeric()}"
|
318
|
+
end</pre>
|
319
|
+
</div>
|
320
|
+
|
321
|
+
</div>
|
322
|
+
|
323
|
+
|
324
|
+
|
325
|
+
|
326
|
+
</div>
|
327
|
+
|
328
|
+
|
329
|
+
<div id="get_appcontroller-method" class="method-detail ">
|
330
|
+
<a name="method-c-get_appcontroller"></a>
|
331
|
+
|
332
|
+
|
333
|
+
<div class="method-heading">
|
334
|
+
<span class="method-name">get_appcontroller</span><span
|
335
|
+
class="method-args">(job_data)</span>
|
336
|
+
<span class="method-click-advice">click to toggle source</span>
|
337
|
+
</div>
|
338
|
+
|
339
|
+
|
340
|
+
<div class="method-description">
|
341
|
+
|
342
|
+
<p>Returns an <a href="AppControllerClient.html">AppControllerClient</a> for
|
343
|
+
the given job data.</p>
|
344
|
+
|
345
|
+
|
346
|
+
|
347
|
+
<div class="method-source-code"
|
348
|
+
id="get_appcontroller-source">
|
349
|
+
<pre>
|
350
|
+
<span class="ruby-comment"># File lib/babel.rb, line 142</span>
|
351
|
+
def self.get_appcontroller(job_data)
|
352
|
+
keyname = job_data[<span class="ruby-string">"@keyname"</span>] || <span class="ruby-string">"appscale"</span>
|
353
|
+
shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
|
354
|
+
secret = <span class="ruby-constant">CommonFunctions</span>.get_secret_key(keyname)
|
355
|
+
return <span class="ruby-constant">AppControllerClient</span>.new(shadow_ip, secret)
|
356
|
+
end</pre>
|
357
|
+
</div>
|
358
|
+
|
359
|
+
</div>
|
360
|
+
|
361
|
+
|
362
|
+
|
363
|
+
|
364
|
+
</div>
|
365
|
+
|
366
|
+
|
367
|
+
<div id="get_bucket_for_local_data-method" class="method-detail ">
|
368
|
+
<a name="method-c-get_bucket_for_local_data"></a>
|
369
|
+
|
370
|
+
|
371
|
+
<div class="method-heading">
|
372
|
+
<span class="method-name">get_bucket_for_local_data</span><span
|
373
|
+
class="method-args">(job_data)</span>
|
374
|
+
<span class="method-click-advice">click to toggle source</span>
|
375
|
+
</div>
|
376
|
+
|
377
|
+
|
378
|
+
<div class="method-description">
|
379
|
+
|
380
|
+
<p>Provides a common way for callers to get the name of the bucket that should
|
381
|
+
be used for Neptune jobs where the code is stored locally.</p>
|
382
|
+
|
383
|
+
|
384
|
+
|
385
|
+
<div class="method-source-code"
|
386
|
+
id="get_bucket_for_local_data-source">
|
387
|
+
<pre>
|
388
|
+
<span class="ruby-comment"># File lib/babel.rb, line 95</span>
|
389
|
+
def self.get_bucket_for_local_data(job_data)
|
390
|
+
bucket_name = job_data[<span class="ruby-string">"@bucket_name"</span>] || <span class="ruby-constant">ENV</span>[<span class="ruby-string">'BABEL_BUCKET_NAME'</span>]
|
391
|
+
|
392
|
+
if bucket_name.nil?
|
393
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-constant">NEEDS_BUCKET_INFO</span>)
|
394
|
+
end
|
395
|
+
|
396
|
+
<span class="ruby-comment"># If the bucket name starts with a slash, remove it</span>
|
397
|
+
if bucket_name[0].chr == <span class="ruby-string">"/"</span>
|
398
|
+
bucket_name = bucket_name[1, bucket_name.length]
|
399
|
+
end
|
400
|
+
|
401
|
+
return bucket_name
|
402
|
+
end</pre>
|
403
|
+
</div>
|
404
|
+
|
405
|
+
</div>
|
406
|
+
|
407
|
+
|
408
|
+
|
409
|
+
|
410
|
+
</div>
|
411
|
+
|
412
|
+
|
413
|
+
<div id="put_code-method" class="method-detail ">
|
414
|
+
<a name="method-c-put_code"></a>
|
415
|
+
|
416
|
+
|
417
|
+
<div class="method-heading">
|
418
|
+
<span class="method-name">put_code</span><span
|
419
|
+
class="method-args">(job_data)</span>
|
420
|
+
<span class="method-click-advice">click to toggle source</span>
|
421
|
+
</div>
|
422
|
+
|
423
|
+
|
424
|
+
<div class="method-description">
|
425
|
+
|
426
|
+
<p>Stores the user’s code (and the directory it’s in, and directories in
|
427
|
+
the same directory as the user’s code, since there could be libraries
|
428
|
+
used) in the remote datastore.</p>
|
429
|
+
|
430
|
+
|
431
|
+
|
432
|
+
<div class="method-source-code"
|
433
|
+
id="put_code-source">
|
434
|
+
<pre>
|
435
|
+
<span class="ruby-comment"># File lib/babel.rb, line 152</span>
|
436
|
+
def self.put_code(job_data)
|
437
|
+
code_dir = <span class="ruby-constant">File</span>.dirname(job_data[<span class="ruby-string">"@code"</span>])
|
438
|
+
code = <span class="ruby-constant">File</span>.basename(job_data[<span class="ruby-string">"@code"</span>])
|
439
|
+
remote_code_dir = self.put_file(code_dir, job_data)
|
440
|
+
job_data[<span class="ruby-string">"@code"</span>] = remote_code_dir + <span class="ruby-string">"/"</span> + code
|
441
|
+
return job_data[<span class="ruby-string">"@code"</span>]
|
442
|
+
end</pre>
|
443
|
+
</div>
|
444
|
+
|
445
|
+
</div>
|
446
|
+
|
447
|
+
|
448
|
+
|
449
|
+
|
450
|
+
</div>
|
451
|
+
|
452
|
+
|
453
|
+
<div id="put_file-method" class="method-detail ">
|
454
|
+
<a name="method-c-put_file"></a>
|
455
|
+
|
456
|
+
|
457
|
+
<div class="method-heading">
|
458
|
+
<span class="method-name">put_file</span><span
|
459
|
+
class="method-args">(local_path, job_data)</span>
|
460
|
+
<span class="method-click-advice">click to toggle source</span>
|
461
|
+
</div>
|
462
|
+
|
463
|
+
|
464
|
+
<div class="method-description">
|
465
|
+
|
466
|
+
<p>If the user gives us local code or local inputs, this function will run a
|
467
|
+
Neptune ‘input’ job to store the data remotely.</p>
|
468
|
+
|
469
|
+
|
470
|
+
|
471
|
+
<div class="method-source-code"
|
472
|
+
id="put_file-source">
|
473
|
+
<pre>
|
474
|
+
<span class="ruby-comment"># File lib/babel.rb, line 181</span>
|
475
|
+
def self.put_file(local_path, job_data)
|
476
|
+
input_data = self.convert_to_neptune_params(job_data)
|
477
|
+
input_data[:type] = <span class="ruby-string">"input"</span>
|
478
|
+
input_data[:local] = local_path
|
479
|
+
|
480
|
+
bucket_name = self.get_bucket_for_local_data(job_data)
|
481
|
+
input_data[:remote] = "/#{bucket_name}/babel#{local_path}"
|
482
|
+
|
483
|
+
<span class="ruby-constant">Kernel</span>.neptune(input_data)
|
484
|
+
|
485
|
+
return input_data[:remote]
|
486
|
+
end</pre>
|
487
|
+
</div>
|
488
|
+
|
489
|
+
</div>
|
490
|
+
|
491
|
+
|
492
|
+
|
493
|
+
|
494
|
+
</div>
|
495
|
+
|
496
|
+
|
497
|
+
<div id="put_inputs-method" class="method-detail ">
|
498
|
+
<a name="method-c-put_inputs"></a>
|
499
|
+
|
500
|
+
|
501
|
+
<div class="method-heading">
|
502
|
+
<span class="method-name">put_inputs</span><span
|
503
|
+
class="method-args">(job_data)</span>
|
504
|
+
<span class="method-click-advice">click to toggle source</span>
|
505
|
+
</div>
|
506
|
+
|
507
|
+
|
508
|
+
<div class="method-description">
|
509
|
+
|
510
|
+
<p>If any input files are specified, they are copied to the remote datastore
|
511
|
+
via Neptune ‘input’ jobs. Inputs are assumed to be files on the local
|
512
|
+
filesystem if they begin with a slash, and job_data gets updated with the
|
513
|
+
remote location of these files.</p>
|
514
|
+
|
515
|
+
|
516
|
+
|
517
|
+
<div class="method-source-code"
|
518
|
+
id="put_inputs-source">
|
519
|
+
<pre>
|
520
|
+
<span class="ruby-comment"># File lib/babel.rb, line 164</span>
|
521
|
+
def self.put_inputs(job_data)
|
522
|
+
if job_data[<span class="ruby-string">"@argv"</span>].nil? or job_data[<span class="ruby-string">"@argv"</span>].empty?
|
523
|
+
return job_data
|
524
|
+
end
|
525
|
+
|
526
|
+
job_data[<span class="ruby-string">"@argv"</span>].each_index { |i|
|
527
|
+
arg = job_data[<span class="ruby-string">"@argv"</span>][i]
|
528
|
+
if arg[0].chr == <span class="ruby-string">"/"</span>
|
529
|
+
job_data[<span class="ruby-string">"@argv"</span>][i] = self.put_file(arg, job_data)
|
530
|
+
end
|
531
|
+
}
|
532
|
+
|
533
|
+
return job_data
|
534
|
+
end</pre>
|
535
|
+
</div>
|
536
|
+
|
537
|
+
</div>
|
538
|
+
|
539
|
+
|
540
|
+
|
541
|
+
|
542
|
+
</div>
|
543
|
+
|
544
|
+
|
545
|
+
<div id="run_job-method" class="method-detail ">
|
546
|
+
<a name="method-c-run_job"></a>
|
547
|
+
|
548
|
+
|
549
|
+
<div class="method-heading">
|
550
|
+
<span class="method-name">run_job</span><span
|
551
|
+
class="method-args">(job_data)</span>
|
552
|
+
<span class="method-click-advice">click to toggle source</span>
|
553
|
+
</div>
|
554
|
+
|
555
|
+
|
556
|
+
<div class="method-description">
|
557
|
+
|
558
|
+
<p>Constructs a Neptune job to run the user’s code as a Babel job (task
|
559
|
+
queue) from the given parameters.</p>
|
560
|
+
|
561
|
+
|
562
|
+
|
563
|
+
<div class="method-source-code"
|
564
|
+
id="run_job-source">
|
565
|
+
<pre>
|
566
|
+
<span class="ruby-comment"># File lib/babel.rb, line 222</span>
|
567
|
+
def self.run_job(job_data)
|
568
|
+
run_data = self.convert_to_neptune_params(job_data)
|
569
|
+
run_data[:type] = <span class="ruby-string">"babel"</span>
|
570
|
+
|
571
|
+
<span class="ruby-comment"># TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to</span>
|
572
|
+
<span class="ruby-comment"># exec tasks over it, instead of locally.</span>
|
573
|
+
if job_data[<span class="ruby-string">"@run_local"</span>].nil?
|
574
|
+
run_data[:run_local] = true
|
575
|
+
run_data[:engine] = <span class="ruby-string">"executor-sqs"</span>
|
576
|
+
end
|
577
|
+
|
578
|
+
return <span class="ruby-constant">Kernel</span>.neptune(run_data)
|
579
|
+
end</pre>
|
580
|
+
</div>
|
581
|
+
|
582
|
+
</div>
|
583
|
+
|
584
|
+
|
585
|
+
|
586
|
+
|
587
|
+
</div>
|
588
|
+
|
589
|
+
|
590
|
+
<div id="validate_inputs-method" class="method-detail ">
|
591
|
+
<a name="method-c-validate_inputs"></a>
|
592
|
+
|
593
|
+
|
594
|
+
<div class="method-heading">
|
595
|
+
<span class="method-name">validate_inputs</span><span
|
596
|
+
class="method-args">(job_data)</span>
|
597
|
+
<span class="method-click-advice">click to toggle source</span>
|
598
|
+
</div>
|
599
|
+
|
600
|
+
|
601
|
+
<div class="method-description">
|
602
|
+
|
603
|
+
<p>For jobs where the code is stored remotely, this method ensures that the
|
604
|
+
code and any possible inputs actually do exist, before attempting to use
|
605
|
+
them for computation.</p>
|
606
|
+
|
607
|
+
|
608
|
+
|
609
|
+
<div class="method-source-code"
|
610
|
+
id="validate_inputs-source">
|
611
|
+
<pre>
|
612
|
+
<span class="ruby-comment"># File lib/babel.rb, line 113</span>
|
613
|
+
def self.validate_inputs(job_data)
|
614
|
+
controller = self.get_appcontroller(job_data)
|
615
|
+
|
616
|
+
<span class="ruby-comment"># First, make sure the code exists</span>
|
617
|
+
<span class="ruby-constant">NeptuneHelper</span>.require_file_to_exist(job_data[<span class="ruby-string">"@code"</span>], job_data, controller)
|
618
|
+
|
619
|
+
if job_data[<span class="ruby-string">"@argv"</span>].nil? or job_data[<span class="ruby-string">"@argv"</span>].empty?
|
620
|
+
return
|
621
|
+
end
|
622
|
+
|
623
|
+
<span class="ruby-comment"># We assume anything that begins with a slash is a remote file</span>
|
624
|
+
job_data[<span class="ruby-string">"@argv"</span>].each { |arg|
|
625
|
+
if arg[0].chr == <span class="ruby-string">"/"</span>
|
626
|
+
<span class="ruby-constant">NeptuneHelper</span>.require_file_to_exist(arg, job_data, controller)
|
627
|
+
end
|
628
|
+
}
|
629
|
+
end</pre>
|
630
|
+
</div>
|
631
|
+
|
632
|
+
</div>
|
633
|
+
|
634
|
+
|
635
|
+
|
636
|
+
|
637
|
+
</div>
|
638
|
+
|
639
|
+
|
640
|
+
<div id="wait_and_get_output-method" class="method-detail ">
|
641
|
+
<a name="method-c-wait_and_get_output"></a>
|
642
|
+
|
643
|
+
|
644
|
+
<div class="method-heading">
|
645
|
+
<span class="method-name">wait_and_get_output</span><span
|
646
|
+
class="method-args">(job_data)</span>
|
647
|
+
<span class="method-click-advice">click to toggle source</span>
|
648
|
+
</div>
|
649
|
+
|
650
|
+
|
651
|
+
<div class="method-description">
|
652
|
+
|
653
|
+
<p>Constructs a Neptune job to get the output of a Babel job. If the job is
|
654
|
+
not yet finished, this function waits until it does, and then returns the
|
655
|
+
output of the job.</p>
|
656
|
+
|
657
|
+
|
658
|
+
|
659
|
+
<div class="method-source-code"
|
660
|
+
id="wait_and_get_output-source">
|
661
|
+
<pre>
|
662
|
+
<span class="ruby-comment"># File lib/babel.rb, line 239</span>
|
663
|
+
def self.wait_and_get_output(job_data)
|
664
|
+
output_data = self.convert_to_neptune_params(job_data)
|
665
|
+
output_data[:type] = <span class="ruby-string">"output"</span>
|
666
|
+
|
667
|
+
output = <span class="ruby-string">""</span>
|
668
|
+
time_to_sleep = <span class="ruby-constant">SLEEP_TIME</span>
|
669
|
+
loop {
|
670
|
+
output = <span class="ruby-constant">Kernel</span>.neptune(output_data)[:output]
|
671
|
+
if output == <span class="ruby-constant">DOES_NOT_EXIST</span>
|
672
|
+
<span class="ruby-comment"># Exponentially back off, up to a limit of MAX_SLEEP_TIME</span>
|
673
|
+
<span class="ruby-constant">Kernel</span>.sleep(time_to_sleep)
|
674
|
+
if time_to_sleep < <span class="ruby-constant">MAX_SLEEP_TIME</span>
|
675
|
+
time_to_sleep *= 2
|
676
|
+
end
|
677
|
+
else
|
678
|
+
break
|
679
|
+
end
|
680
|
+
}
|
681
|
+
|
682
|
+
return output
|
683
|
+
end</pre>
|
684
|
+
</div>
|
685
|
+
|
686
|
+
</div>
|
687
|
+
|
688
|
+
|
689
|
+
|
690
|
+
|
691
|
+
</div>
|
692
|
+
|
693
|
+
|
694
|
+
</div>
|
695
|
+
|
696
|
+
|
697
|
+
</div>
|
698
|
+
|
699
|
+
<div id="validator-badges">
|
700
|
+
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
701
|
+
<p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
|
702
|
+
Rdoc Generator</a> 2</small>.</p>
|
703
|
+
</div>
|
704
|
+
|
705
|
+
</body>
|
706
|
+
</html>
|
707
|
+
|