neptune 0.1.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README CHANGED
@@ -67,6 +67,10 @@ http://www.neptune-lang.org/2011/6/Neptune-Picks-up-Best-Paper-at-ScienceCloud-2
67
67
 
68
68
  Version History:
69
69
 
70
+ February 11, 2012 - 0.2.0 released, adding support for Babel jobs (code that
71
+ runs over multiple clouds, using their queues automatically) and the babel()
72
+ method, to automatically upload code, run it, and retrieve the output.
73
+
70
74
  December 6, 2011 - 0.1.4 released, allowing MPI jobs (and by extension,
71
75
  UPC/X10/KDT jobs) to specify command-line arguments via :argv that should be
72
76
  passed to the executable.
@@ -3,8 +3,15 @@
3
3
  $VERBOSE = nil # to surpress excessive SSL cert warnings
4
4
 
5
5
  $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
6
+ require 'babel'
6
7
  require 'neptune'
7
8
 
9
+
10
+ # If the user gives us only one argument, then we assume it's the location of a
11
+ # Neptune script and execute it accordingly. In all other conditions, we assume
12
+ # the user is trying to generate a Neptune script on-the-fly, and (if an even
13
+ # number of params are given) we generate a hash from the params and exec it
14
+ # as a Neptune job.
8
15
  if ARGV.length == 1
9
16
  file = ARGV[0]
10
17
  unless File.exists?(file)
@@ -73,10 +73,14 @@
73
73
 
74
74
  <li><a href="#method-i-compile_code">#compile_code</a></li>
75
75
 
76
+ <li><a href="#method-i-does_file_exist-3F">#does_file_exist?</a></li>
77
+
76
78
  <li><a href="#method-i-get_acl">#get_acl</a></li>
77
79
 
78
80
  <li><a href="#method-i-get_output">#get_output</a></li>
79
81
 
82
+ <li><a href="#method-i-get_supported_babel_engines">#get_supported_babel_engines</a></li>
83
+
80
84
  <li><a href="#method-i-make_call">#make_call</a></li>
81
85
 
82
86
  <li><a href="#method-i-put_input">#put_input</a></li>
@@ -114,9 +118,17 @@
114
118
 
115
119
  <li><a href="./AppControllerClient.html">AppControllerClient</a></li>
116
120
 
121
+ <li><a href="./AppControllerException.html">AppControllerException</a></li>
122
+
123
+ <li><a href="./BabelHelper.html">BabelHelper</a></li>
124
+
125
+ <li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
126
+
117
127
  <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
118
128
 
119
- <li><a href="./Kernel.html">Kernel</a></li>
129
+ <li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
130
+
131
+ <li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
120
132
 
121
133
  <li><a href="./Object.html">Object</a></li>
122
134
 
@@ -245,6 +257,8 @@ def initialize(ip, secret)
245
257
  <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_get_acl&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
246
258
  <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_set_acl&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
247
259
  <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_compile_code&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
260
+ <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_get_supported_babel_engines&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
261
+ <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_does_file_exist&quot;</span>, <span class="ruby-string">&quot;file&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
248
262
  end</pre>
249
263
  </div>
250
264
 
@@ -282,13 +296,51 @@ end</pre>
282
296
  <div class="method-source-code"
283
297
  id="compile_code-source">
284
298
  <pre>
285
- <span class="ruby-comment"># File lib/app_controller_client.rb, line 152</span>
299
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 154</span>
286
300
  def compile_code(job_data)
287
301
  result = <span class="ruby-string">&quot;&quot;</span>
288
302
  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
289
303
  result = conn.neptune_compile_code(job_data, <span class="ruby-ivar">@secret</span>)
290
304
  }
291
- abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
305
+ raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
306
+ return result
307
+ end</pre>
308
+ </div>
309
+
310
+ </div>
311
+
312
+
313
+
314
+
315
+ </div>
316
+
317
+
318
+ <div id="does_file_exist-3F-method" class="method-detail ">
319
+ <a name="method-i-does_file_exist-3F"></a>
320
+
321
+
322
+ <div class="method-heading">
323
+ <span class="method-name">does_file_exist?</span><span
324
+ class="method-args">(file, job_data)</span>
325
+ <span class="method-click-advice">click to toggle source</span>
326
+ </div>
327
+
328
+
329
+ <div class="method-description">
330
+
331
+
332
+
333
+
334
+
335
+ <div class="method-source-code"
336
+ id="does_file_exist-3F-source">
337
+ <pre>
338
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 171</span>
339
+ def does_file_exist?(file, job_data)
340
+ result = false
341
+ make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
342
+ result = conn.neptune_does_file_exist(file, job_data, <span class="ruby-ivar">@secret</span>)
343
+ }
292
344
  return result
293
345
  end</pre>
294
346
  </div>
@@ -324,13 +376,13 @@ exceptions mirror that of start_neptune_job.</p>
324
376
  <div class="method-source-code"
325
377
  id="get_acl-source">
326
378
  <pre>
327
- <span class="ruby-comment"># File lib/app_controller_client.rb, line 129</span>
379
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 131</span>
328
380
  def get_acl(job_data)
329
381
  result = <span class="ruby-string">&quot;&quot;</span>
330
382
  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
331
383
  result = conn.neptune_get_acl(job_data, <span class="ruby-ivar">@secret</span>)
332
384
  }
333
- abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
385
+ raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
334
386
  return result
335
387
  end</pre>
336
388
  </div>
@@ -365,21 +417,59 @@ string to the caller, but as this may be inefficient for non-trivial output
365
417
  jobs, the next version of Neptune will add an additional call to directly
366
418
  copy the output to a file on the local filesystem. See <a
367
419
  href="AppControllerClient.html#method-i-start_neptune_job">start_neptune_job</a>
368
- for conditions by which this method can abort as well as the input format
369
- used for job_data.</p>
420
+ for conditions by which this method can raise AppControllerException.new as
421
+ well as the input format used for job_data.</p>
370
422
 
371
423
 
372
424
 
373
425
  <div class="method-source-code"
374
426
  id="get_output-source">
375
427
  <pre>
376
- <span class="ruby-comment"># File lib/app_controller_client.rb, line 115</span>
428
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 117</span>
377
429
  def get_output(job_data)
378
430
  result = <span class="ruby-string">&quot;&quot;</span>
379
431
  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
380
432
  result = conn.neptune_get_output(job_data, <span class="ruby-ivar">@secret</span>)
381
433
  }
382
- abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
434
+ raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
435
+ return result
436
+ end</pre>
437
+ </div>
438
+
439
+ </div>
440
+
441
+
442
+
443
+
444
+ </div>
445
+
446
+
447
+ <div id="get_supported_babel_engines-method" class="method-detail ">
448
+ <a name="method-i-get_supported_babel_engines"></a>
449
+
450
+
451
+ <div class="method-heading">
452
+ <span class="method-name">get_supported_babel_engines</span><span
453
+ class="method-args">(job_data)</span>
454
+ <span class="method-click-advice">click to toggle source</span>
455
+ </div>
456
+
457
+
458
+ <div class="method-description">
459
+
460
+
461
+
462
+
463
+
464
+ <div class="method-source-code"
465
+ id="get_supported_babel_engines-source">
466
+ <pre>
467
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 163</span>
468
+ def get_supported_babel_engines(job_data)
469
+ result = []
470
+ make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
471
+ result = conn.neptune_get_supported_babel_engines(job_data, <span class="ruby-ivar">@secret</span>)
472
+ }
383
473
  return result
384
474
  end</pre>
385
475
  </div>
@@ -423,7 +513,7 @@ The result of the block is returned to the caller.</p>
423
513
  <div class="method-source-code"
424
514
  id="make_call-source">
425
515
  <pre>
426
- <span class="ruby-comment"># File lib/app_controller_client.rb, line 52</span>
516
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 54</span>
427
517
  def make_call(time, retry_on_except)
428
518
  begin
429
519
  <span class="ruby-constant">Timeout</span>::timeout(time) {
@@ -433,7 +523,7 @@ def make_call(time, retry_on_except)
433
523
  if retry_on_except
434
524
  retry
435
525
  else
436
- abort(<span class="ruby-string">&quot;Connection was refused. Is the AppController running?&quot;</span>)
526
+ raise <span class="ruby-constant">AppControllerException</span>.new(<span class="ruby-string">&quot;Connection was refused. Is the AppController running?&quot;</span>)
437
527
  end
438
528
  rescue <span class="ruby-constant">OpenSSL</span>::<span class="ruby-constant">SSL</span>::<span class="ruby-constant">SSLError</span>, <span class="ruby-constant">NotImplementedError</span>, <span class="ruby-constant">Timeout</span>::<span class="ruby-constant">Error</span>
439
529
  retry
@@ -441,7 +531,7 @@ def make_call(time, retry_on_except)
441
531
  if retry_on_except
442
532
  retry
443
533
  else
444
- abort(&quot;We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.&quot;)
534
+ raise <span class="ruby-constant">AppControllerException</span>.new(&quot;We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.&quot;)
445
535
  end
446
536
  end
447
537
  end</pre>
@@ -479,13 +569,13 @@ parameter.</p>
479
569
  <div class="method-source-code"
480
570
  id="put_input-source">
481
571
  <pre>
482
- <span class="ruby-comment"># File lib/app_controller_client.rb, line 96</span>
572
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 98</span>
483
573
  def put_input(job_data)
484
574
  result = <span class="ruby-string">&quot;&quot;</span>
485
575
  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
486
576
  result = conn.neptune_put_input(job_data, <span class="ruby-ivar">@secret</span>)
487
577
  }
488
- abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
578
+ raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
489
579
  return result
490
580
  end</pre>
491
581
  </div>
@@ -523,13 +613,13 @@ that of start_neptune_job.</p>
523
613
  <div class="method-source-code"
524
614
  id="set_acl-source">
525
615
  <pre>
526
- <span class="ruby-comment"># File lib/app_controller_client.rb, line 143</span>
616
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 145</span>
527
617
  def set_acl(job_data)
528
618
  result = <span class="ruby-string">&quot;&quot;</span>
529
619
  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
530
620
  result = conn.neptune_set_acl(job_data, <span class="ruby-ivar">@secret</span>)
531
621
  }
532
- abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
622
+ raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
533
623
  return result
534
624
  end</pre>
535
625
  </div>
@@ -559,23 +649,24 @@ end</pre>
559
649
  MapReduce), or a scaling job (e.g., for AppScale itself). This method
560
650
  should not be used for retrieving the output of a job or getting / setting
561
651
  output ACLs, but just for starting new HPC / scaling jobs. This method
562
- takes a hash containing the parameters of the job to run, and can abort if
563
- the AppController it calls returns an error (e.g., if a bad secret is used
564
- or the machine isn’t running). Otherwise, the return value of this method
565
- is the result returned from the AppController.</p>
652
+ takes a hash containing the parameters of the job to run, and can raise
653
+ AppControllerException.new if the AppController it calls returns an error
654
+ (e.g., if a bad secret is used or the machine isn’t running). Otherwise,
655
+ the return value of this method is the result returned from the
656
+ AppController.</p>
566
657
 
567
658
 
568
659
 
569
660
  <div class="method-source-code"
570
661
  id="start_neptune_job-source">
571
662
  <pre>
572
- <span class="ruby-comment"># File lib/app_controller_client.rb, line 82</span>
663
+ <span class="ruby-comment"># File lib/app_controller_client.rb, line 84</span>
573
664
  def start_neptune_job(job_data)
574
665
  result = <span class="ruby-string">&quot;&quot;</span>
575
666
  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
576
667
  result = conn.neptune_start_job(job_data, <span class="ruby-ivar">@secret</span>)
577
668
  }
578
- abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
669
+ raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
579
670
  return result
580
671
  end</pre>
581
672
  </div>
@@ -5,7 +5,7 @@
5
5
  <head>
6
6
  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
7
7
 
8
- <title>Module: Kernel</title>
8
+ <title>Class: AppControllerException</title>
9
9
 
10
10
  <link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
11
11
 
@@ -19,7 +19,7 @@
19
19
  charset="utf-8"></script>
20
20
 
21
21
  </head>
22
- <body class="module">
22
+ <body class="class">
23
23
 
24
24
  <div id="metadata">
25
25
  <div id="home-metadata">
@@ -38,8 +38,8 @@
38
38
  <div class="section-body">
39
39
  <ul>
40
40
 
41
- <li><a href="./lib/common_functions_rb.html?TB_iframe=true&amp;height=550&amp;width=785"
42
- class="thickbox" title="lib/common_functions.rb">lib/common_functions.rb</a></li>
41
+ <li><a href="./lib/custom_exceptions_rb.html?TB_iframe=true&amp;height=550&amp;width=785"
42
+ class="thickbox" title="lib/custom_exceptions.rb">lib/custom_exceptions.rb</a></li>
43
43
 
44
44
  </ul>
45
45
  </div>
@@ -52,21 +52,19 @@
52
52
 
53
53
  <!-- Parent Class -->
54
54
 
55
+ <div id="parent-class-section" class="section">
56
+ <h3 class="section-header">Parent</h3>
57
+
58
+ <p class="link">Exception</p>
59
+
60
+ </div>
61
+
55
62
 
56
63
  <!-- Namespace Contents -->
57
64
 
58
65
 
59
66
  <!-- Method Quickref -->
60
67
 
61
- <div id="method-list-section" class="section">
62
- <h3 class="section-header">Methods</h3>
63
- <ul class="link-list">
64
-
65
- <li><a href="#method-i-shell">#shell</a></li>
66
-
67
- </ul>
68
- </div>
69
-
70
68
 
71
69
  <!-- Included Modules -->
72
70
 
@@ -93,9 +91,17 @@
93
91
 
94
92
  <li><a href="./AppControllerClient.html">AppControllerClient</a></li>
95
93
 
94
+ <li><a href="./AppControllerException.html">AppControllerException</a></li>
95
+
96
+ <li><a href="./BabelHelper.html">BabelHelper</a></li>
97
+
98
+ <li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
99
+
96
100
  <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
97
101
 
98
- <li><a href="./Kernel.html">Kernel</a></li>
102
+ <li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
103
+
104
+ <li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
99
105
 
100
106
  <li><a href="./Object.html">Object</a></li>
101
107
 
@@ -108,10 +114,12 @@
108
114
  </div>
109
115
 
110
116
  <div id="documentation">
111
- <h1 class="module">Kernel</h1>
117
+ <h1 class="class">AppControllerException</h1>
112
118
 
113
119
  <div id="description">
114
120
 
121
+ <p>Programmer: Chris Bunch</p>
122
+
115
123
  </div>
116
124
 
117
125
  <!-- Constants -->
@@ -122,46 +130,6 @@
122
130
 
123
131
  <!-- Methods -->
124
132
 
125
- <div id="public-instance-method-details" class="method-section section">
126
- <h3 class="section-header">Public Instance Methods</h3>
127
-
128
-
129
- <div id="shell-method" class="method-detail ">
130
- <a name="method-i-shell"></a>
131
-
132
-
133
- <div class="method-heading">
134
- <span class="method-name">shell</span><span
135
- class="method-args">(command)</span>
136
- <span class="method-click-advice">click to toggle source</span>
137
- </div>
138
-
139
-
140
- <div class="method-description">
141
-
142
-
143
-
144
-
145
-
146
- <div class="method-source-code"
147
- id="shell-source">
148
- <pre>
149
- <span class="ruby-comment"># File lib/common_functions.rb, line 13</span>
150
- def shell(command)
151
- return `#{command}`
152
- end</pre>
153
- </div>
154
-
155
- </div>
156
-
157
-
158
-
159
-
160
- </div>
161
-
162
-
163
- </div>
164
-
165
133
 
166
134
  </div>
167
135
 
@@ -0,0 +1,707 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
3
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
4
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
5
+ <head>
6
+ <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
7
+
8
+ <title>Module: BabelHelper</title>
9
+
10
+ <link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
11
+
12
+ <script src="./js/jquery.js" type="text/javascript"
13
+ charset="utf-8"></script>
14
+ <script src="./js/thickbox-compressed.js" type="text/javascript"
15
+ charset="utf-8"></script>
16
+ <script src="./js/quicksearch.js" type="text/javascript"
17
+ charset="utf-8"></script>
18
+ <script src="./js/darkfish.js" type="text/javascript"
19
+ charset="utf-8"></script>
20
+
21
+ </head>
22
+ <body class="module">
23
+
24
+ <div id="metadata">
25
+ <div id="home-metadata">
26
+ <div id="home-section" class="section">
27
+ <h3 class="section-header">
28
+ <a href="./index.html">Home</a>
29
+ <a href="./index.html#classes">Classes</a>
30
+ <a href="./index.html#methods">Methods</a>
31
+ </h3>
32
+ </div>
33
+ </div>
34
+
35
+ <div id="file-metadata">
36
+ <div id="file-list-section" class="section">
37
+ <h3 class="section-header">In Files</h3>
38
+ <div class="section-body">
39
+ <ul>
40
+
41
+ <li><a href="./lib/babel_rb.html?TB_iframe=true&amp;height=550&amp;width=785"
42
+ class="thickbox" title="lib/babel.rb">lib/babel.rb</a></li>
43
+
44
+ </ul>
45
+ </div>
46
+ </div>
47
+
48
+
49
+ </div>
50
+
51
+ <div id="class-metadata">
52
+
53
+ <!-- Parent Class -->
54
+
55
+
56
+ <!-- Namespace Contents -->
57
+
58
+
59
+ <!-- Method Quickref -->
60
+
61
+ <div id="method-list-section" class="section">
62
+ <h3 class="section-header">Methods</h3>
63
+ <ul class="link-list">
64
+
65
+ <li><a href="#method-c-convert_from_neptune_params">::convert_from_neptune_params</a></li>
66
+
67
+ <li><a href="#method-c-convert_to_neptune_params">::convert_to_neptune_params</a></li>
68
+
69
+ <li><a href="#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist</a></li>
70
+
71
+ <li><a href="#method-c-generate_output_location">::generate_output_location</a></li>
72
+
73
+ <li><a href="#method-c-get_appcontroller">::get_appcontroller</a></li>
74
+
75
+ <li><a href="#method-c-get_bucket_for_local_data">::get_bucket_for_local_data</a></li>
76
+
77
+ <li><a href="#method-c-put_code">::put_code</a></li>
78
+
79
+ <li><a href="#method-c-put_file">::put_file</a></li>
80
+
81
+ <li><a href="#method-c-put_inputs">::put_inputs</a></li>
82
+
83
+ <li><a href="#method-c-run_job">::run_job</a></li>
84
+
85
+ <li><a href="#method-c-validate_inputs">::validate_inputs</a></li>
86
+
87
+ <li><a href="#method-c-wait_and_get_output">::wait_and_get_output</a></li>
88
+
89
+ </ul>
90
+ </div>
91
+
92
+
93
+ <!-- Included Modules -->
94
+
95
+ </div>
96
+
97
+ <div id="project-metadata">
98
+
99
+
100
+
101
+ <div id="classindex-section" class="section project-section">
102
+ <h3 class="section-header">Class/Module Index
103
+ <span class="search-toggle"><img src="./images/find.png"
104
+ height="16" width="16" alt="[+]"
105
+ title="show/hide quicksearch" /></span></h3>
106
+ <form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
107
+ <fieldset>
108
+ <legend>Quicksearch</legend>
109
+ <input type="text" name="quicksearch" value=""
110
+ class="quicksearch-field" />
111
+ </fieldset>
112
+ </form>
113
+
114
+ <ul class="link-list">
115
+
116
+ <li><a href="./AppControllerClient.html">AppControllerClient</a></li>
117
+
118
+ <li><a href="./AppControllerException.html">AppControllerException</a></li>
119
+
120
+ <li><a href="./BabelHelper.html">BabelHelper</a></li>
121
+
122
+ <li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
123
+
124
+ <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
125
+
126
+ <li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
127
+
128
+ <li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
129
+
130
+ <li><a href="./Object.html">Object</a></li>
131
+
132
+ </ul>
133
+ <div id="no-class-search-results" style="display: none;">No matching classes.</div>
134
+ </div>
135
+
136
+
137
+ </div>
138
+ </div>
139
+
140
+ <div id="documentation">
141
+ <h1 class="module">BabelHelper</h1>
142
+
143
+ <div id="description">
144
+
145
+ <p>This module provides convenience functions for babel().</p>
146
+
147
+ </div>
148
+
149
+ <!-- Constants -->
150
+
151
+
152
+ <!-- Attributes -->
153
+
154
+
155
+ <!-- Methods -->
156
+
157
+ <div id="public-class-method-details" class="method-section section">
158
+ <h3 class="section-header">Public Class Methods</h3>
159
+
160
+
161
+ <div id="convert_from_neptune_params-method" class="method-detail ">
162
+ <a name="method-c-convert_from_neptune_params"></a>
163
+
164
+
165
+ <div class="method-heading">
166
+ <span class="method-name">convert_from_neptune_params</span><span
167
+ class="method-args">(params)</span>
168
+ <span class="method-click-advice">click to toggle source</span>
169
+ </div>
170
+
171
+
172
+ <div class="method-description">
173
+
174
+ <p>Neptune internally uses job_data with keys of the form @name, but since the
175
+ user has given them to us in the form :name, we convert it here. TODO(cgb):
176
+ It looks like this conversion to/from may be unnecessary since neptune()
177
+ just re-converts it - how can we remove it?</p>
178
+
179
+
180
+
181
+ <div class="method-source-code"
182
+ id="convert_from_neptune_params-source">
183
+ <pre>
184
+ <span class="ruby-comment"># File lib/babel.rb, line 198</span>
185
+ def self.convert_from_neptune_params(params)
186
+ job_data = {}
187
+ params.each { |k, v|
188
+ key = &quot;@#{k}&quot;
189
+ job_data[key] = v
190
+ }
191
+ return job_data
192
+ end</pre>
193
+ </div>
194
+
195
+ </div>
196
+
197
+
198
+
199
+
200
+ </div>
201
+
202
+
203
+ <div id="convert_to_neptune_params-method" class="method-detail ">
204
+ <a name="method-c-convert_to_neptune_params"></a>
205
+
206
+
207
+ <div class="method-heading">
208
+ <span class="method-name">convert_to_neptune_params</span><span
209
+ class="method-args">(job_data)</span>
210
+ <span class="method-click-advice">click to toggle source</span>
211
+ </div>
212
+
213
+
214
+ <div class="method-description">
215
+
216
+ <p>Neptune input jobs expect keys of the form :name, but since we’ve already
217
+ converted them to the form @name, this function reverses that conversion.</p>
218
+
219
+
220
+
221
+ <div class="method-source-code"
222
+ id="convert_to_neptune_params-source">
223
+ <pre>
224
+ <span class="ruby-comment"># File lib/babel.rb, line 209</span>
225
+ def self.convert_to_neptune_params(job_data)
226
+ neptune_params = {}
227
+
228
+ job_data.each { |k, v|
229
+ key = k.delete(<span class="ruby-string">&quot;@&quot;</span>).to_sym
230
+ neptune_params[key] = v
231
+ }
232
+
233
+ return neptune_params
234
+ end</pre>
235
+ </div>
236
+
237
+ </div>
238
+
239
+
240
+
241
+
242
+ </div>
243
+
244
+
245
+ <div id="ensure_output_does_not_exist-method" class="method-detail ">
246
+ <a name="method-c-ensure_output_does_not_exist"></a>
247
+
248
+
249
+ <div class="method-heading">
250
+ <span class="method-name">ensure_output_does_not_exist</span><span
251
+ class="method-args">(job_data)</span>
252
+ <span class="method-click-advice">click to toggle source</span>
253
+ </div>
254
+
255
+
256
+ <div class="method-description">
257
+
258
+ <p>To avoid accidentally overwriting outputs from previous jobs, we first
259
+ check to make sure an output file doesn’t exist before starting a new job
260
+ with the given name.</p>
261
+
262
+
263
+
264
+ <div class="method-source-code"
265
+ id="ensure_output_does_not_exist-source">
266
+ <pre>
267
+ <span class="ruby-comment"># File lib/babel.rb, line 134</span>
268
+ def self.ensure_output_does_not_exist(job_data)
269
+ file = job_data[<span class="ruby-string">&quot;@output&quot;</span>]
270
+ controller = self.get_appcontroller(job_data)
271
+ puts job_data.inspect
272
+ <span class="ruby-constant">NeptuneHelper</span>.require_file_to_not_exist(file, job_data, controller)
273
+ end</pre>
274
+ </div>
275
+
276
+ </div>
277
+
278
+
279
+
280
+
281
+ </div>
282
+
283
+
284
+ <div id="generate_output_location-method" class="method-detail ">
285
+ <a name="method-c-generate_output_location"></a>
286
+
287
+
288
+ <div class="method-heading">
289
+ <span class="method-name">generate_output_location</span><span
290
+ class="method-args">(job_data)</span>
291
+ <span class="method-click-advice">click to toggle source</span>
292
+ </div>
293
+
294
+
295
+ <div class="method-description">
296
+
297
+ <p>If the user fails to give us an output location, this function will
298
+ generate one for them, based on either the location of their code (for
299
+ remotely specified code), or a babel parameter (for locally specified
300
+ code).</p>
301
+
302
+
303
+
304
+ <div class="method-source-code"
305
+ id="generate_output_location-source">
306
+ <pre>
307
+ <span class="ruby-comment"># File lib/babel.rb, line 81</span>
308
+ def self.generate_output_location(job_data)
309
+ if job_data[<span class="ruby-string">&quot;@is_remote&quot;</span>]
310
+ <span class="ruby-comment"># We already know the bucket name - the same one that the user</span>
311
+ <span class="ruby-comment"># has told us their code is located in.</span>
312
+ prefix = job_data[<span class="ruby-string">&quot;@code&quot;</span>].scan(<span class="ruby-regexp">/\/(.*?)\//</span>)[0].to_s
313
+ else
314
+ prefix = self.get_bucket_for_local_data(job_data)
315
+ end
316
+
317
+ return &quot;/#{prefix}/babel/temp-#{CommonFunctions.get_random_alphanumeric()}&quot;
318
+ end</pre>
319
+ </div>
320
+
321
+ </div>
322
+
323
+
324
+
325
+
326
+ </div>
327
+
328
+
329
+ <div id="get_appcontroller-method" class="method-detail ">
330
+ <a name="method-c-get_appcontroller"></a>
331
+
332
+
333
+ <div class="method-heading">
334
+ <span class="method-name">get_appcontroller</span><span
335
+ class="method-args">(job_data)</span>
336
+ <span class="method-click-advice">click to toggle source</span>
337
+ </div>
338
+
339
+
340
+ <div class="method-description">
341
+
342
+ <p>Returns an <a href="AppControllerClient.html">AppControllerClient</a> for
343
+ the given job data.</p>
344
+
345
+
346
+
347
+ <div class="method-source-code"
348
+ id="get_appcontroller-source">
349
+ <pre>
350
+ <span class="ruby-comment"># File lib/babel.rb, line 142</span>
351
+ def self.get_appcontroller(job_data)
352
+ keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>] || <span class="ruby-string">&quot;appscale&quot;</span>
353
+ shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
354
+ secret = <span class="ruby-constant">CommonFunctions</span>.get_secret_key(keyname)
355
+ return <span class="ruby-constant">AppControllerClient</span>.new(shadow_ip, secret)
356
+ end</pre>
357
+ </div>
358
+
359
+ </div>
360
+
361
+
362
+
363
+
364
+ </div>
365
+
366
+
367
+ <div id="get_bucket_for_local_data-method" class="method-detail ">
368
+ <a name="method-c-get_bucket_for_local_data"></a>
369
+
370
+
371
+ <div class="method-heading">
372
+ <span class="method-name">get_bucket_for_local_data</span><span
373
+ class="method-args">(job_data)</span>
374
+ <span class="method-click-advice">click to toggle source</span>
375
+ </div>
376
+
377
+
378
+ <div class="method-description">
379
+
380
+ <p>Provides a common way for callers to get the name of the bucket that should
381
+ be used for Neptune jobs where the code is stored locally.</p>
382
+
383
+
384
+
385
+ <div class="method-source-code"
386
+ id="get_bucket_for_local_data-source">
387
+ <pre>
388
+ <span class="ruby-comment"># File lib/babel.rb, line 95</span>
389
+ def self.get_bucket_for_local_data(job_data)
390
+ bucket_name = job_data[<span class="ruby-string">&quot;@bucket_name&quot;</span>] || <span class="ruby-constant">ENV</span>[<span class="ruby-string">'BABEL_BUCKET_NAME'</span>]
391
+
392
+ if bucket_name.nil?
393
+ raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-constant">NEEDS_BUCKET_INFO</span>)
394
+ end
395
+
396
+ <span class="ruby-comment"># If the bucket name starts with a slash, remove it</span>
397
+ if bucket_name[0].chr == <span class="ruby-string">&quot;/&quot;</span>
398
+ bucket_name = bucket_name[1, bucket_name.length]
399
+ end
400
+
401
+ return bucket_name
402
+ end</pre>
403
+ </div>
404
+
405
+ </div>
406
+
407
+
408
+
409
+
410
+ </div>
411
+
412
+
413
+ <div id="put_code-method" class="method-detail ">
414
+ <a name="method-c-put_code"></a>
415
+
416
+
417
+ <div class="method-heading">
418
+ <span class="method-name">put_code</span><span
419
+ class="method-args">(job_data)</span>
420
+ <span class="method-click-advice">click to toggle source</span>
421
+ </div>
422
+
423
+
424
+ <div class="method-description">
425
+
426
+ <p>Stores the user’s code (and the directory it’s in, and directories in
427
+ the same directory as the user’s code, since there could be libraries
428
+ used) in the remote datastore.</p>
429
+
430
+
431
+
432
+ <div class="method-source-code"
433
+ id="put_code-source">
434
+ <pre>
435
+ <span class="ruby-comment"># File lib/babel.rb, line 152</span>
436
+ def self.put_code(job_data)
437
+ code_dir = <span class="ruby-constant">File</span>.dirname(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
438
+ code = <span class="ruby-constant">File</span>.basename(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
439
+ remote_code_dir = self.put_file(code_dir, job_data)
440
+ job_data[<span class="ruby-string">&quot;@code&quot;</span>] = remote_code_dir + <span class="ruby-string">&quot;/&quot;</span> + code
441
+ return job_data[<span class="ruby-string">&quot;@code&quot;</span>]
442
+ end</pre>
443
+ </div>
444
+
445
+ </div>
446
+
447
+
448
+
449
+
450
+ </div>
451
+
452
+
453
+ <div id="put_file-method" class="method-detail ">
454
+ <a name="method-c-put_file"></a>
455
+
456
+
457
+ <div class="method-heading">
458
+ <span class="method-name">put_file</span><span
459
+ class="method-args">(local_path, job_data)</span>
460
+ <span class="method-click-advice">click to toggle source</span>
461
+ </div>
462
+
463
+
464
+ <div class="method-description">
465
+
466
+ <p>If the user gives us local code or local inputs, this function will run a
467
+ Neptune ‘input’ job to store the data remotely.</p>
468
+
469
+
470
+
471
+ <div class="method-source-code"
472
+ id="put_file-source">
473
+ <pre>
474
+ <span class="ruby-comment"># File lib/babel.rb, line 181</span>
475
+ def self.put_file(local_path, job_data)
476
+ input_data = self.convert_to_neptune_params(job_data)
477
+ input_data[:type] = <span class="ruby-string">&quot;input&quot;</span>
478
+ input_data[:local] = local_path
479
+
480
+ bucket_name = self.get_bucket_for_local_data(job_data)
481
+ input_data[:remote] = &quot;/#{bucket_name}/babel#{local_path}&quot;
482
+
483
+ <span class="ruby-constant">Kernel</span>.neptune(input_data)
484
+
485
+ return input_data[:remote]
486
+ end</pre>
487
+ </div>
488
+
489
+ </div>
490
+
491
+
492
+
493
+
494
+ </div>
495
+
496
+
497
+ <div id="put_inputs-method" class="method-detail ">
498
+ <a name="method-c-put_inputs"></a>
499
+
500
+
501
+ <div class="method-heading">
502
+ <span class="method-name">put_inputs</span><span
503
+ class="method-args">(job_data)</span>
504
+ <span class="method-click-advice">click to toggle source</span>
505
+ </div>
506
+
507
+
508
+ <div class="method-description">
509
+
510
+ <p>If any input files are specified, they are copied to the remote datastore
511
+ via Neptune ‘input’ jobs. Inputs are assumed to be files on the local
512
+ filesystem if they begin with a slash, and job_data gets updated with the
513
+ remote location of these files.</p>
514
+
515
+
516
+
517
+ <div class="method-source-code"
518
+ id="put_inputs-source">
519
+ <pre>
520
+ <span class="ruby-comment"># File lib/babel.rb, line 164</span>
521
+ def self.put_inputs(job_data)
522
+ if job_data[<span class="ruby-string">&quot;@argv&quot;</span>].nil? or job_data[<span class="ruby-string">&quot;@argv&quot;</span>].empty?
523
+ return job_data
524
+ end
525
+
526
+ job_data[<span class="ruby-string">&quot;@argv&quot;</span>].each_index { |i|
527
+ arg = job_data[<span class="ruby-string">&quot;@argv&quot;</span>][i]
528
+ if arg[0].chr == <span class="ruby-string">&quot;/&quot;</span>
529
+ job_data[<span class="ruby-string">&quot;@argv&quot;</span>][i] = self.put_file(arg, job_data)
530
+ end
531
+ }
532
+
533
+ return job_data
534
+ end</pre>
535
+ </div>
536
+
537
+ </div>
538
+
539
+
540
+
541
+
542
+ </div>
543
+
544
+
545
+ <div id="run_job-method" class="method-detail ">
546
+ <a name="method-c-run_job"></a>
547
+
548
+
549
+ <div class="method-heading">
550
+ <span class="method-name">run_job</span><span
551
+ class="method-args">(job_data)</span>
552
+ <span class="method-click-advice">click to toggle source</span>
553
+ </div>
554
+
555
+
556
+ <div class="method-description">
557
+
558
+ <p>Constructs a Neptune job to run the user’s code as a Babel job (task
559
+ queue) from the given parameters.</p>
560
+
561
+
562
+
563
+ <div class="method-source-code"
564
+ id="run_job-source">
565
+ <pre>
566
+ <span class="ruby-comment"># File lib/babel.rb, line 222</span>
567
+ def self.run_job(job_data)
568
+ run_data = self.convert_to_neptune_params(job_data)
569
+ run_data[:type] = <span class="ruby-string">&quot;babel&quot;</span>
570
+
571
+ <span class="ruby-comment"># TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to</span>
572
+ <span class="ruby-comment"># exec tasks over it, instead of locally.</span>
573
+ if job_data[<span class="ruby-string">&quot;@run_local&quot;</span>].nil?
574
+ run_data[:run_local] = true
575
+ run_data[:engine] = <span class="ruby-string">&quot;executor-sqs&quot;</span>
576
+ end
577
+
578
+ return <span class="ruby-constant">Kernel</span>.neptune(run_data)
579
+ end</pre>
580
+ </div>
581
+
582
+ </div>
583
+
584
+
585
+
586
+
587
+ </div>
588
+
589
+
590
+ <div id="validate_inputs-method" class="method-detail ">
591
+ <a name="method-c-validate_inputs"></a>
592
+
593
+
594
+ <div class="method-heading">
595
+ <span class="method-name">validate_inputs</span><span
596
+ class="method-args">(job_data)</span>
597
+ <span class="method-click-advice">click to toggle source</span>
598
+ </div>
599
+
600
+
601
+ <div class="method-description">
602
+
603
+ <p>For jobs where the code is stored remotely, this method ensures that the
604
+ code and any possible inputs actually do exist, before attempting to use
605
+ them for computation.</p>
606
+
607
+
608
+
609
+ <div class="method-source-code"
610
+ id="validate_inputs-source">
611
+ <pre>
612
+ <span class="ruby-comment"># File lib/babel.rb, line 113</span>
613
+ def self.validate_inputs(job_data)
614
+ controller = self.get_appcontroller(job_data)
615
+
616
+ <span class="ruby-comment"># First, make sure the code exists</span>
617
+ <span class="ruby-constant">NeptuneHelper</span>.require_file_to_exist(job_data[<span class="ruby-string">&quot;@code&quot;</span>], job_data, controller)
618
+
619
+ if job_data[<span class="ruby-string">&quot;@argv&quot;</span>].nil? or job_data[<span class="ruby-string">&quot;@argv&quot;</span>].empty?
620
+ return
621
+ end
622
+
623
+ <span class="ruby-comment"># We assume anything that begins with a slash is a remote file</span>
624
+ job_data[<span class="ruby-string">&quot;@argv&quot;</span>].each { |arg|
625
+ if arg[0].chr == <span class="ruby-string">&quot;/&quot;</span>
626
+ <span class="ruby-constant">NeptuneHelper</span>.require_file_to_exist(arg, job_data, controller)
627
+ end
628
+ }
629
+ end</pre>
630
+ </div>
631
+
632
+ </div>
633
+
634
+
635
+
636
+
637
+ </div>
638
+
639
+
640
+ <div id="wait_and_get_output-method" class="method-detail ">
641
+ <a name="method-c-wait_and_get_output"></a>
642
+
643
+
644
+ <div class="method-heading">
645
+ <span class="method-name">wait_and_get_output</span><span
646
+ class="method-args">(job_data)</span>
647
+ <span class="method-click-advice">click to toggle source</span>
648
+ </div>
649
+
650
+
651
+ <div class="method-description">
652
+
653
+ <p>Constructs a Neptune job to get the output of a Babel job. If the job is
654
+ not yet finished, this function waits until it does, and then returns the
655
+ output of the job.</p>
656
+
657
+
658
+
659
+ <div class="method-source-code"
660
+ id="wait_and_get_output-source">
661
+ <pre>
662
+ <span class="ruby-comment"># File lib/babel.rb, line 239</span>
663
+ def self.wait_and_get_output(job_data)
664
+ output_data = self.convert_to_neptune_params(job_data)
665
+ output_data[:type] = <span class="ruby-string">&quot;output&quot;</span>
666
+
667
+ output = <span class="ruby-string">&quot;&quot;</span>
668
+ time_to_sleep = <span class="ruby-constant">SLEEP_TIME</span>
669
+ loop {
670
+ output = <span class="ruby-constant">Kernel</span>.neptune(output_data)[:output]
671
+ if output == <span class="ruby-constant">DOES_NOT_EXIST</span>
672
+ <span class="ruby-comment"># Exponentially back off, up to a limit of MAX_SLEEP_TIME</span>
673
+ <span class="ruby-constant">Kernel</span>.sleep(time_to_sleep)
674
+ if time_to_sleep &lt; <span class="ruby-constant">MAX_SLEEP_TIME</span>
675
+ time_to_sleep *= 2
676
+ end
677
+ else
678
+ break
679
+ end
680
+ }
681
+
682
+ return output
683
+ end</pre>
684
+ </div>
685
+
686
+ </div>
687
+
688
+
689
+
690
+
691
+ </div>
692
+
693
+
694
+ </div>
695
+
696
+
697
+ </div>
698
+
699
+ <div id="validator-badges">
700
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
701
+ <p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
702
+ Rdoc Generator</a> 2</small>.</p>
703
+ </div>
704
+
705
+ </body>
706
+ </html>
707
+