neptune 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +4 -0
- data/doc/BabelHelper.html +393 -376
- data/doc/BadConfigurationException.html +121 -127
- data/doc/CommonFunctions.html +237 -265
- data/doc/ExodusHelper.html +820 -0
- data/doc/ExodusTaskInfo.html +263 -0
- data/doc/FileNotFoundException.html +121 -127
- data/doc/NeptuneHelper.html +527 -592
- data/doc/NeptuneManagerClient.html +696 -0
- data/doc/NeptuneManagerException.html +139 -0
- data/doc/Object.html +334 -236
- data/doc/TaskInfo.html +428 -0
- data/doc/created.rid +8 -5
- data/doc/images/add.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/index.html +74 -142
- data/doc/js/darkfish.js +99 -62
- data/doc/js/jquery.js +15 -29
- data/doc/js/navigation.js +142 -0
- data/doc/js/search.js +94 -0
- data/doc/js/search_index.js +1 -0
- data/doc/js/searcher.js +228 -0
- data/doc/table_of_contents.html +226 -0
- data/lib/babel.rb +116 -50
- data/lib/custom_exceptions.rb +2 -2
- data/lib/exodus.rb +311 -0
- data/lib/exodus_task_info.rb +36 -0
- data/lib/neptune.rb +52 -18
- data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
- data/lib/task_info.rb +155 -0
- data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
- data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
- data/test/test_exodus.rb +687 -0
- data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
- data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
- data/test/test_task_info.rb +32 -0
- data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
- metadata +30 -34
- data/doc/AppControllerClient.html +0 -702
- data/doc/AppControllerException.html +0 -145
- data/doc/bin/neptune.html +0 -56
- data/doc/js/quicksearch.js +0 -114
- data/doc/js/thickbox-compressed.js +0 -10
- data/doc/lib/app_controller_client_rb.html +0 -60
- data/doc/lib/babel_rb.html +0 -68
- data/doc/lib/common_functions_rb.html +0 -70
- data/doc/lib/custom_exceptions_rb.html +0 -54
- data/doc/lib/neptune_rb.html +0 -60
- data/test/integration/tc_c.rb +0 -57
- data/test/integration/tc_dfsp.rb +0 -37
- data/test/integration/tc_dwssa.rb +0 -38
- data/test/integration/tc_erlang.rb +0 -183
- data/test/integration/tc_mapreduce.rb +0 -282
- data/test/integration/tc_mpi.rb +0 -160
- data/test/integration/tc_storage.rb +0 -209
- data/test/integration/tc_upc.rb +0 -75
- data/test/integration/tc_x10.rb +0 -94
- data/test/integration/test_helper.rb +0 -135
- data/test/integration/ts_neptune.rb +0 -40
data/README
CHANGED
|
@@ -67,6 +67,10 @@ http://www.neptune-lang.org/2011/6/Neptune-Picks-up-Best-Paper-at-ScienceCloud-2
|
|
|
67
67
|
|
|
68
68
|
Version History:
|
|
69
69
|
|
|
70
|
+
July 30, 2012 - 0.2.2 released, adding batch support for babel calls, and first
|
|
71
|
+
pass on exodus(), which automatically finds the right cloud compute, storage, and
|
|
72
|
+
queue service to deploy your tasks to.
|
|
73
|
+
|
|
70
74
|
February 12, 2012 - 0.2.1 released, adding support for the babel() method on
|
|
71
75
|
non-Babel job types (e.g., MPI, MapReduce), and removed the is_remote
|
|
72
76
|
parameter for babel() calls (since it can always be inferred from :storage).
|
data/doc/BabelHelper.html
CHANGED
|
@@ -1,166 +1,201 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
|
|
3
|
+
<html>
|
|
5
4
|
<head>
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
<title>Module: BabelHelper</title>
|
|
9
|
-
|
|
10
|
-
<link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
|
|
11
|
-
|
|
12
|
-
<script src="./js/jquery.js" type="text/javascript"
|
|
13
|
-
charset="utf-8"></script>
|
|
14
|
-
<script src="./js/thickbox-compressed.js" type="text/javascript"
|
|
15
|
-
charset="utf-8"></script>
|
|
16
|
-
<script src="./js/quicksearch.js" type="text/javascript"
|
|
17
|
-
charset="utf-8"></script>
|
|
18
|
-
<script src="./js/darkfish.js" type="text/javascript"
|
|
19
|
-
charset="utf-8"></script>
|
|
20
|
-
|
|
21
|
-
</head>
|
|
22
|
-
<body class="module">
|
|
23
|
-
|
|
24
|
-
<div id="metadata">
|
|
25
|
-
<div id="home-metadata">
|
|
26
|
-
<div id="home-section" class="section">
|
|
27
|
-
<h3 class="section-header">
|
|
28
|
-
<a href="./index.html">Home</a>
|
|
29
|
-
<a href="./index.html#classes">Classes</a>
|
|
30
|
-
<a href="./index.html#methods">Methods</a>
|
|
31
|
-
</h3>
|
|
32
|
-
</div>
|
|
33
|
-
</div>
|
|
34
|
-
|
|
35
|
-
<div id="file-metadata">
|
|
36
|
-
<div id="file-list-section" class="section">
|
|
37
|
-
<h3 class="section-header">In Files</h3>
|
|
38
|
-
<div class="section-body">
|
|
39
|
-
<ul>
|
|
40
|
-
|
|
41
|
-
<li><a href="./lib/babel_rb.html?TB_iframe=true&height=550&width=785"
|
|
42
|
-
class="thickbox" title="lib/babel.rb">lib/babel.rb</a></li>
|
|
43
|
-
|
|
44
|
-
</ul>
|
|
45
|
-
</div>
|
|
46
|
-
</div>
|
|
5
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
|
47
6
|
|
|
48
|
-
|
|
49
|
-
</div>
|
|
7
|
+
<title>module BabelHelper - RDoc Documentation</title>
|
|
50
8
|
|
|
51
|
-
|
|
9
|
+
<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
|
|
52
10
|
|
|
53
|
-
|
|
54
|
-
|
|
11
|
+
<script type="text/javascript">
|
|
12
|
+
var rdoc_rel_prefix = "./";
|
|
13
|
+
</script>
|
|
55
14
|
|
|
56
|
-
|
|
57
|
-
|
|
15
|
+
<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
|
|
16
|
+
<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
|
|
17
|
+
<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
|
|
18
|
+
<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
|
|
19
|
+
<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
|
|
20
|
+
<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
|
|
58
21
|
|
|
59
|
-
<!-- Method Quickref -->
|
|
60
|
-
|
|
61
|
-
<div id="method-list-section" class="section">
|
|
62
|
-
<h3 class="section-header">Methods</h3>
|
|
63
|
-
<ul class="link-list">
|
|
64
|
-
|
|
65
|
-
<li><a href="#method-c-convert_from_neptune_params">::convert_from_neptune_params</a></li>
|
|
66
|
-
|
|
67
|
-
<li><a href="#method-c-convert_to_neptune_params">::convert_to_neptune_params</a></li>
|
|
68
|
-
|
|
69
|
-
<li><a href="#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist</a></li>
|
|
70
|
-
|
|
71
|
-
<li><a href="#method-c-generate_output_location">::generate_output_location</a></li>
|
|
72
|
-
|
|
73
|
-
<li><a href="#method-c-get_appcontroller">::get_appcontroller</a></li>
|
|
74
|
-
|
|
75
|
-
<li><a href="#method-c-get_bucket_for_local_data">::get_bucket_for_local_data</a></li>
|
|
76
|
-
|
|
77
|
-
<li><a href="#method-c-put_code">::put_code</a></li>
|
|
78
|
-
|
|
79
|
-
<li><a href="#method-c-put_file">::put_file</a></li>
|
|
80
|
-
|
|
81
|
-
<li><a href="#method-c-put_inputs">::put_inputs</a></li>
|
|
82
|
-
|
|
83
|
-
<li><a href="#method-c-run_job">::run_job</a></li>
|
|
84
|
-
|
|
85
|
-
<li><a href="#method-c-validate_inputs">::validate_inputs</a></li>
|
|
86
|
-
|
|
87
|
-
<li><a href="#method-c-wait_and_get_output">::wait_and_get_output</a></li>
|
|
88
|
-
|
|
89
|
-
</ul>
|
|
90
|
-
</div>
|
|
91
|
-
|
|
92
22
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
23
|
+
<body id="top" class="module">
|
|
24
|
+
<nav id="metadata">
|
|
25
|
+
<nav id="home-section" class="section">
|
|
26
|
+
<h3 class="section-header">
|
|
27
|
+
<a href="./index.html">Home</a>
|
|
28
|
+
<a href="./table_of_contents.html#classes">Classes</a>
|
|
29
|
+
<a href="./table_of_contents.html#methods">Methods</a>
|
|
30
|
+
</h3>
|
|
31
|
+
</nav>
|
|
96
32
|
|
|
97
|
-
<div id="project-metadata">
|
|
98
|
-
|
|
99
|
-
|
|
100
33
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
<legend>Quicksearch</legend>
|
|
109
|
-
<input type="text" name="quicksearch" value=""
|
|
110
|
-
class="quicksearch-field" />
|
|
111
|
-
</fieldset>
|
|
112
|
-
</form>
|
|
34
|
+
<nav id="search-section" class="section project-section" class="initially-hidden">
|
|
35
|
+
<form action="#" method="get" accept-charset="utf-8">
|
|
36
|
+
<h3 class="section-header">
|
|
37
|
+
<input type="text" name="search" placeholder="Search" id="search-field"
|
|
38
|
+
title="Type to search, Up and Down to navigate, Enter to load">
|
|
39
|
+
</h3>
|
|
40
|
+
</form>
|
|
113
41
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
42
|
+
<ul id="search-results" class="initially-hidden"></ul>
|
|
43
|
+
</nav>
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
<div id="file-metadata">
|
|
47
|
+
<nav id="file-list-section" class="section">
|
|
48
|
+
<h3 class="section-header">Defined In</h3>
|
|
49
|
+
<ul>
|
|
50
|
+
<li>lib/babel.rb
|
|
51
|
+
</ul>
|
|
52
|
+
</nav>
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
<div id="class-metadata">
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
<!-- Method Quickref -->
|
|
62
|
+
<nav id="method-list-section" class="section">
|
|
63
|
+
<h3 class="section-header">Methods</h3>
|
|
64
|
+
|
|
65
|
+
<ul class="link-list">
|
|
66
|
+
|
|
67
|
+
<li><a href="#method-c-check_output_files">::check_output_files</a>
|
|
68
|
+
|
|
69
|
+
<li><a href="#method-c-convert_from_neptune_params">::convert_from_neptune_params</a>
|
|
70
|
+
|
|
71
|
+
<li><a href="#method-c-convert_to_neptune_params">::convert_to_neptune_params</a>
|
|
72
|
+
|
|
73
|
+
<li><a href="#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist</a>
|
|
74
|
+
|
|
75
|
+
<li><a href="#method-c-generate_output_location">::generate_output_location</a>
|
|
76
|
+
|
|
77
|
+
<li><a href="#method-c-get_bucket_for_local_data">::get_bucket_for_local_data</a>
|
|
78
|
+
|
|
79
|
+
<li><a href="#method-c-get_neptune_manager_client">::get_neptune_manager_client</a>
|
|
80
|
+
|
|
81
|
+
<li><a href="#method-c-put_code">::put_code</a>
|
|
82
|
+
|
|
83
|
+
<li><a href="#method-c-put_file">::put_file</a>
|
|
84
|
+
|
|
85
|
+
<li><a href="#method-c-put_inputs">::put_inputs</a>
|
|
86
|
+
|
|
87
|
+
<li><a href="#method-c-run_job">::run_job</a>
|
|
88
|
+
|
|
89
|
+
<li><a href="#method-c-validate_inputs">::validate_inputs</a>
|
|
90
|
+
|
|
91
|
+
<li><a href="#method-c-wait_and_get_output">::wait_and_get_output</a>
|
|
92
|
+
|
|
93
|
+
</ul>
|
|
94
|
+
</nav>
|
|
95
|
+
|
|
96
|
+
</div>
|
|
97
|
+
|
|
98
|
+
<div id="project-metadata">
|
|
99
|
+
|
|
100
|
+
<nav id="classindex-section" class="section project-section">
|
|
101
|
+
<h3 class="section-header">Class and Module Index</h3>
|
|
102
|
+
|
|
103
|
+
<ul class="link-list">
|
|
104
|
+
|
|
105
|
+
<li><a href="./BabelHelper.html">BabelHelper</a>
|
|
106
|
+
|
|
107
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a>
|
|
108
|
+
|
|
109
|
+
<li><a href="./CommonFunctions.html">CommonFunctions</a>
|
|
110
|
+
|
|
111
|
+
<li><a href="./ExodusHelper.html">ExodusHelper</a>
|
|
112
|
+
|
|
113
|
+
<li><a href="./ExodusTaskInfo.html">ExodusTaskInfo</a>
|
|
114
|
+
|
|
115
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a>
|
|
116
|
+
|
|
117
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a>
|
|
118
|
+
|
|
119
|
+
<li><a href="./NeptuneManagerClient.html">NeptuneManagerClient</a>
|
|
120
|
+
|
|
121
|
+
<li><a href="./NeptuneManagerException.html">NeptuneManagerException</a>
|
|
122
|
+
|
|
123
|
+
<li><a href="./Object.html">Object</a>
|
|
124
|
+
|
|
125
|
+
<li><a href="./TaskInfo.html">TaskInfo</a>
|
|
126
|
+
|
|
127
|
+
</ul>
|
|
128
|
+
</nav>
|
|
135
129
|
|
|
136
|
-
|
|
137
|
-
</div>
|
|
138
130
|
</div>
|
|
131
|
+
</nav>
|
|
139
132
|
|
|
140
|
-
|
|
141
|
-
|
|
133
|
+
<div id="documentation">
|
|
134
|
+
<h1 class="module">module BabelHelper</h1>
|
|
142
135
|
|
|
143
|
-
|
|
144
|
-
|
|
136
|
+
<div id="description" class="description">
|
|
137
|
+
|
|
145
138
|
<p>This module provides convenience functions for babel().</p>
|
|
146
139
|
|
|
147
|
-
|
|
140
|
+
</div><!-- description -->
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
<section id="5Buntitled-5D" class="documentation-section">
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
|
|
148
149
|
|
|
149
|
-
<!-- Constants -->
|
|
150
150
|
|
|
151
151
|
|
|
152
|
-
<!-- Attributes -->
|
|
153
152
|
|
|
154
153
|
|
|
155
154
|
<!-- Methods -->
|
|
156
155
|
|
|
157
|
-
|
|
156
|
+
<section id="public-class-5Buntitled-5D-method-details" class="method-section section">
|
|
158
157
|
<h3 class="section-header">Public Class Methods</h3>
|
|
159
158
|
|
|
160
159
|
|
|
161
|
-
<div id="
|
|
162
|
-
|
|
160
|
+
<div id="method-c-check_output_files" class="method-detail ">
|
|
161
|
+
|
|
162
|
+
<div class="method-heading">
|
|
163
|
+
<span class="method-name">check_output_files</span><span
|
|
164
|
+
class="method-args">(job_data)</span>
|
|
165
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
166
|
+
</div>
|
|
167
|
+
|
|
163
168
|
|
|
169
|
+
<div class="method-description">
|
|
170
|
+
|
|
171
|
+
<p>babel() callers do not have to specify a location where the standard output
|
|
172
|
+
and error the task produces should be placed. If they don’t, generate
|
|
173
|
+
locations for them and make sure they don’t exist beforehand.</p>
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
<div class="method-source-code" id="check_output_files-source">
|
|
178
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 145</span>
|
|
179
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">check_output_files</span>(<span class="ruby-identifier">job_data</span>)
|
|
180
|
+
[<span class="ruby-string">"@output"</span>, <span class="ruby-string">"@error"</span>, <span class="ruby-string">"@metadata"</span>].<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">item</span><span class="ruby-operator">|</span>
|
|
181
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-identifier">item</span>].<span class="ruby-identifier">nil?</span> <span class="ruby-keyword">or</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-identifier">item</span>].<span class="ruby-identifier">empty?</span>
|
|
182
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-identifier">item</span>] = <span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">generate_output_location</span>(<span class="ruby-identifier">job_data</span>)
|
|
183
|
+
<span class="ruby-keyword">else</span>
|
|
184
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">ensure_output_does_not_exist</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">job_data</span>[<span class="ruby-identifier">item</span>])
|
|
185
|
+
<span class="ruby-keyword">end</span>
|
|
186
|
+
}
|
|
187
|
+
<span class="ruby-keyword">end</span></pre>
|
|
188
|
+
</div><!-- check_output_files-source -->
|
|
189
|
+
|
|
190
|
+
</div>
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
</div><!-- check_output_files-method -->
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
<div id="method-c-convert_from_neptune_params" class="method-detail ">
|
|
164
199
|
|
|
165
200
|
<div class="method-heading">
|
|
166
201
|
<span class="method-name">convert_from_neptune_params</span><span
|
|
@@ -178,31 +213,27 @@ just re-converts it - how can we remove it?</p>
|
|
|
178
213
|
|
|
179
214
|
|
|
180
215
|
|
|
181
|
-
<div class="method-source-code"
|
|
182
|
-
|
|
183
|
-
<
|
|
184
|
-
<span class="ruby-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
key = "@#{k}"
|
|
189
|
-
job_data[key] = v
|
|
216
|
+
<div class="method-source-code" id="convert_from_neptune_params-source">
|
|
217
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 256</span>
|
|
218
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">convert_from_neptune_params</span>(<span class="ruby-identifier">params</span>)
|
|
219
|
+
<span class="ruby-identifier">job_data</span> = {}
|
|
220
|
+
<span class="ruby-identifier">params</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">k</span>, <span class="ruby-identifier">v</span><span class="ruby-operator">|</span>
|
|
221
|
+
<span class="ruby-identifier">key</span> = <span class="ruby-node">"@#{k}"</span>
|
|
222
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-identifier">key</span>] = <span class="ruby-identifier">v</span>
|
|
190
223
|
}
|
|
191
|
-
return job_data
|
|
192
|
-
end</pre>
|
|
193
|
-
</div
|
|
224
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">job_data</span>
|
|
225
|
+
<span class="ruby-keyword">end</span></pre>
|
|
226
|
+
</div><!-- convert_from_neptune_params-source -->
|
|
194
227
|
|
|
195
228
|
</div>
|
|
196
229
|
|
|
197
230
|
|
|
198
231
|
|
|
199
232
|
|
|
200
|
-
</div
|
|
233
|
+
</div><!-- convert_from_neptune_params-method -->
|
|
201
234
|
|
|
202
235
|
|
|
203
|
-
<div id="convert_to_neptune_params
|
|
204
|
-
<a name="method-c-convert_to_neptune_params"></a>
|
|
205
|
-
|
|
236
|
+
<div id="method-c-convert_to_neptune_params" class="method-detail ">
|
|
206
237
|
|
|
207
238
|
<div class="method-heading">
|
|
208
239
|
<span class="method-name">convert_to_neptune_params</span><span
|
|
@@ -218,37 +249,33 @@ converted them to the form @name, this function reverses that conversion.</p>
|
|
|
218
249
|
|
|
219
250
|
|
|
220
251
|
|
|
221
|
-
<div class="method-source-code"
|
|
222
|
-
|
|
223
|
-
<
|
|
224
|
-
<span class="ruby-
|
|
225
|
-
def self.convert_to_neptune_params(job_data)
|
|
226
|
-
neptune_params = {}
|
|
252
|
+
<div class="method-source-code" id="convert_to_neptune_params-source">
|
|
253
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 268</span>
|
|
254
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">convert_to_neptune_params</span>(<span class="ruby-identifier">job_data</span>)
|
|
255
|
+
<span class="ruby-identifier">neptune_params</span> = {}
|
|
227
256
|
|
|
228
|
-
job_data
|
|
229
|
-
key = k
|
|
230
|
-
neptune_params[key] = v
|
|
257
|
+
<span class="ruby-identifier">job_data</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">k</span>, <span class="ruby-identifier">v</span><span class="ruby-operator">|</span>
|
|
258
|
+
<span class="ruby-identifier">key</span> = <span class="ruby-identifier">k</span>.<span class="ruby-identifier">delete</span>(<span class="ruby-string">"@"</span>).<span class="ruby-identifier">to_sym</span>
|
|
259
|
+
<span class="ruby-identifier">neptune_params</span>[<span class="ruby-identifier">key</span>] = <span class="ruby-identifier">v</span>
|
|
231
260
|
}
|
|
232
261
|
|
|
233
|
-
return neptune_params
|
|
234
|
-
end</pre>
|
|
235
|
-
</div
|
|
262
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">neptune_params</span>
|
|
263
|
+
<span class="ruby-keyword">end</span></pre>
|
|
264
|
+
</div><!-- convert_to_neptune_params-source -->
|
|
236
265
|
|
|
237
266
|
</div>
|
|
238
267
|
|
|
239
268
|
|
|
240
269
|
|
|
241
270
|
|
|
242
|
-
</div
|
|
271
|
+
</div><!-- convert_to_neptune_params-method -->
|
|
243
272
|
|
|
244
273
|
|
|
245
|
-
<div id="ensure_output_does_not_exist
|
|
246
|
-
<a name="method-c-ensure_output_does_not_exist"></a>
|
|
247
|
-
|
|
274
|
+
<div id="method-c-ensure_output_does_not_exist" class="method-detail ">
|
|
248
275
|
|
|
249
276
|
<div class="method-heading">
|
|
250
277
|
<span class="method-name">ensure_output_does_not_exist</span><span
|
|
251
|
-
class="method-args">(job_data)</span>
|
|
278
|
+
class="method-args">(job_data, remote_file)</span>
|
|
252
279
|
<span class="method-click-advice">click to toggle source</span>
|
|
253
280
|
</div>
|
|
254
281
|
|
|
@@ -261,29 +288,23 @@ with the given name.</p>
|
|
|
261
288
|
|
|
262
289
|
|
|
263
290
|
|
|
264
|
-
<div class="method-source-code"
|
|
265
|
-
|
|
266
|
-
<
|
|
267
|
-
<span class="ruby-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
<span class="ruby-comment"># Kernel.puts job_data.inspect</span>
|
|
272
|
-
<span class="ruby-constant">NeptuneHelper</span>.require_file_to_not_exist(file, job_data, controller)
|
|
273
|
-
end</pre>
|
|
274
|
-
</div>
|
|
291
|
+
<div class="method-source-code" id="ensure_output_does_not_exist-source">
|
|
292
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 181</span>
|
|
293
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">ensure_output_does_not_exist</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">remote_file</span>)
|
|
294
|
+
<span class="ruby-identifier">controller</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_neptune_manager_client</span>(<span class="ruby-identifier">job_data</span>)
|
|
295
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">require_file_to_not_exist</span>(<span class="ruby-identifier">remote_file</span>, <span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
|
|
296
|
+
<span class="ruby-keyword">end</span></pre>
|
|
297
|
+
</div><!-- ensure_output_does_not_exist-source -->
|
|
275
298
|
|
|
276
299
|
</div>
|
|
277
300
|
|
|
278
301
|
|
|
279
302
|
|
|
280
303
|
|
|
281
|
-
</div
|
|
304
|
+
</div><!-- ensure_output_does_not_exist-method -->
|
|
282
305
|
|
|
283
306
|
|
|
284
|
-
<div id="generate_output_location
|
|
285
|
-
<a name="method-c-generate_output_location"></a>
|
|
286
|
-
|
|
307
|
+
<div id="method-c-generate_output_location" class="method-detail ">
|
|
287
308
|
|
|
288
309
|
<div class="method-heading">
|
|
289
310
|
<span class="method-name">generate_output_location</span><span
|
|
@@ -301,37 +322,33 @@ code).</p>
|
|
|
301
322
|
|
|
302
323
|
|
|
303
324
|
|
|
304
|
-
<div class="method-source-code"
|
|
305
|
-
|
|
306
|
-
<
|
|
307
|
-
<span class="ruby-
|
|
308
|
-
def self.generate_output_location(job_data)
|
|
309
|
-
if job_data[<span class="ruby-string">"@storage"</span>]
|
|
325
|
+
<div class="method-source-code" id="generate_output_location-source">
|
|
326
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 109</span>
|
|
327
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">generate_output_location</span>(<span class="ruby-identifier">job_data</span>)
|
|
328
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@is_remote"</span>]
|
|
310
329
|
<span class="ruby-comment"># We already know the bucket name - the same one that the user</span>
|
|
311
330
|
<span class="ruby-comment"># has told us their code is located in.</span>
|
|
312
|
-
prefix = job_data[<span class="ruby-string">"@code"</span>]
|
|
313
|
-
else
|
|
314
|
-
prefix = self
|
|
315
|
-
end
|
|
331
|
+
<span class="ruby-identifier">prefix</span> = <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@code"</span>].<span class="ruby-identifier">scan</span>(<span class="ruby-regexp">%r\/(.*?)\//</span>)[<span class="ruby-value">0</span>].<span class="ruby-identifier">to_s</span>
|
|
332
|
+
<span class="ruby-keyword">else</span>
|
|
333
|
+
<span class="ruby-identifier">prefix</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_bucket_for_local_data</span>(<span class="ruby-identifier">job_data</span>)
|
|
334
|
+
<span class="ruby-keyword">end</span>
|
|
316
335
|
|
|
317
|
-
return
|
|
318
|
-
end</pre>
|
|
319
|
-
</div
|
|
336
|
+
<span class="ruby-keyword">return</span> <span class="ruby-node">"/#{prefix}/babel/temp-#{CommonFunctions.get_random_alphanumeric()}"</span>
|
|
337
|
+
<span class="ruby-keyword">end</span></pre>
|
|
338
|
+
</div><!-- generate_output_location-source -->
|
|
320
339
|
|
|
321
340
|
</div>
|
|
322
341
|
|
|
323
342
|
|
|
324
343
|
|
|
325
344
|
|
|
326
|
-
</div
|
|
345
|
+
</div><!-- generate_output_location-method -->
|
|
327
346
|
|
|
328
347
|
|
|
329
|
-
<div id="
|
|
330
|
-
<a name="method-c-get_appcontroller"></a>
|
|
331
|
-
|
|
348
|
+
<div id="method-c-get_bucket_for_local_data" class="method-detail ">
|
|
332
349
|
|
|
333
350
|
<div class="method-heading">
|
|
334
|
-
<span class="method-name">
|
|
351
|
+
<span class="method-name">get_bucket_for_local_data</span><span
|
|
335
352
|
class="method-args">(job_data)</span>
|
|
336
353
|
<span class="method-click-advice">click to toggle source</span>
|
|
337
354
|
</div>
|
|
@@ -339,37 +356,43 @@ end</pre>
|
|
|
339
356
|
|
|
340
357
|
<div class="method-description">
|
|
341
358
|
|
|
342
|
-
<p>
|
|
343
|
-
the
|
|
359
|
+
<p>Provides a common way for callers to get the name of the bucket that should
|
|
360
|
+
be used for Neptune jobs where the code is stored locally.</p>
|
|
344
361
|
|
|
345
362
|
|
|
346
363
|
|
|
347
|
-
<div class="method-source-code"
|
|
348
|
-
|
|
349
|
-
<
|
|
350
|
-
<span class="ruby-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
end</
|
|
357
|
-
|
|
364
|
+
<div class="method-source-code" id="get_bucket_for_local_data-source">
|
|
365
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 124</span>
|
|
366
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_bucket_for_local_data</span>(<span class="ruby-identifier">job_data</span>)
|
|
367
|
+
<span class="ruby-identifier">bucket_name</span> = <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@bucket_name"</span>] <span class="ruby-operator">||</span> <span class="ruby-constant">ENV</span>[<span class="ruby-string">'BABEL_BUCKET_NAME'</span>] <span class="ruby-operator">||</span>
|
|
368
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@S3_bucket_name"</span>] <span class="ruby-operator">||</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@Walrus_bucket_name"</span>] <span class="ruby-operator">||</span>
|
|
369
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@GStorage_bucket_name"</span>] <span class="ruby-operator">||</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@WAZ_Container_Name"</span>]
|
|
370
|
+
|
|
371
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">bucket_name</span>.<span class="ruby-identifier">nil?</span>
|
|
372
|
+
<span class="ruby-identifier">raise</span> <span class="ruby-constant">BadConfigurationException</span>.<span class="ruby-identifier">new</span>(<span class="ruby-constant">NEEDS_BUCKET_INFO</span>)
|
|
373
|
+
<span class="ruby-keyword">end</span>
|
|
374
|
+
|
|
375
|
+
<span class="ruby-comment"># If the bucket name starts with a slash, remove it</span>
|
|
376
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">bucket_name</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">chr</span> <span class="ruby-operator">==</span> <span class="ruby-string">"/"</span>
|
|
377
|
+
<span class="ruby-identifier">bucket_name</span> = <span class="ruby-identifier">bucket_name</span>[<span class="ruby-value">1</span>, <span class="ruby-identifier">bucket_name</span>.<span class="ruby-identifier">length</span>]
|
|
378
|
+
<span class="ruby-keyword">end</span>
|
|
379
|
+
|
|
380
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">bucket_name</span>
|
|
381
|
+
<span class="ruby-keyword">end</span></pre>
|
|
382
|
+
</div><!-- get_bucket_for_local_data-source -->
|
|
358
383
|
|
|
359
384
|
</div>
|
|
360
385
|
|
|
361
386
|
|
|
362
387
|
|
|
363
388
|
|
|
364
|
-
</div
|
|
389
|
+
</div><!-- get_bucket_for_local_data-method -->
|
|
365
390
|
|
|
366
391
|
|
|
367
|
-
<div id="
|
|
368
|
-
<a name="method-c-get_bucket_for_local_data"></a>
|
|
369
|
-
|
|
392
|
+
<div id="method-c-get_neptune_manager_client" class="method-detail ">
|
|
370
393
|
|
|
371
394
|
<div class="method-heading">
|
|
372
|
-
<span class="method-name">
|
|
395
|
+
<span class="method-name">get_neptune_manager_client</span><span
|
|
373
396
|
class="method-args">(job_data)</span>
|
|
374
397
|
<span class="method-click-advice">click to toggle source</span>
|
|
375
398
|
</div>
|
|
@@ -377,42 +400,30 @@ end</pre>
|
|
|
377
400
|
|
|
378
401
|
<div class="method-description">
|
|
379
402
|
|
|
380
|
-
<p>
|
|
381
|
-
|
|
403
|
+
<p>Returns an <a href="NeptuneManagerClient.html">NeptuneManagerClient</a> for
|
|
404
|
+
the given job data.</p>
|
|
382
405
|
|
|
383
406
|
|
|
384
407
|
|
|
385
|
-
<div class="method-source-code"
|
|
386
|
-
|
|
387
|
-
<
|
|
388
|
-
<span class="ruby-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
end
|
|
395
|
-
|
|
396
|
-
<span class="ruby-comment"># If the bucket name starts with a slash, remove it</span>
|
|
397
|
-
if bucket_name[0].chr == <span class="ruby-string">"/"</span>
|
|
398
|
-
bucket_name = bucket_name[1, bucket_name.length]
|
|
399
|
-
end
|
|
400
|
-
|
|
401
|
-
return bucket_name
|
|
402
|
-
end</pre>
|
|
403
|
-
</div>
|
|
408
|
+
<div class="method-source-code" id="get_neptune_manager_client-source">
|
|
409
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 188</span>
|
|
410
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_neptune_manager_client</span>(<span class="ruby-identifier">job_data</span>)
|
|
411
|
+
<span class="ruby-identifier">keyname</span> = <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@keyname"</span>] <span class="ruby-operator">||</span> <span class="ruby-string">"appscale"</span>
|
|
412
|
+
<span class="ruby-identifier">shadow_ip</span> = <span class="ruby-constant">CommonFunctions</span>.<span class="ruby-identifier">get_from_yaml</span>(<span class="ruby-identifier">keyname</span>, <span class="ruby-value">:shadow</span>)
|
|
413
|
+
<span class="ruby-identifier">secret</span> = <span class="ruby-constant">CommonFunctions</span>.<span class="ruby-identifier">get_secret_key</span>(<span class="ruby-identifier">keyname</span>)
|
|
414
|
+
<span class="ruby-keyword">return</span> <span class="ruby-constant">NeptuneManagerClient</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">shadow_ip</span>, <span class="ruby-identifier">secret</span>)
|
|
415
|
+
<span class="ruby-keyword">end</span></pre>
|
|
416
|
+
</div><!-- get_neptune_manager_client-source -->
|
|
404
417
|
|
|
405
418
|
</div>
|
|
406
419
|
|
|
407
420
|
|
|
408
421
|
|
|
409
422
|
|
|
410
|
-
</div
|
|
423
|
+
</div><!-- get_neptune_manager_client-method -->
|
|
411
424
|
|
|
412
425
|
|
|
413
|
-
<div id="put_code
|
|
414
|
-
<a name="method-c-put_code"></a>
|
|
415
|
-
|
|
426
|
+
<div id="method-c-put_code" class="method-detail ">
|
|
416
427
|
|
|
417
428
|
<div class="method-heading">
|
|
418
429
|
<span class="method-name">put_code</span><span
|
|
@@ -429,30 +440,26 @@ used) in the remote datastore.</p>
|
|
|
429
440
|
|
|
430
441
|
|
|
431
442
|
|
|
432
|
-
<div class="method-source-code"
|
|
433
|
-
|
|
434
|
-
<
|
|
435
|
-
<span class="ruby-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
end</pre>
|
|
443
|
-
</div>
|
|
443
|
+
<div class="method-source-code" id="put_code-source">
|
|
444
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 199</span>
|
|
445
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">put_code</span>(<span class="ruby-identifier">job_data</span>)
|
|
446
|
+
<span class="ruby-identifier">code_dir</span> = <span class="ruby-constant">File</span>.<span class="ruby-identifier">dirname</span>(<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@code"</span>])
|
|
447
|
+
<span class="ruby-identifier">code</span> = <span class="ruby-constant">File</span>.<span class="ruby-identifier">basename</span>(<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@code"</span>])
|
|
448
|
+
<span class="ruby-identifier">remote_code_dir</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">put_file</span>(<span class="ruby-identifier">code_dir</span>, <span class="ruby-identifier">job_data</span>)
|
|
449
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@code"</span>] = <span class="ruby-identifier">remote_code_dir</span> <span class="ruby-operator">+</span> <span class="ruby-string">"/"</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">code</span>
|
|
450
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@code"</span>]
|
|
451
|
+
<span class="ruby-keyword">end</span></pre>
|
|
452
|
+
</div><!-- put_code-source -->
|
|
444
453
|
|
|
445
454
|
</div>
|
|
446
455
|
|
|
447
456
|
|
|
448
457
|
|
|
449
458
|
|
|
450
|
-
</div
|
|
459
|
+
</div><!-- put_code-method -->
|
|
451
460
|
|
|
452
461
|
|
|
453
|
-
<div id="put_file
|
|
454
|
-
<a name="method-c-put_file"></a>
|
|
455
|
-
|
|
462
|
+
<div id="method-c-put_file" class="method-detail ">
|
|
456
463
|
|
|
457
464
|
<div class="method-heading">
|
|
458
465
|
<span class="method-name">put_file</span><span
|
|
@@ -468,35 +475,39 @@ Neptune ‘input’ job to store the data remotely.</p>
|
|
|
468
475
|
|
|
469
476
|
|
|
470
477
|
|
|
471
|
-
<div class="method-source-code"
|
|
472
|
-
|
|
473
|
-
<
|
|
474
|
-
<span class="ruby-
|
|
475
|
-
|
|
476
|
-
input_data =
|
|
477
|
-
input_data[:type] = <span class="ruby-string">"input"</span>
|
|
478
|
-
input_data[:local] = local_path
|
|
478
|
+
<div class="method-source-code" id="put_file-source">
|
|
479
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 230</span>
|
|
480
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">put_file</span>(<span class="ruby-identifier">local_path</span>, <span class="ruby-identifier">job_data</span>)
|
|
481
|
+
<span class="ruby-identifier">input_data</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">convert_to_neptune_params</span>(<span class="ruby-identifier">job_data</span>)
|
|
482
|
+
<span class="ruby-identifier">input_data</span>[<span class="ruby-value">:type</span>] = <span class="ruby-string">"input"</span>
|
|
483
|
+
<span class="ruby-identifier">input_data</span>[<span class="ruby-value">:local</span>] = <span class="ruby-identifier">local_path</span>
|
|
479
484
|
|
|
480
|
-
bucket_name = self
|
|
481
|
-
input_data[
|
|
485
|
+
<span class="ruby-identifier">bucket_name</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_bucket_for_local_data</span>(<span class="ruby-identifier">job_data</span>)
|
|
486
|
+
<span class="ruby-identifier">input_data</span>[<span class="ruby-value">:remote</span>] = <span class="ruby-node">"/#{bucket_name}/babel#{local_path}"</span>
|
|
487
|
+
|
|
488
|
+
<span class="ruby-identifier">start</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
|
|
489
|
+
<span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">neptune</span>(<span class="ruby-identifier">input_data</span>)
|
|
490
|
+
<span class="ruby-identifier">fin</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
|
|
491
|
+
|
|
492
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">'@metadata_info'</span>].<span class="ruby-identifier">nil?</span>
|
|
493
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">'@metadata_info'</span>] = {<span class="ruby-string">'time_to_store_inputs'</span> =<span class="ruby-operator">></span> <span class="ruby-value">0.0</span>}
|
|
494
|
+
<span class="ruby-keyword">end</span>
|
|
482
495
|
|
|
483
|
-
<span class="ruby-
|
|
496
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">'@metadata_info'</span>][<span class="ruby-string">'time_to_store_inputs'</span>] <span class="ruby-operator">+=</span> <span class="ruby-identifier">fin</span> <span class="ruby-operator">-</span> <span class="ruby-identifier">start</span>
|
|
484
497
|
|
|
485
|
-
return input_data[
|
|
486
|
-
end</pre>
|
|
487
|
-
</div
|
|
498
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">input_data</span>[<span class="ruby-value">:remote</span>]
|
|
499
|
+
<span class="ruby-keyword">end</span></pre>
|
|
500
|
+
</div><!-- put_file-source -->
|
|
488
501
|
|
|
489
502
|
</div>
|
|
490
503
|
|
|
491
504
|
|
|
492
505
|
|
|
493
506
|
|
|
494
|
-
</div
|
|
507
|
+
</div><!-- put_file-method -->
|
|
495
508
|
|
|
496
509
|
|
|
497
|
-
<div id="put_inputs
|
|
498
|
-
<a name="method-c-put_inputs"></a>
|
|
499
|
-
|
|
510
|
+
<div id="method-c-put_inputs" class="method-detail ">
|
|
500
511
|
|
|
501
512
|
<div class="method-heading">
|
|
502
513
|
<span class="method-name">put_inputs</span><span
|
|
@@ -514,41 +525,37 @@ remote location of these files.</p>
|
|
|
514
525
|
|
|
515
526
|
|
|
516
527
|
|
|
517
|
-
<div class="method-source-code"
|
|
518
|
-
|
|
519
|
-
<
|
|
520
|
-
<span class="ruby-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
return job_data
|
|
524
|
-
end
|
|
528
|
+
<div class="method-source-code" id="put_inputs-source">
|
|
529
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 212</span>
|
|
530
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">put_inputs</span>(<span class="ruby-identifier">job_data</span>)
|
|
531
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>].<span class="ruby-identifier">nil?</span> <span class="ruby-keyword">or</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>].<span class="ruby-identifier">empty?</span>
|
|
532
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">job_data</span>
|
|
533
|
+
<span class="ruby-keyword">end</span>
|
|
525
534
|
|
|
526
|
-
job_data[<span class="ruby-string">"@argv"</span>]
|
|
527
|
-
arg = job_data[<span class="ruby-string">"@argv"</span>][i]
|
|
528
|
-
if arg[0]
|
|
529
|
-
job_data[<span class="ruby-string">"@argv"</span>][i] = self
|
|
530
|
-
end
|
|
535
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>].<span class="ruby-identifier">each_index</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">i</span><span class="ruby-operator">|</span>
|
|
536
|
+
<span class="ruby-identifier">arg</span> = <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>][<span class="ruby-identifier">i</span>]
|
|
537
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">arg</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">chr</span> <span class="ruby-operator">==</span> <span class="ruby-string">"/"</span>
|
|
538
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>][<span class="ruby-identifier">i</span>] = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">put_file</span>(<span class="ruby-identifier">arg</span>, <span class="ruby-identifier">job_data</span>)
|
|
539
|
+
<span class="ruby-keyword">end</span>
|
|
531
540
|
}
|
|
532
541
|
|
|
533
|
-
return job_data
|
|
534
|
-
end</pre>
|
|
535
|
-
</div
|
|
542
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">job_data</span>
|
|
543
|
+
<span class="ruby-keyword">end</span></pre>
|
|
544
|
+
</div><!-- put_inputs-source -->
|
|
536
545
|
|
|
537
546
|
</div>
|
|
538
547
|
|
|
539
548
|
|
|
540
549
|
|
|
541
550
|
|
|
542
|
-
</div
|
|
551
|
+
</div><!-- put_inputs-method -->
|
|
543
552
|
|
|
544
553
|
|
|
545
|
-
<div id="run_job
|
|
546
|
-
<a name="method-c-run_job"></a>
|
|
547
|
-
|
|
554
|
+
<div id="method-c-run_job" class="method-detail ">
|
|
548
555
|
|
|
549
556
|
<div class="method-heading">
|
|
550
557
|
<span class="method-name">run_job</span><span
|
|
551
|
-
class="method-args">(
|
|
558
|
+
class="method-args">(job_data_list)</span>
|
|
552
559
|
<span class="method-click-advice">click to toggle source</span>
|
|
553
560
|
</div>
|
|
554
561
|
|
|
@@ -560,40 +567,56 @@ queue) from the given parameters.</p>
|
|
|
560
567
|
|
|
561
568
|
|
|
562
569
|
|
|
563
|
-
<div class="method-source-code"
|
|
564
|
-
|
|
565
|
-
<
|
|
566
|
-
<span class="ruby-
|
|
567
|
-
|
|
568
|
-
|
|
570
|
+
<div class="method-source-code" id="run_job-source">
|
|
571
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 282</span>
|
|
572
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">job_data_list</span>)
|
|
573
|
+
<span class="ruby-identifier">run_data_list</span> = []
|
|
574
|
+
|
|
575
|
+
<span class="ruby-identifier">job_data_list</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">job_data</span><span class="ruby-operator">|</span>
|
|
576
|
+
<span class="ruby-identifier">run_data</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">convert_to_neptune_params</span>(<span class="ruby-identifier">job_data</span>)
|
|
569
577
|
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
578
|
+
<span class="ruby-comment"># Default to babel as the job type, if the user doesn't specify one.</span>
|
|
579
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">run_data</span>[<span class="ruby-value">:type</span>].<span class="ruby-identifier">nil?</span> <span class="ruby-keyword">or</span> <span class="ruby-identifier">run_data</span>[<span class="ruby-value">:type</span>].<span class="ruby-identifier">empty?</span>
|
|
580
|
+
<span class="ruby-identifier">run_data</span>[<span class="ruby-value">:type</span>] = <span class="ruby-string">"babel"</span>
|
|
581
|
+
<span class="ruby-keyword">end</span>
|
|
574
582
|
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
583
|
+
<span class="ruby-comment"># TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to</span>
|
|
584
|
+
<span class="ruby-comment"># exec tasks over it, instead of locally.</span>
|
|
585
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@run_local"</span>].<span class="ruby-identifier">nil?</span>
|
|
586
|
+
<span class="ruby-identifier">run_data</span>[<span class="ruby-value">:run_local</span>] = <span class="ruby-keyword">true</span>
|
|
587
|
+
<span class="ruby-identifier">run_data</span>[<span class="ruby-value">:engine</span>] = <span class="ruby-string">"executor-sqs"</span>
|
|
588
|
+
<span class="ruby-keyword">end</span>
|
|
581
589
|
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
590
|
+
<span class="ruby-identifier">run_data</span>[<span class="ruby-value">:failed_attempts</span>] = <span class="ruby-value">0</span>
|
|
591
|
+
<span class="ruby-identifier">run_data_list</span> <span class="ruby-operator"><<</span> <span class="ruby-identifier">run_data</span>
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
<span class="ruby-identifier">loop</span> {
|
|
595
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">run_data_list</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">==</span> <span class="ruby-value">1</span>
|
|
596
|
+
<span class="ruby-identifier">run_job</span> = <span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">neptune</span>(<span class="ruby-identifier">run_data_list</span>[<span class="ruby-value">0</span>])
|
|
597
|
+
<span class="ruby-keyword">else</span>
|
|
598
|
+
<span class="ruby-identifier">run_job</span> = <span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">neptune</span>(<span class="ruby-identifier">run_data_list</span>)
|
|
599
|
+
<span class="ruby-keyword">end</span>
|
|
600
|
+
|
|
601
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">run_job</span>[<span class="ruby-value">:result</span>] <span class="ruby-operator">==</span> <span class="ruby-value">:success</span>
|
|
602
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">run_job</span>
|
|
603
|
+
<span class="ruby-keyword">else</span>
|
|
604
|
+
<span class="ruby-identifier">run_data_list</span>[<span class="ruby-value">0</span>][<span class="ruby-value">:failed_attempts</span>] <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
|
|
605
|
+
<span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">sleep</span>(<span class="ruby-constant">SLEEP_TIME</span>) <span class="ruby-comment"># TODO(cgb): this should exponentially backoff</span>
|
|
606
|
+
<span class="ruby-keyword">end</span>
|
|
607
|
+
}
|
|
608
|
+
<span class="ruby-keyword">end</span></pre>
|
|
609
|
+
</div><!-- run_job-source -->
|
|
585
610
|
|
|
586
611
|
</div>
|
|
587
612
|
|
|
588
613
|
|
|
589
614
|
|
|
590
615
|
|
|
591
|
-
</div
|
|
616
|
+
</div><!-- run_job-method -->
|
|
592
617
|
|
|
593
618
|
|
|
594
|
-
<div id="validate_inputs
|
|
595
|
-
<a name="method-c-validate_inputs"></a>
|
|
596
|
-
|
|
619
|
+
<div id="method-c-validate_inputs" class="method-detail ">
|
|
597
620
|
|
|
598
621
|
<div class="method-heading">
|
|
599
622
|
<span class="method-name">validate_inputs</span><span
|
|
@@ -610,44 +633,40 @@ them for computation.</p>
|
|
|
610
633
|
|
|
611
634
|
|
|
612
635
|
|
|
613
|
-
<div class="method-source-code"
|
|
614
|
-
|
|
615
|
-
<
|
|
616
|
-
<span class="ruby-
|
|
617
|
-
def self.validate_inputs(job_data)
|
|
618
|
-
controller = self.get_appcontroller(job_data)
|
|
636
|
+
<div class="method-source-code" id="validate_inputs-source">
|
|
637
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 159</span>
|
|
638
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">validate_inputs</span>(<span class="ruby-identifier">job_data</span>)
|
|
639
|
+
<span class="ruby-identifier">controller</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_neptune_manager_client</span>(<span class="ruby-identifier">job_data</span>)
|
|
619
640
|
|
|
620
641
|
<span class="ruby-comment"># First, make sure the code exists</span>
|
|
621
|
-
<span class="ruby-constant">NeptuneHelper</span
|
|
642
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">require_file_to_exist</span>(<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@code"</span>], <span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
|
|
622
643
|
|
|
623
|
-
if job_data[<span class="ruby-string">"@argv"</span>]
|
|
624
|
-
return
|
|
625
|
-
end
|
|
644
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>].<span class="ruby-identifier">nil?</span> <span class="ruby-keyword">or</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>].<span class="ruby-identifier">empty?</span>
|
|
645
|
+
<span class="ruby-keyword">return</span>
|
|
646
|
+
<span class="ruby-keyword">end</span>
|
|
626
647
|
|
|
627
648
|
<span class="ruby-comment"># We assume anything that begins with a slash is a remote file</span>
|
|
628
|
-
job_data[<span class="ruby-string">"@argv"</span>]
|
|
629
|
-
if arg[0]
|
|
630
|
-
<span class="ruby-constant">NeptuneHelper</span
|
|
631
|
-
end
|
|
649
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@argv"</span>].<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">arg</span><span class="ruby-operator">|</span>
|
|
650
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">arg</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">chr</span> <span class="ruby-operator">==</span> <span class="ruby-string">"/"</span>
|
|
651
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">require_file_to_exist</span>(<span class="ruby-identifier">arg</span>, <span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
|
|
652
|
+
<span class="ruby-keyword">end</span>
|
|
632
653
|
}
|
|
633
|
-
end</pre>
|
|
634
|
-
</div
|
|
654
|
+
<span class="ruby-keyword">end</span></pre>
|
|
655
|
+
</div><!-- validate_inputs-source -->
|
|
635
656
|
|
|
636
657
|
</div>
|
|
637
658
|
|
|
638
659
|
|
|
639
660
|
|
|
640
661
|
|
|
641
|
-
</div
|
|
662
|
+
</div><!-- validate_inputs-method -->
|
|
642
663
|
|
|
643
664
|
|
|
644
|
-
<div id="wait_and_get_output
|
|
645
|
-
<a name="method-c-wait_and_get_output"></a>
|
|
646
|
-
|
|
665
|
+
<div id="method-c-wait_and_get_output" class="method-detail ">
|
|
647
666
|
|
|
648
667
|
<div class="method-heading">
|
|
649
668
|
<span class="method-name">wait_and_get_output</span><span
|
|
650
|
-
class="method-args">(job_data)</span>
|
|
669
|
+
class="method-args">(job_data, output_location)</span>
|
|
651
670
|
<span class="method-click-advice">click to toggle source</span>
|
|
652
671
|
</div>
|
|
653
672
|
|
|
@@ -660,52 +679,50 @@ output of the job.</p>
|
|
|
660
679
|
|
|
661
680
|
|
|
662
681
|
|
|
663
|
-
<div class="method-source-code"
|
|
664
|
-
|
|
665
|
-
<
|
|
666
|
-
<span class="ruby-
|
|
667
|
-
|
|
668
|
-
output_data =
|
|
669
|
-
output_data[:type] = <span class="ruby-string">"output"</span>
|
|
682
|
+
<div class="method-source-code" id="wait_and_get_output-source">
|
|
683
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 324</span>
|
|
684
|
+
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">wait_and_get_output</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">output_location</span>)
|
|
685
|
+
<span class="ruby-identifier">output_data</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">convert_to_neptune_params</span>(<span class="ruby-identifier">job_data</span>)
|
|
686
|
+
<span class="ruby-identifier">output_data</span>[<span class="ruby-value">:type</span>] = <span class="ruby-string">"output"</span>
|
|
687
|
+
<span class="ruby-identifier">output_data</span>[<span class="ruby-value">:output</span>] = <span class="ruby-identifier">output_location</span>
|
|
670
688
|
|
|
671
|
-
output = <span class="ruby-string">""</span>
|
|
672
|
-
time_to_sleep = <span class="ruby-constant">SLEEP_TIME</span>
|
|
673
|
-
loop {
|
|
674
|
-
output = <span class="ruby-constant">Kernel</span
|
|
675
|
-
if output
|
|
689
|
+
<span class="ruby-identifier">output</span> = <span class="ruby-string">""</span>
|
|
690
|
+
<span class="ruby-identifier">time_to_sleep</span> = <span class="ruby-constant">SLEEP_TIME</span>
|
|
691
|
+
<span class="ruby-identifier">loop</span> {
|
|
692
|
+
<span class="ruby-identifier">output</span> = <span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">neptune</span>(<span class="ruby-identifier">output_data</span>)[<span class="ruby-value">:output</span>]
|
|
693
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">output</span> <span class="ruby-operator">==</span> <span class="ruby-constant">DOES_NOT_EXIST</span>
|
|
676
694
|
<span class="ruby-comment"># Exponentially back off, up to a limit of MAX_SLEEP_TIME</span>
|
|
677
|
-
<span class="ruby-constant">Kernel</span
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
end
|
|
681
|
-
else
|
|
682
|
-
break
|
|
683
|
-
end
|
|
695
|
+
<span class="ruby-constant">Kernel</span>.<span class="ruby-identifier">sleep</span>(<span class="ruby-identifier">time_to_sleep</span>)
|
|
696
|
+
<span class="ruby-comment">#if time_to_sleep < MAX_SLEEP_TIME</span>
|
|
697
|
+
<span class="ruby-comment"># time_to_sleep *= 2</span>
|
|
698
|
+
<span class="ruby-comment">#end</span>
|
|
699
|
+
<span class="ruby-keyword">else</span>
|
|
700
|
+
<span class="ruby-keyword">break</span>
|
|
701
|
+
<span class="ruby-keyword">end</span>
|
|
684
702
|
}
|
|
685
703
|
|
|
686
|
-
return output
|
|
687
|
-
end</pre>
|
|
688
|
-
</div
|
|
704
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">output</span>
|
|
705
|
+
<span class="ruby-keyword">end</span></pre>
|
|
706
|
+
</div><!-- wait_and_get_output-source -->
|
|
689
707
|
|
|
690
708
|
</div>
|
|
691
709
|
|
|
692
710
|
|
|
693
711
|
|
|
694
712
|
|
|
695
|
-
</div
|
|
713
|
+
</div><!-- wait_and_get_output-method -->
|
|
696
714
|
|
|
697
715
|
|
|
698
|
-
</
|
|
716
|
+
</section><!-- public-class-method-details -->
|
|
699
717
|
|
|
718
|
+
</section><!-- 5Buntitled-5D -->
|
|
700
719
|
|
|
701
|
-
|
|
720
|
+
</div><!-- documentation -->
|
|
702
721
|
|
|
703
|
-
<div id="validator-badges">
|
|
704
|
-
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
|
705
|
-
<p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
|
|
706
|
-
Rdoc Generator</a> 2</small>.</p>
|
|
707
|
-
</div>
|
|
708
722
|
|
|
709
|
-
|
|
710
|
-
</
|
|
723
|
+
<footer id="validator-badges">
|
|
724
|
+
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
|
725
|
+
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
|
|
726
|
+
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
|
727
|
+
</footer>
|
|
711
728
|
|