neptune 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README +4 -0
- data/doc/BabelHelper.html +393 -376
- data/doc/BadConfigurationException.html +121 -127
- data/doc/CommonFunctions.html +237 -265
- data/doc/ExodusHelper.html +820 -0
- data/doc/ExodusTaskInfo.html +263 -0
- data/doc/FileNotFoundException.html +121 -127
- data/doc/NeptuneHelper.html +527 -592
- data/doc/NeptuneManagerClient.html +696 -0
- data/doc/NeptuneManagerException.html +139 -0
- data/doc/Object.html +334 -236
- data/doc/TaskInfo.html +428 -0
- data/doc/created.rid +8 -5
- data/doc/images/add.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/index.html +74 -142
- data/doc/js/darkfish.js +99 -62
- data/doc/js/jquery.js +15 -29
- data/doc/js/navigation.js +142 -0
- data/doc/js/search.js +94 -0
- data/doc/js/search_index.js +1 -0
- data/doc/js/searcher.js +228 -0
- data/doc/table_of_contents.html +226 -0
- data/lib/babel.rb +116 -50
- data/lib/custom_exceptions.rb +2 -2
- data/lib/exodus.rb +311 -0
- data/lib/exodus_task_info.rb +36 -0
- data/lib/neptune.rb +52 -18
- data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
- data/lib/task_info.rb +155 -0
- data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
- data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
- data/test/test_exodus.rb +687 -0
- data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
- data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
- data/test/test_task_info.rb +32 -0
- data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
- metadata +30 -34
- data/doc/AppControllerClient.html +0 -702
- data/doc/AppControllerException.html +0 -145
- data/doc/bin/neptune.html +0 -56
- data/doc/js/quicksearch.js +0 -114
- data/doc/js/thickbox-compressed.js +0 -10
- data/doc/lib/app_controller_client_rb.html +0 -60
- data/doc/lib/babel_rb.html +0 -68
- data/doc/lib/common_functions_rb.html +0 -70
- data/doc/lib/custom_exceptions_rb.html +0 -54
- data/doc/lib/neptune_rb.html +0 -60
- data/test/integration/tc_c.rb +0 -57
- data/test/integration/tc_dfsp.rb +0 -37
- data/test/integration/tc_dwssa.rb +0 -38
- data/test/integration/tc_erlang.rb +0 -183
- data/test/integration/tc_mapreduce.rb +0 -282
- data/test/integration/tc_mpi.rb +0 -160
- data/test/integration/tc_storage.rb +0 -209
- data/test/integration/tc_upc.rb +0 -75
- data/test/integration/tc_x10.rb +0 -94
- data/test/integration/test_helper.rb +0 -135
- data/test/integration/ts_neptune.rb +0 -40
@@ -0,0 +1,139 @@
|
|
1
|
+
<!DOCTYPE html>
|
2
|
+
|
3
|
+
<html>
|
4
|
+
<head>
|
5
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
6
|
+
|
7
|
+
<title>class NeptuneManagerException - RDoc Documentation</title>
|
8
|
+
|
9
|
+
<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
|
10
|
+
|
11
|
+
<script type="text/javascript">
|
12
|
+
var rdoc_rel_prefix = "./";
|
13
|
+
</script>
|
14
|
+
|
15
|
+
<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
|
16
|
+
<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
|
17
|
+
<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
|
18
|
+
<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
|
19
|
+
<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
|
20
|
+
<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
|
21
|
+
|
22
|
+
|
23
|
+
<body id="top" class="class">
|
24
|
+
<nav id="metadata">
|
25
|
+
<nav id="home-section" class="section">
|
26
|
+
<h3 class="section-header">
|
27
|
+
<a href="./index.html">Home</a>
|
28
|
+
<a href="./table_of_contents.html#classes">Classes</a>
|
29
|
+
<a href="./table_of_contents.html#methods">Methods</a>
|
30
|
+
</h3>
|
31
|
+
</nav>
|
32
|
+
|
33
|
+
|
34
|
+
<nav id="search-section" class="section project-section" class="initially-hidden">
|
35
|
+
<form action="#" method="get" accept-charset="utf-8">
|
36
|
+
<h3 class="section-header">
|
37
|
+
<input type="text" name="search" placeholder="Search" id="search-field"
|
38
|
+
title="Type to search, Up and Down to navigate, Enter to load">
|
39
|
+
</h3>
|
40
|
+
</form>
|
41
|
+
|
42
|
+
<ul id="search-results" class="initially-hidden"></ul>
|
43
|
+
</nav>
|
44
|
+
|
45
|
+
|
46
|
+
<div id="file-metadata">
|
47
|
+
<nav id="file-list-section" class="section">
|
48
|
+
<h3 class="section-header">Defined In</h3>
|
49
|
+
<ul>
|
50
|
+
<li>lib/custom_exceptions.rb
|
51
|
+
</ul>
|
52
|
+
</nav>
|
53
|
+
|
54
|
+
|
55
|
+
</div>
|
56
|
+
|
57
|
+
<div id="class-metadata">
|
58
|
+
|
59
|
+
<nav id="parent-class-section" class="section">
|
60
|
+
<h3 class="section-header">Parent</h3>
|
61
|
+
|
62
|
+
<p class="link">Exception
|
63
|
+
|
64
|
+
</nav>
|
65
|
+
|
66
|
+
|
67
|
+
|
68
|
+
</div>
|
69
|
+
|
70
|
+
<div id="project-metadata">
|
71
|
+
|
72
|
+
<nav id="classindex-section" class="section project-section">
|
73
|
+
<h3 class="section-header">Class and Module Index</h3>
|
74
|
+
|
75
|
+
<ul class="link-list">
|
76
|
+
|
77
|
+
<li><a href="./BabelHelper.html">BabelHelper</a>
|
78
|
+
|
79
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a>
|
80
|
+
|
81
|
+
<li><a href="./CommonFunctions.html">CommonFunctions</a>
|
82
|
+
|
83
|
+
<li><a href="./ExodusHelper.html">ExodusHelper</a>
|
84
|
+
|
85
|
+
<li><a href="./ExodusTaskInfo.html">ExodusTaskInfo</a>
|
86
|
+
|
87
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a>
|
88
|
+
|
89
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a>
|
90
|
+
|
91
|
+
<li><a href="./NeptuneManagerClient.html">NeptuneManagerClient</a>
|
92
|
+
|
93
|
+
<li><a href="./NeptuneManagerException.html">NeptuneManagerException</a>
|
94
|
+
|
95
|
+
<li><a href="./Object.html">Object</a>
|
96
|
+
|
97
|
+
<li><a href="./TaskInfo.html">TaskInfo</a>
|
98
|
+
|
99
|
+
</ul>
|
100
|
+
</nav>
|
101
|
+
|
102
|
+
</div>
|
103
|
+
</nav>
|
104
|
+
|
105
|
+
<div id="documentation">
|
106
|
+
<h1 class="class">class NeptuneManagerException</h1>
|
107
|
+
|
108
|
+
<div id="description" class="description">
|
109
|
+
|
110
|
+
<p>A special class of exceptions that are thrown whenever the NeptuneManager
|
111
|
+
experiences an unexpected result.</p>
|
112
|
+
|
113
|
+
</div><!-- description -->
|
114
|
+
|
115
|
+
|
116
|
+
|
117
|
+
|
118
|
+
<section id="5Buntitled-5D" class="documentation-section">
|
119
|
+
|
120
|
+
|
121
|
+
|
122
|
+
|
123
|
+
|
124
|
+
|
125
|
+
|
126
|
+
|
127
|
+
<!-- Methods -->
|
128
|
+
|
129
|
+
</section><!-- 5Buntitled-5D -->
|
130
|
+
|
131
|
+
</div><!-- documentation -->
|
132
|
+
|
133
|
+
|
134
|
+
<footer id="validator-badges">
|
135
|
+
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
136
|
+
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
|
137
|
+
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
138
|
+
</footer>
|
139
|
+
|
data/doc/Object.html
CHANGED
@@ -1,239 +1,233 @@
|
|
1
|
-
|
2
|
-
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
3
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
4
|
-
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
5
|
-
<head>
|
6
|
-
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
|
7
|
-
|
8
|
-
<title>Class: Object</title>
|
9
|
-
|
10
|
-
<link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
|
11
|
-
|
12
|
-
<script src="./js/jquery.js" type="text/javascript"
|
13
|
-
charset="utf-8"></script>
|
14
|
-
<script src="./js/thickbox-compressed.js" type="text/javascript"
|
15
|
-
charset="utf-8"></script>
|
16
|
-
<script src="./js/quicksearch.js" type="text/javascript"
|
17
|
-
charset="utf-8"></script>
|
18
|
-
<script src="./js/darkfish.js" type="text/javascript"
|
19
|
-
charset="utf-8"></script>
|
20
|
-
|
21
|
-
</head>
|
22
|
-
<body class="class">
|
23
|
-
|
24
|
-
<div id="metadata">
|
25
|
-
<div id="home-metadata">
|
26
|
-
<div id="home-section" class="section">
|
27
|
-
<h3 class="section-header">
|
28
|
-
<a href="./index.html">Home</a>
|
29
|
-
<a href="./index.html#classes">Classes</a>
|
30
|
-
<a href="./index.html#methods">Methods</a>
|
31
|
-
</h3>
|
32
|
-
</div>
|
33
|
-
</div>
|
34
|
-
|
35
|
-
<div id="file-metadata">
|
36
|
-
<div id="file-list-section" class="section">
|
37
|
-
<h3 class="section-header">In Files</h3>
|
38
|
-
<div class="section-body">
|
39
|
-
<ul>
|
40
|
-
|
41
|
-
<li><a href="./lib/app_controller_client_rb.html?TB_iframe=true&height=550&width=785"
|
42
|
-
class="thickbox" title="lib/app_controller_client.rb">lib/app_controller_client.rb</a></li>
|
43
|
-
|
44
|
-
<li><a href="./lib/babel_rb.html?TB_iframe=true&height=550&width=785"
|
45
|
-
class="thickbox" title="lib/babel.rb">lib/babel.rb</a></li>
|
46
|
-
|
47
|
-
<li><a href="./lib/neptune_rb.html?TB_iframe=true&height=550&width=785"
|
48
|
-
class="thickbox" title="lib/neptune.rb">lib/neptune.rb</a></li>
|
49
|
-
|
50
|
-
</ul>
|
51
|
-
</div>
|
52
|
-
</div>
|
53
|
-
|
54
|
-
|
55
|
-
</div>
|
1
|
+
<!DOCTYPE html>
|
56
2
|
|
57
|
-
|
3
|
+
<html>
|
4
|
+
<head>
|
5
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
6
|
+
|
7
|
+
<title>class Object - RDoc Documentation</title>
|
8
|
+
|
9
|
+
<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
|
10
|
+
|
11
|
+
<script type="text/javascript">
|
12
|
+
var rdoc_rel_prefix = "./";
|
13
|
+
</script>
|
14
|
+
|
15
|
+
<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
|
16
|
+
<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
|
17
|
+
<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
|
18
|
+
<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
|
19
|
+
<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
|
20
|
+
<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
|
21
|
+
|
22
|
+
|
23
|
+
<body id="top" class="class">
|
24
|
+
<nav id="metadata">
|
25
|
+
<nav id="home-section" class="section">
|
26
|
+
<h3 class="section-header">
|
27
|
+
<a href="./index.html">Home</a>
|
28
|
+
<a href="./table_of_contents.html#classes">Classes</a>
|
29
|
+
<a href="./table_of_contents.html#methods">Methods</a>
|
30
|
+
</h3>
|
31
|
+
</nav>
|
32
|
+
|
33
|
+
|
34
|
+
<nav id="search-section" class="section project-section" class="initially-hidden">
|
35
|
+
<form action="#" method="get" accept-charset="utf-8">
|
36
|
+
<h3 class="section-header">
|
37
|
+
<input type="text" name="search" placeholder="Search" id="search-field"
|
38
|
+
title="Type to search, Up and Down to navigate, Enter to load">
|
39
|
+
</h3>
|
40
|
+
</form>
|
41
|
+
|
42
|
+
<ul id="search-results" class="initially-hidden"></ul>
|
43
|
+
</nav>
|
44
|
+
|
45
|
+
|
46
|
+
<div id="file-metadata">
|
47
|
+
<nav id="file-list-section" class="section">
|
48
|
+
<h3 class="section-header">Defined In</h3>
|
49
|
+
<ul>
|
50
|
+
<li>lib/exodus.rb
|
51
|
+
<li>lib/babel.rb
|
52
|
+
<li>lib/neptune_manager_client.rb
|
53
|
+
<li>lib/neptune.rb
|
54
|
+
</ul>
|
55
|
+
</nav>
|
58
56
|
|
59
|
-
|
60
|
-
|
61
|
-
<div id="parent-class-section" class="section">
|
62
|
-
<h3 class="section-header">Parent</h3>
|
63
|
-
|
64
|
-
<p class="link"></p>
|
65
|
-
|
66
|
-
</div>
|
67
|
-
|
57
|
+
|
58
|
+
</div>
|
68
59
|
|
69
|
-
|
70
|
-
|
60
|
+
<div id="class-metadata">
|
61
|
+
|
62
|
+
<nav id="parent-class-section" class="section">
|
63
|
+
<h3 class="section-header">Parent</h3>
|
64
|
+
|
65
|
+
<p class="link">
|
66
|
+
|
67
|
+
</nav>
|
71
68
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
<ul class="link-list">
|
77
|
-
|
78
|
-
<li><a href="#method-i-babel">#babel</a></li>
|
79
|
-
|
80
|
-
<li><a href="#method-i-neptune">#neptune</a></li>
|
81
|
-
|
82
|
-
</ul>
|
83
|
-
</div>
|
84
|
-
|
69
|
+
|
70
|
+
<!-- Method Quickref -->
|
71
|
+
<nav id="method-list-section" class="section">
|
72
|
+
<h3 class="section-header">Methods</h3>
|
85
73
|
|
86
|
-
|
87
|
-
|
88
|
-
</
|
74
|
+
<ul class="link-list">
|
75
|
+
|
76
|
+
<li><a href="#method-i-babel">#babel</a>
|
77
|
+
|
78
|
+
<li><a href="#method-i-exodus">#exodus</a>
|
79
|
+
|
80
|
+
<li><a href="#method-i-neptune">#neptune</a>
|
81
|
+
|
82
|
+
</ul>
|
83
|
+
</nav>
|
89
84
|
|
90
|
-
|
91
|
-
|
92
|
-
|
85
|
+
</div>
|
93
86
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
title="show/hide quicksearch" /></span></h3>
|
99
|
-
<form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
|
100
|
-
<fieldset>
|
101
|
-
<legend>Quicksearch</legend>
|
102
|
-
<input type="text" name="quicksearch" value=""
|
103
|
-
class="quicksearch-field" />
|
104
|
-
</fieldset>
|
105
|
-
</form>
|
87
|
+
<div id="project-metadata">
|
88
|
+
|
89
|
+
<nav id="classindex-section" class="section project-section">
|
90
|
+
<h3 class="section-header">Class and Module Index</h3>
|
106
91
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
92
|
+
<ul class="link-list">
|
93
|
+
|
94
|
+
<li><a href="./BabelHelper.html">BabelHelper</a>
|
95
|
+
|
96
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a>
|
97
|
+
|
98
|
+
<li><a href="./CommonFunctions.html">CommonFunctions</a>
|
99
|
+
|
100
|
+
<li><a href="./ExodusHelper.html">ExodusHelper</a>
|
101
|
+
|
102
|
+
<li><a href="./ExodusTaskInfo.html">ExodusTaskInfo</a>
|
103
|
+
|
104
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a>
|
105
|
+
|
106
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a>
|
107
|
+
|
108
|
+
<li><a href="./NeptuneManagerClient.html">NeptuneManagerClient</a>
|
109
|
+
|
110
|
+
<li><a href="./NeptuneManagerException.html">NeptuneManagerException</a>
|
111
|
+
|
112
|
+
<li><a href="./Object.html">Object</a>
|
113
|
+
|
114
|
+
<li><a href="./TaskInfo.html">TaskInfo</a>
|
115
|
+
|
116
|
+
</ul>
|
117
|
+
</nav>
|
128
118
|
|
129
|
-
|
130
|
-
</div>
|
131
119
|
</div>
|
120
|
+
</nav>
|
132
121
|
|
133
|
-
|
134
|
-
|
122
|
+
<div id="documentation">
|
123
|
+
<h1 class="class">class Object</h1>
|
135
124
|
|
136
|
-
|
137
|
-
|
125
|
+
<div id="description" class="description">
|
126
|
+
|
138
127
|
<p>Since we’re monkeypatching <a href="Object.html">Object</a> to add
|
139
128
|
neptune() and babel(), a short blurb is necessary here to make rdoc happy.</p>
|
140
129
|
|
141
|
-
|
130
|
+
</div><!-- description -->
|
142
131
|
|
143
|
-
|
132
|
+
|
133
|
+
|
134
|
+
|
135
|
+
<section id="5Buntitled-5D" class="documentation-section">
|
144
136
|
|
145
|
-
|
137
|
+
|
138
|
+
|
139
|
+
|
140
|
+
|
141
|
+
<!-- Constants -->
|
142
|
+
<section id="constants-list" class="section">
|
146
143
|
<h3 class="section-header">Constants</h3>
|
147
144
|
<dl>
|
148
145
|
|
149
|
-
<dt
|
146
|
+
<dt id="ALLOWED_JOB_TYPES">ALLOWED_JOB_TYPES
|
150
147
|
|
151
|
-
<dd class="description"><p>
|
152
|
-
sent over the network: for this first version we don’t want these calls
|
153
|
-
to endlessly timeout and retry, so as a hack, just don’t let them
|
154
|
-
timeout. The next version should replace this and properly timeout and not
|
155
|
-
use long calls unless necessary.</p></dd>
|
148
|
+
<dd class="description"><p>A list of all the Neptune job types that we support</p>
|
156
149
|
|
157
150
|
|
158
|
-
<dt
|
151
|
+
<dt id="ALLOWED_STORAGE_TYPES">ALLOWED_STORAGE_TYPES
|
159
152
|
|
160
|
-
<dd class="description"><p>
|
161
|
-
|
153
|
+
<dd class="description"><p>A list of storage mechanisms that we can use to store and retrieve data to
|
154
|
+
for Neptune jobs.</p>
|
162
155
|
|
163
156
|
|
164
|
-
<dt
|
157
|
+
<dt id="DOES_NOT_EXIST">DOES_NOT_EXIST
|
165
158
|
|
166
159
|
<dd class="description"><p>The constant string that a Neptune output job returns if the output does
|
167
|
-
not yet exist.</p
|
160
|
+
not yet exist.</p>
|
168
161
|
|
169
162
|
|
170
|
-
<dt
|
163
|
+
<dt id="JOB_TYPE_NOT_ALLOWED">JOB_TYPE_NOT_ALLOWED
|
171
164
|
|
172
|
-
<dd class="description"><p>The
|
173
|
-
requests. An exponential backoff is used with this value as the starting
|
174
|
-
sleep time.</p></dd>
|
165
|
+
<dd class="description"><p>The string to display for disallowed job types.</p>
|
175
166
|
|
176
167
|
|
177
|
-
<dt
|
168
|
+
<dt id="MAX_SLEEP_TIME">MAX_SLEEP_TIME
|
178
169
|
|
179
170
|
<dd class="description"><p>The maximum amount of time that we should sleep to, when waiting for output
|
180
|
-
job requests.</p
|
171
|
+
job requests.</p>
|
181
172
|
|
182
173
|
|
183
|
-
<dt
|
174
|
+
<dt id="NEEDS_BUCKET_INFO">NEEDS_BUCKET_INFO
|
184
175
|
|
185
|
-
<dd class="description"><p>
|
176
|
+
<dd class="description"><p>If the user doesn’t give us enough info to infer what bucket we should
|
177
|
+
place their code in, this message is displayed and execution aborts.</p>
|
186
178
|
|
187
179
|
|
188
|
-
<dt
|
180
|
+
<dt id="NEED_PREPROCESSING">NEED_PREPROCESSING
|
189
181
|
|
190
|
-
<dd class="description"><p>
|
182
|
+
<dd class="description"><p>A list of jobs that require some kind of work to be done before the actual
|
183
|
+
computation can be performed.</p>
|
191
184
|
|
192
185
|
|
193
|
-
<dt
|
186
|
+
<dt id="NO_NODES_NEEDED">NO_NODES_NEEDED
|
194
187
|
|
195
188
|
<dd class="description"><p>A list of Neptune jobs that do not require nodes to be spawned up for
|
196
|
-
computation</p
|
189
|
+
computation</p>
|
197
190
|
|
198
191
|
|
199
|
-
<dt
|
192
|
+
<dt id="NO_OUTPUT_NEEDED">NO_OUTPUT_NEEDED
|
200
193
|
|
201
194
|
<dd class="description"><p>A list of Neptune jobs that do not require the output to be specified
|
202
|
-
beforehand</p
|
195
|
+
beforehand</p>
|
203
196
|
|
204
197
|
|
205
|
-
<dt
|
198
|
+
<dt id="NO_TIMEOUT">NO_TIMEOUT
|
206
199
|
|
207
|
-
<dd class="description"><p>
|
208
|
-
for
|
200
|
+
<dd class="description"><p>Sometimes SOAP calls take a long time if large amounts of data are being
|
201
|
+
sent over the network: for this first version we don’t want these calls
|
202
|
+
to endlessly timeout and retry, so as a hack, just don’t let them
|
203
|
+
timeout. The next version should replace this and properly timeout and not
|
204
|
+
use long calls unless necessary.</p>
|
209
205
|
|
210
206
|
|
211
|
-
<dt
|
207
|
+
<dt id="SLEEP_TIME">SLEEP_TIME
|
212
208
|
|
213
|
-
<dd class="description"><p>
|
214
|
-
|
209
|
+
<dd class="description"><p>The initial amount of time, in seconds, to sleep between output job
|
210
|
+
requests. An exponential backoff is used with this value as the starting
|
211
|
+
sleep time.</p>
|
215
212
|
|
216
213
|
|
217
214
|
</dl>
|
218
|
-
</
|
215
|
+
</section>
|
219
216
|
|
220
217
|
|
221
|
-
<!-- Attributes -->
|
222
218
|
|
223
219
|
|
224
220
|
<!-- Methods -->
|
225
221
|
|
226
|
-
|
222
|
+
<section id="public-instance-5Buntitled-5D-method-details" class="method-section section">
|
227
223
|
<h3 class="section-header">Public Instance Methods</h3>
|
228
224
|
|
229
225
|
|
230
|
-
<div id="babel
|
231
|
-
<a name="method-i-babel"></a>
|
232
|
-
|
226
|
+
<div id="method-i-babel" class="method-detail ">
|
233
227
|
|
234
228
|
<div class="method-heading">
|
235
229
|
<span class="method-name">babel</span><span
|
236
|
-
class="method-args">(
|
230
|
+
class="method-args">(jobs)</span>
|
237
231
|
<span class="method-click-advice">click to toggle source</span>
|
238
232
|
</div>
|
239
233
|
|
@@ -247,61 +241,141 @@ handles this.</p>
|
|
247
241
|
|
248
242
|
|
249
243
|
|
250
|
-
<div class="method-source-code"
|
251
|
-
|
252
|
-
<
|
253
|
-
<span class="ruby-comment"># File lib/babel.rb, line 43</span>
|
254
|
-
def babel(params)
|
244
|
+
<div class="method-source-code" id="babel-source">
|
245
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 44</span>
|
246
|
+
<span class="ruby-keyword">def</span> <span class="ruby-identifier">babel</span>(<span class="ruby-identifier">jobs</span>)
|
255
247
|
<span class="ruby-comment"># Since this whole function should run asynchronously, we run it as a future.</span>
|
256
248
|
<span class="ruby-comment"># It automatically starts running in a new thread, and attempting to get the</span>
|
257
249
|
<span class="ruby-comment"># value of what this returns causes it to block until the job completes.</span>
|
258
|
-
future {
|
259
|
-
if
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
<span class="ruby-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
<span class="ruby-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
250
|
+
<span class="ruby-comment">#future {</span>
|
251
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Hash</span>
|
252
|
+
<span class="ruby-identifier">was_one_job</span> = <span class="ruby-keyword">true</span>
|
253
|
+
<span class="ruby-identifier">jobs</span> = [<span class="ruby-identifier">jobs</span>]
|
254
|
+
<span class="ruby-keyword">else</span>
|
255
|
+
<span class="ruby-identifier">was_one_job</span> = <span class="ruby-keyword">false</span>
|
256
|
+
<span class="ruby-keyword">end</span>
|
257
|
+
|
258
|
+
<span class="ruby-identifier">job_data_list</span> = []
|
259
|
+
<span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">params</span><span class="ruby-operator">|</span>
|
260
|
+
<span class="ruby-identifier">job_data</span> = <span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">convert_from_neptune_params</span>(<span class="ruby-identifier">params</span>)
|
261
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">'@metadata_info'</span>] = {<span class="ruby-string">'time_to_store_inputs'</span> =<span class="ruby-operator">></span> <span class="ruby-value">0.0</span>}
|
262
|
+
|
263
|
+
<span class="ruby-comment"># Add in S3 storage parameters</span>
|
264
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">validate_storage_params</span>(<span class="ruby-identifier">job_data</span>)
|
265
|
+
|
266
|
+
<span class="ruby-comment"># :code is the only required parameter</span>
|
267
|
+
<span class="ruby-comment"># everything else can use default vals</span>
|
268
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">require_param</span>(<span class="ruby-string">"@code"</span>, <span class="ruby-identifier">job_data</span>)
|
269
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">check_output_files</span>(<span class="ruby-identifier">job_data</span>)
|
270
|
+
|
271
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@is_remote"</span>]
|
272
|
+
<span class="ruby-comment">#BabelHelper.validate_inputs(job_data)</span>
|
273
|
+
<span class="ruby-keyword">else</span>
|
274
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">put_code</span>(<span class="ruby-identifier">job_data</span>)
|
275
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">put_inputs</span>(<span class="ruby-identifier">job_data</span>)
|
276
|
+
<span class="ruby-keyword">end</span>
|
277
|
+
|
278
|
+
<span class="ruby-identifier">job_data_list</span> <span class="ruby-operator"><<</span> <span class="ruby-identifier">job_data</span>
|
279
|
+
}
|
280
|
+
|
281
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">job_data_list</span>)
|
282
|
+
|
283
|
+
<span class="ruby-comment"># Return an object to the user that has all the information about their</span>
|
284
|
+
<span class="ruby-comment"># task, including its standard out, err, debugging info, and profiling</span>
|
285
|
+
<span class="ruby-comment"># info. Since the job may not be done when the user asks for this info,</span>
|
286
|
+
<span class="ruby-comment"># its the responsibility of TaskInfo objects to block until that info</span>
|
287
|
+
<span class="ruby-comment"># is ready. We don't explicitly return the TaskInfo object, because it's</span>
|
288
|
+
<span class="ruby-comment"># in a Future block - it will automatically return whatever the last</span>
|
289
|
+
<span class="ruby-comment"># statement returns.</span>
|
290
|
+
<span class="ruby-identifier">tasks</span> = []
|
291
|
+
<span class="ruby-identifier">job_data_list</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">job_data</span><span class="ruby-operator">|</span>
|
292
|
+
<span class="ruby-identifier">tasks</span> <span class="ruby-operator"><<</span> <span class="ruby-constant">TaskInfo</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">job_data</span>)
|
293
|
+
}
|
294
|
+
|
295
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">was_one_job</span>
|
296
|
+
<span class="ruby-identifier">tasks</span>[<span class="ruby-value">0</span>]
|
297
|
+
<span class="ruby-keyword">else</span>
|
298
|
+
<span class="ruby-identifier">tasks</span>
|
299
|
+
<span class="ruby-keyword">end</span>
|
300
|
+
<span class="ruby-comment">#}</span>
|
301
|
+
<span class="ruby-keyword">end</span></pre>
|
302
|
+
</div><!-- babel-source -->
|
289
303
|
|
290
304
|
</div>
|
291
305
|
|
292
306
|
|
293
307
|
|
294
308
|
|
295
|
-
</div
|
309
|
+
</div><!-- babel-method -->
|
296
310
|
|
297
311
|
|
298
|
-
<div id="
|
299
|
-
|
312
|
+
<div id="method-i-exodus" class="method-detail ">
|
313
|
+
|
314
|
+
<div class="method-heading">
|
315
|
+
<span class="method-name">exodus</span><span
|
316
|
+
class="method-args">(jobs)</span>
|
317
|
+
<span class="method-click-advice">click to toggle source</span>
|
318
|
+
</div>
|
319
|
+
|
320
|
+
|
321
|
+
<div class="method-description">
|
322
|
+
|
323
|
+
<p>Exodus provides further improvements to Babel. Instead of making users tell
|
324
|
+
us what compute, storage, and queue services they want to use (required for
|
325
|
+
babel calls), Exodus will automatically handle this for us. Callers need to
|
326
|
+
specify what clouds their job can run over, and Exodus will automatically
|
327
|
+
select the best cloud for their job and run it there.</p>
|
328
|
+
|
329
|
+
|
330
|
+
|
331
|
+
<div class="method-source-code" id="exodus-source">
|
332
|
+
<pre><span class="ruby-comment"># File lib/exodus.rb, line 15</span>
|
333
|
+
<span class="ruby-keyword">def</span> <span class="ruby-identifier">exodus</span>(<span class="ruby-identifier">jobs</span>)
|
334
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Hash</span>
|
335
|
+
<span class="ruby-identifier">job_given_as_hash</span> = <span class="ruby-keyword">true</span>
|
336
|
+
<span class="ruby-identifier">jobs</span> = [<span class="ruby-identifier">jobs</span>]
|
337
|
+
<span class="ruby-keyword">elsif</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Array</span>
|
338
|
+
<span class="ruby-identifier">job_given_as_hash</span> = <span class="ruby-keyword">false</span>
|
339
|
+
<span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">ensure_all_jobs_are_hashes</span>(<span class="ruby-identifier">jobs</span>)
|
340
|
+
<span class="ruby-keyword">else</span>
|
341
|
+
<span class="ruby-identifier">raise</span> <span class="ruby-constant">BadConfigurationException</span>.<span class="ruby-identifier">new</span>(<span class="ruby-node">"jobs was a #{jobs.class}, which "</span> <span class="ruby-operator">+</span>
|
342
|
+
<span class="ruby-string">"is not an acceptable class type"</span>)
|
343
|
+
<span class="ruby-keyword">end</span>
|
344
|
+
|
345
|
+
<span class="ruby-identifier">tasks</span> = []
|
346
|
+
|
347
|
+
<span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">job</span><span class="ruby-operator">|</span>
|
348
|
+
<span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">ensure_all_params_are_present</span>(<span class="ruby-identifier">job</span>)
|
349
|
+
<span class="ruby-identifier">profiling_info</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">get_profiling_info</span>(<span class="ruby-identifier">job</span>)
|
350
|
+
<span class="ruby-identifier">clouds_to_run_task_on</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">get_clouds_to_run_task_on</span>(<span class="ruby-identifier">job</span>,
|
351
|
+
<span class="ruby-identifier">profiling_info</span>)
|
352
|
+
<span class="ruby-identifier">babel_tasks_to_run</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">generate_babel_tasks</span>(<span class="ruby-identifier">job</span>,
|
353
|
+
<span class="ruby-identifier">clouds_to_run_task_on</span>)
|
354
|
+
<span class="ruby-identifier">dispatched_tasks</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">babel_tasks_to_run</span>)
|
355
|
+
<span class="ruby-identifier">tasks</span> <span class="ruby-operator"><<</span> <span class="ruby-constant">ExodusTaskInfo</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">dispatched_tasks</span>)
|
356
|
+
}
|
357
|
+
|
358
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_given_as_hash</span>
|
359
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">tasks</span>[<span class="ruby-value">0</span>]
|
360
|
+
<span class="ruby-keyword">else</span>
|
361
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">tasks</span>
|
362
|
+
<span class="ruby-keyword">end</span>
|
363
|
+
<span class="ruby-keyword">end</span></pre>
|
364
|
+
</div><!-- exodus-source -->
|
365
|
+
|
366
|
+
</div>
|
367
|
+
|
368
|
+
|
300
369
|
|
301
370
|
|
371
|
+
</div><!-- exodus-method -->
|
372
|
+
|
373
|
+
|
374
|
+
<div id="method-i-neptune" class="method-detail ">
|
375
|
+
|
302
376
|
<div class="method-heading">
|
303
377
|
<span class="method-name">neptune</span><span
|
304
|
-
class="method-args">(
|
378
|
+
class="method-args">(jobs)</span>
|
305
379
|
<span class="method-click-advice">click to toggle source</span>
|
306
380
|
</div>
|
307
381
|
|
@@ -317,49 +391,73 @@ vice-versa).</p>
|
|
317
391
|
|
318
392
|
|
319
393
|
|
320
|
-
<div class="method-source-code"
|
321
|
-
|
322
|
-
<
|
323
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 58</span>
|
324
|
-
def neptune(params)
|
394
|
+
<div class="method-source-code" id="neptune-source">
|
395
|
+
<pre><span class="ruby-comment"># File lib/neptune.rb, line 58</span>
|
396
|
+
<span class="ruby-keyword">def</span> <span class="ruby-identifier">neptune</span>(<span class="ruby-identifier">jobs</span>)
|
325
397
|
<span class="ruby-comment"># Kernel.puts "Received a request to run a job."</span>
|
326
398
|
<span class="ruby-comment"># Kernel.puts params[:type]</span>
|
399
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Hash</span>
|
400
|
+
<span class="ruby-identifier">jobs</span> = [<span class="ruby-identifier">jobs</span>]
|
401
|
+
<span class="ruby-keyword">end</span>
|
402
|
+
|
403
|
+
<span class="ruby-identifier">job_data_list</span> = []
|
404
|
+
<span class="ruby-identifier">shadow_ip</span> = <span class="ruby-keyword">nil</span>
|
405
|
+
<span class="ruby-identifier">ssh_args</span> = <span class="ruby-string">""</span>
|
406
|
+
<span class="ruby-identifier">secret</span> = <span class="ruby-string">""</span>
|
407
|
+
<span class="ruby-identifier">controller</span> = <span class="ruby-keyword">nil</span>
|
408
|
+
|
409
|
+
<span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">params</span><span class="ruby-operator">|</span>
|
410
|
+
<span class="ruby-identifier">job_data</span> = <span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">get_job_data</span>(<span class="ruby-identifier">params</span>)
|
411
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">validate_storage_params</span>(<span class="ruby-identifier">job_data</span>)
|
412
|
+
<span class="ruby-comment"># Kernel.puts "job data = #{job_data.inspect}"</span>
|
413
|
+
<span class="ruby-identifier">keyname</span> = <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@keyname"</span>]
|
414
|
+
|
415
|
+
<span class="ruby-identifier">shadow_ip</span> = <span class="ruby-constant">CommonFunctions</span>.<span class="ruby-identifier">get_from_yaml</span>(<span class="ruby-identifier">keyname</span>, <span class="ruby-value">:shadow</span>)
|
416
|
+
<span class="ruby-identifier">secret</span> = <span class="ruby-constant">CommonFunctions</span>.<span class="ruby-identifier">get_secret_key</span>(<span class="ruby-identifier">keyname</span>)
|
417
|
+
<span class="ruby-identifier">ssh_key</span> = <span class="ruby-constant">File</span>.<span class="ruby-identifier">expand_path</span>(<span class="ruby-node">"~/.appscale/#{keyname}.key"</span>)
|
418
|
+
<span class="ruby-identifier">ssh_args</span> = <span class="ruby-node">"-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "</span>
|
419
|
+
|
420
|
+
<span class="ruby-identifier">controller</span> = <span class="ruby-constant">NeptuneManagerClient</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">shadow_ip</span>, <span class="ruby-identifier">secret</span>)
|
421
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">do_preprocessing</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
|
422
|
+
<span class="ruby-identifier">job_data_list</span> <span class="ruby-operator"><<</span> <span class="ruby-identifier">job_data</span>
|
423
|
+
}
|
327
424
|
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
425
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data_list</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">==</span> <span class="ruby-value">1</span>
|
426
|
+
<span class="ruby-keyword">return</span> <span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">job_data_list</span>[<span class="ruby-value">0</span>], <span class="ruby-identifier">ssh_args</span>, <span class="ruby-identifier">shadow_ip</span>,
|
427
|
+
<span class="ruby-identifier">secret</span>)
|
428
|
+
<span class="ruby-keyword">else</span> <span class="ruby-comment"># right now we only support batch run_job operations</span>
|
429
|
+
<span class="ruby-identifier">msg</span> = <span class="ruby-identifier">controller</span>.<span class="ruby-identifier">start_neptune_job</span>(<span class="ruby-identifier">job_data_list</span>)
|
430
|
+
<span class="ruby-identifier">result</span> = {}
|
431
|
+
<span class="ruby-identifier">result</span>[<span class="ruby-value">:msg</span>] = <span class="ruby-identifier">msg</span>
|
432
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">result</span>[<span class="ruby-value">:msg</span>] <span class="ruby-operator">=~</span> <span class="ruby-regexp">%rjob is now running\Z/</span>
|
433
|
+
<span class="ruby-identifier">result</span>[<span class="ruby-value">:result</span>] = <span class="ruby-value">:success</span>
|
434
|
+
<span class="ruby-keyword">else</span>
|
435
|
+
<span class="ruby-identifier">result</span>[<span class="ruby-value">:result</span>] = <span class="ruby-value">:failure</span>
|
436
|
+
<span class="ruby-keyword">end</span>
|
437
|
+
|
438
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">result</span>
|
439
|
+
<span class="ruby-keyword">end</span>
|
440
|
+
<span class="ruby-keyword">end</span></pre>
|
441
|
+
</div><!-- neptune-source -->
|
343
442
|
|
344
443
|
</div>
|
345
444
|
|
346
445
|
|
347
446
|
|
348
447
|
|
349
|
-
</div
|
448
|
+
</div><!-- neptune-method -->
|
350
449
|
|
351
450
|
|
352
|
-
</
|
451
|
+
</section><!-- public-instance-method-details -->
|
353
452
|
|
453
|
+
</section><!-- 5Buntitled-5D -->
|
354
454
|
|
355
|
-
|
455
|
+
</div><!-- documentation -->
|
356
456
|
|
357
|
-
<div id="validator-badges">
|
358
|
-
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
359
|
-
<p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
|
360
|
-
Rdoc Generator</a> 2</small>.</p>
|
361
|
-
</div>
|
362
457
|
|
363
|
-
|
364
|
-
</
|
458
|
+
<footer id="validator-badges">
|
459
|
+
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
460
|
+
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
|
461
|
+
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
462
|
+
</footer>
|
365
463
|
|