neptune 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +4 -0
- data/doc/BabelHelper.html +393 -376
- data/doc/BadConfigurationException.html +121 -127
- data/doc/CommonFunctions.html +237 -265
- data/doc/ExodusHelper.html +820 -0
- data/doc/ExodusTaskInfo.html +263 -0
- data/doc/FileNotFoundException.html +121 -127
- data/doc/NeptuneHelper.html +527 -592
- data/doc/NeptuneManagerClient.html +696 -0
- data/doc/NeptuneManagerException.html +139 -0
- data/doc/Object.html +334 -236
- data/doc/TaskInfo.html +428 -0
- data/doc/created.rid +8 -5
- data/doc/images/add.png +0 -0
- data/doc/images/delete.png +0 -0
- data/doc/images/tag_blue.png +0 -0
- data/doc/images/transparent.png +0 -0
- data/doc/index.html +74 -142
- data/doc/js/darkfish.js +99 -62
- data/doc/js/jquery.js +15 -29
- data/doc/js/navigation.js +142 -0
- data/doc/js/search.js +94 -0
- data/doc/js/search_index.js +1 -0
- data/doc/js/searcher.js +228 -0
- data/doc/table_of_contents.html +226 -0
- data/lib/babel.rb +116 -50
- data/lib/custom_exceptions.rb +2 -2
- data/lib/exodus.rb +311 -0
- data/lib/exodus_task_info.rb +36 -0
- data/lib/neptune.rb +52 -18
- data/lib/{app_controller_client.rb → neptune_manager_client.rb} +54 -38
- data/lib/task_info.rb +155 -0
- data/test/{unit/test_babel.rb → test_babel.rb} +161 -26
- data/test/{unit/test_common_functions.rb → test_common_functions.rb} +1 -1
- data/test/test_exodus.rb +687 -0
- data/test/{unit/test_neptune.rb → test_neptune.rb} +28 -17
- data/test/{unit/test_app_controller_client.rb → test_neptune_manager_client.rb} +15 -16
- data/test/test_task_info.rb +32 -0
- data/test/{unit/ts_all.rb → ts_all.rb} +3 -1
- metadata +30 -34
- data/doc/AppControllerClient.html +0 -702
- data/doc/AppControllerException.html +0 -145
- data/doc/bin/neptune.html +0 -56
- data/doc/js/quicksearch.js +0 -114
- data/doc/js/thickbox-compressed.js +0 -10
- data/doc/lib/app_controller_client_rb.html +0 -60
- data/doc/lib/babel_rb.html +0 -68
- data/doc/lib/common_functions_rb.html +0 -70
- data/doc/lib/custom_exceptions_rb.html +0 -54
- data/doc/lib/neptune_rb.html +0 -60
- data/test/integration/tc_c.rb +0 -57
- data/test/integration/tc_dfsp.rb +0 -37
- data/test/integration/tc_dwssa.rb +0 -38
- data/test/integration/tc_erlang.rb +0 -183
- data/test/integration/tc_mapreduce.rb +0 -282
- data/test/integration/tc_mpi.rb +0 -160
- data/test/integration/tc_storage.rb +0 -209
- data/test/integration/tc_upc.rb +0 -75
- data/test/integration/tc_x10.rb +0 -94
- data/test/integration/test_helper.rb +0 -135
- data/test/integration/ts_neptune.rb +0 -40
@@ -0,0 +1,139 @@
|
|
1
|
+
<!DOCTYPE html>
|
2
|
+
|
3
|
+
<html>
|
4
|
+
<head>
|
5
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
6
|
+
|
7
|
+
<title>class NeptuneManagerException - RDoc Documentation</title>
|
8
|
+
|
9
|
+
<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
|
10
|
+
|
11
|
+
<script type="text/javascript">
|
12
|
+
var rdoc_rel_prefix = "./";
|
13
|
+
</script>
|
14
|
+
|
15
|
+
<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
|
16
|
+
<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
|
17
|
+
<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
|
18
|
+
<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
|
19
|
+
<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
|
20
|
+
<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
|
21
|
+
|
22
|
+
|
23
|
+
<body id="top" class="class">
|
24
|
+
<nav id="metadata">
|
25
|
+
<nav id="home-section" class="section">
|
26
|
+
<h3 class="section-header">
|
27
|
+
<a href="./index.html">Home</a>
|
28
|
+
<a href="./table_of_contents.html#classes">Classes</a>
|
29
|
+
<a href="./table_of_contents.html#methods">Methods</a>
|
30
|
+
</h3>
|
31
|
+
</nav>
|
32
|
+
|
33
|
+
|
34
|
+
<nav id="search-section" class="section project-section" class="initially-hidden">
|
35
|
+
<form action="#" method="get" accept-charset="utf-8">
|
36
|
+
<h3 class="section-header">
|
37
|
+
<input type="text" name="search" placeholder="Search" id="search-field"
|
38
|
+
title="Type to search, Up and Down to navigate, Enter to load">
|
39
|
+
</h3>
|
40
|
+
</form>
|
41
|
+
|
42
|
+
<ul id="search-results" class="initially-hidden"></ul>
|
43
|
+
</nav>
|
44
|
+
|
45
|
+
|
46
|
+
<div id="file-metadata">
|
47
|
+
<nav id="file-list-section" class="section">
|
48
|
+
<h3 class="section-header">Defined In</h3>
|
49
|
+
<ul>
|
50
|
+
<li>lib/custom_exceptions.rb
|
51
|
+
</ul>
|
52
|
+
</nav>
|
53
|
+
|
54
|
+
|
55
|
+
</div>
|
56
|
+
|
57
|
+
<div id="class-metadata">
|
58
|
+
|
59
|
+
<nav id="parent-class-section" class="section">
|
60
|
+
<h3 class="section-header">Parent</h3>
|
61
|
+
|
62
|
+
<p class="link">Exception
|
63
|
+
|
64
|
+
</nav>
|
65
|
+
|
66
|
+
|
67
|
+
|
68
|
+
</div>
|
69
|
+
|
70
|
+
<div id="project-metadata">
|
71
|
+
|
72
|
+
<nav id="classindex-section" class="section project-section">
|
73
|
+
<h3 class="section-header">Class and Module Index</h3>
|
74
|
+
|
75
|
+
<ul class="link-list">
|
76
|
+
|
77
|
+
<li><a href="./BabelHelper.html">BabelHelper</a>
|
78
|
+
|
79
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a>
|
80
|
+
|
81
|
+
<li><a href="./CommonFunctions.html">CommonFunctions</a>
|
82
|
+
|
83
|
+
<li><a href="./ExodusHelper.html">ExodusHelper</a>
|
84
|
+
|
85
|
+
<li><a href="./ExodusTaskInfo.html">ExodusTaskInfo</a>
|
86
|
+
|
87
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a>
|
88
|
+
|
89
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a>
|
90
|
+
|
91
|
+
<li><a href="./NeptuneManagerClient.html">NeptuneManagerClient</a>
|
92
|
+
|
93
|
+
<li><a href="./NeptuneManagerException.html">NeptuneManagerException</a>
|
94
|
+
|
95
|
+
<li><a href="./Object.html">Object</a>
|
96
|
+
|
97
|
+
<li><a href="./TaskInfo.html">TaskInfo</a>
|
98
|
+
|
99
|
+
</ul>
|
100
|
+
</nav>
|
101
|
+
|
102
|
+
</div>
|
103
|
+
</nav>
|
104
|
+
|
105
|
+
<div id="documentation">
|
106
|
+
<h1 class="class">class NeptuneManagerException</h1>
|
107
|
+
|
108
|
+
<div id="description" class="description">
|
109
|
+
|
110
|
+
<p>A special class of exceptions that are thrown whenever the NeptuneManager
|
111
|
+
experiences an unexpected result.</p>
|
112
|
+
|
113
|
+
</div><!-- description -->
|
114
|
+
|
115
|
+
|
116
|
+
|
117
|
+
|
118
|
+
<section id="5Buntitled-5D" class="documentation-section">
|
119
|
+
|
120
|
+
|
121
|
+
|
122
|
+
|
123
|
+
|
124
|
+
|
125
|
+
|
126
|
+
|
127
|
+
<!-- Methods -->
|
128
|
+
|
129
|
+
</section><!-- 5Buntitled-5D -->
|
130
|
+
|
131
|
+
</div><!-- documentation -->
|
132
|
+
|
133
|
+
|
134
|
+
<footer id="validator-badges">
|
135
|
+
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
136
|
+
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
|
137
|
+
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
138
|
+
</footer>
|
139
|
+
|
data/doc/Object.html
CHANGED
@@ -1,239 +1,233 @@
|
|
1
|
-
|
2
|
-
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
3
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
4
|
-
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
5
|
-
<head>
|
6
|
-
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
|
7
|
-
|
8
|
-
<title>Class: Object</title>
|
9
|
-
|
10
|
-
<link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
|
11
|
-
|
12
|
-
<script src="./js/jquery.js" type="text/javascript"
|
13
|
-
charset="utf-8"></script>
|
14
|
-
<script src="./js/thickbox-compressed.js" type="text/javascript"
|
15
|
-
charset="utf-8"></script>
|
16
|
-
<script src="./js/quicksearch.js" type="text/javascript"
|
17
|
-
charset="utf-8"></script>
|
18
|
-
<script src="./js/darkfish.js" type="text/javascript"
|
19
|
-
charset="utf-8"></script>
|
20
|
-
|
21
|
-
</head>
|
22
|
-
<body class="class">
|
23
|
-
|
24
|
-
<div id="metadata">
|
25
|
-
<div id="home-metadata">
|
26
|
-
<div id="home-section" class="section">
|
27
|
-
<h3 class="section-header">
|
28
|
-
<a href="./index.html">Home</a>
|
29
|
-
<a href="./index.html#classes">Classes</a>
|
30
|
-
<a href="./index.html#methods">Methods</a>
|
31
|
-
</h3>
|
32
|
-
</div>
|
33
|
-
</div>
|
34
|
-
|
35
|
-
<div id="file-metadata">
|
36
|
-
<div id="file-list-section" class="section">
|
37
|
-
<h3 class="section-header">In Files</h3>
|
38
|
-
<div class="section-body">
|
39
|
-
<ul>
|
40
|
-
|
41
|
-
<li><a href="./lib/app_controller_client_rb.html?TB_iframe=true&height=550&width=785"
|
42
|
-
class="thickbox" title="lib/app_controller_client.rb">lib/app_controller_client.rb</a></li>
|
43
|
-
|
44
|
-
<li><a href="./lib/babel_rb.html?TB_iframe=true&height=550&width=785"
|
45
|
-
class="thickbox" title="lib/babel.rb">lib/babel.rb</a></li>
|
46
|
-
|
47
|
-
<li><a href="./lib/neptune_rb.html?TB_iframe=true&height=550&width=785"
|
48
|
-
class="thickbox" title="lib/neptune.rb">lib/neptune.rb</a></li>
|
49
|
-
|
50
|
-
</ul>
|
51
|
-
</div>
|
52
|
-
</div>
|
53
|
-
|
54
|
-
|
55
|
-
</div>
|
1
|
+
<!DOCTYPE html>
|
56
2
|
|
57
|
-
|
3
|
+
<html>
|
4
|
+
<head>
|
5
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
|
6
|
+
|
7
|
+
<title>class Object - RDoc Documentation</title>
|
8
|
+
|
9
|
+
<link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet">
|
10
|
+
|
11
|
+
<script type="text/javascript">
|
12
|
+
var rdoc_rel_prefix = "./";
|
13
|
+
</script>
|
14
|
+
|
15
|
+
<script type="text/javascript" charset="utf-8" src="./js/jquery.js"></script>
|
16
|
+
<script type="text/javascript" charset="utf-8" src="./js/navigation.js"></script>
|
17
|
+
<script type="text/javascript" charset="utf-8" src="./js/search_index.js"></script>
|
18
|
+
<script type="text/javascript" charset="utf-8" src="./js/search.js"></script>
|
19
|
+
<script type="text/javascript" charset="utf-8" src="./js/searcher.js"></script>
|
20
|
+
<script type="text/javascript" charset="utf-8" src="./js/darkfish.js"></script>
|
21
|
+
|
22
|
+
|
23
|
+
<body id="top" class="class">
|
24
|
+
<nav id="metadata">
|
25
|
+
<nav id="home-section" class="section">
|
26
|
+
<h3 class="section-header">
|
27
|
+
<a href="./index.html">Home</a>
|
28
|
+
<a href="./table_of_contents.html#classes">Classes</a>
|
29
|
+
<a href="./table_of_contents.html#methods">Methods</a>
|
30
|
+
</h3>
|
31
|
+
</nav>
|
32
|
+
|
33
|
+
|
34
|
+
<nav id="search-section" class="section project-section" class="initially-hidden">
|
35
|
+
<form action="#" method="get" accept-charset="utf-8">
|
36
|
+
<h3 class="section-header">
|
37
|
+
<input type="text" name="search" placeholder="Search" id="search-field"
|
38
|
+
title="Type to search, Up and Down to navigate, Enter to load">
|
39
|
+
</h3>
|
40
|
+
</form>
|
41
|
+
|
42
|
+
<ul id="search-results" class="initially-hidden"></ul>
|
43
|
+
</nav>
|
44
|
+
|
45
|
+
|
46
|
+
<div id="file-metadata">
|
47
|
+
<nav id="file-list-section" class="section">
|
48
|
+
<h3 class="section-header">Defined In</h3>
|
49
|
+
<ul>
|
50
|
+
<li>lib/exodus.rb
|
51
|
+
<li>lib/babel.rb
|
52
|
+
<li>lib/neptune_manager_client.rb
|
53
|
+
<li>lib/neptune.rb
|
54
|
+
</ul>
|
55
|
+
</nav>
|
58
56
|
|
59
|
-
|
60
|
-
|
61
|
-
<div id="parent-class-section" class="section">
|
62
|
-
<h3 class="section-header">Parent</h3>
|
63
|
-
|
64
|
-
<p class="link"></p>
|
65
|
-
|
66
|
-
</div>
|
67
|
-
|
57
|
+
|
58
|
+
</div>
|
68
59
|
|
69
|
-
|
70
|
-
|
60
|
+
<div id="class-metadata">
|
61
|
+
|
62
|
+
<nav id="parent-class-section" class="section">
|
63
|
+
<h3 class="section-header">Parent</h3>
|
64
|
+
|
65
|
+
<p class="link">
|
66
|
+
|
67
|
+
</nav>
|
71
68
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
<ul class="link-list">
|
77
|
-
|
78
|
-
<li><a href="#method-i-babel">#babel</a></li>
|
79
|
-
|
80
|
-
<li><a href="#method-i-neptune">#neptune</a></li>
|
81
|
-
|
82
|
-
</ul>
|
83
|
-
</div>
|
84
|
-
|
69
|
+
|
70
|
+
<!-- Method Quickref -->
|
71
|
+
<nav id="method-list-section" class="section">
|
72
|
+
<h3 class="section-header">Methods</h3>
|
85
73
|
|
86
|
-
|
87
|
-
|
88
|
-
</
|
74
|
+
<ul class="link-list">
|
75
|
+
|
76
|
+
<li><a href="#method-i-babel">#babel</a>
|
77
|
+
|
78
|
+
<li><a href="#method-i-exodus">#exodus</a>
|
79
|
+
|
80
|
+
<li><a href="#method-i-neptune">#neptune</a>
|
81
|
+
|
82
|
+
</ul>
|
83
|
+
</nav>
|
89
84
|
|
90
|
-
|
91
|
-
|
92
|
-
|
85
|
+
</div>
|
93
86
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
title="show/hide quicksearch" /></span></h3>
|
99
|
-
<form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
|
100
|
-
<fieldset>
|
101
|
-
<legend>Quicksearch</legend>
|
102
|
-
<input type="text" name="quicksearch" value=""
|
103
|
-
class="quicksearch-field" />
|
104
|
-
</fieldset>
|
105
|
-
</form>
|
87
|
+
<div id="project-metadata">
|
88
|
+
|
89
|
+
<nav id="classindex-section" class="section project-section">
|
90
|
+
<h3 class="section-header">Class and Module Index</h3>
|
106
91
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
92
|
+
<ul class="link-list">
|
93
|
+
|
94
|
+
<li><a href="./BabelHelper.html">BabelHelper</a>
|
95
|
+
|
96
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a>
|
97
|
+
|
98
|
+
<li><a href="./CommonFunctions.html">CommonFunctions</a>
|
99
|
+
|
100
|
+
<li><a href="./ExodusHelper.html">ExodusHelper</a>
|
101
|
+
|
102
|
+
<li><a href="./ExodusTaskInfo.html">ExodusTaskInfo</a>
|
103
|
+
|
104
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a>
|
105
|
+
|
106
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a>
|
107
|
+
|
108
|
+
<li><a href="./NeptuneManagerClient.html">NeptuneManagerClient</a>
|
109
|
+
|
110
|
+
<li><a href="./NeptuneManagerException.html">NeptuneManagerException</a>
|
111
|
+
|
112
|
+
<li><a href="./Object.html">Object</a>
|
113
|
+
|
114
|
+
<li><a href="./TaskInfo.html">TaskInfo</a>
|
115
|
+
|
116
|
+
</ul>
|
117
|
+
</nav>
|
128
118
|
|
129
|
-
|
130
|
-
</div>
|
131
119
|
</div>
|
120
|
+
</nav>
|
132
121
|
|
133
|
-
|
134
|
-
|
122
|
+
<div id="documentation">
|
123
|
+
<h1 class="class">class Object</h1>
|
135
124
|
|
136
|
-
|
137
|
-
|
125
|
+
<div id="description" class="description">
|
126
|
+
|
138
127
|
<p>Since we’re monkeypatching <a href="Object.html">Object</a> to add
|
139
128
|
neptune() and babel(), a short blurb is necessary here to make rdoc happy.</p>
|
140
129
|
|
141
|
-
|
130
|
+
</div><!-- description -->
|
142
131
|
|
143
|
-
|
132
|
+
|
133
|
+
|
134
|
+
|
135
|
+
<section id="5Buntitled-5D" class="documentation-section">
|
144
136
|
|
145
|
-
|
137
|
+
|
138
|
+
|
139
|
+
|
140
|
+
|
141
|
+
<!-- Constants -->
|
142
|
+
<section id="constants-list" class="section">
|
146
143
|
<h3 class="section-header">Constants</h3>
|
147
144
|
<dl>
|
148
145
|
|
149
|
-
<dt
|
146
|
+
<dt id="ALLOWED_JOB_TYPES">ALLOWED_JOB_TYPES
|
150
147
|
|
151
|
-
<dd class="description"><p>
|
152
|
-
sent over the network: for this first version we don’t want these calls
|
153
|
-
to endlessly timeout and retry, so as a hack, just don’t let them
|
154
|
-
timeout. The next version should replace this and properly timeout and not
|
155
|
-
use long calls unless necessary.</p></dd>
|
148
|
+
<dd class="description"><p>A list of all the Neptune job types that we support</p>
|
156
149
|
|
157
150
|
|
158
|
-
<dt
|
151
|
+
<dt id="ALLOWED_STORAGE_TYPES">ALLOWED_STORAGE_TYPES
|
159
152
|
|
160
|
-
<dd class="description"><p>
|
161
|
-
|
153
|
+
<dd class="description"><p>A list of storage mechanisms that we can use to store and retrieve data to
|
154
|
+
for Neptune jobs.</p>
|
162
155
|
|
163
156
|
|
164
|
-
<dt
|
157
|
+
<dt id="DOES_NOT_EXIST">DOES_NOT_EXIST
|
165
158
|
|
166
159
|
<dd class="description"><p>The constant string that a Neptune output job returns if the output does
|
167
|
-
not yet exist.</p
|
160
|
+
not yet exist.</p>
|
168
161
|
|
169
162
|
|
170
|
-
<dt
|
163
|
+
<dt id="JOB_TYPE_NOT_ALLOWED">JOB_TYPE_NOT_ALLOWED
|
171
164
|
|
172
|
-
<dd class="description"><p>The
|
173
|
-
requests. An exponential backoff is used with this value as the starting
|
174
|
-
sleep time.</p></dd>
|
165
|
+
<dd class="description"><p>The string to display for disallowed job types.</p>
|
175
166
|
|
176
167
|
|
177
|
-
<dt
|
168
|
+
<dt id="MAX_SLEEP_TIME">MAX_SLEEP_TIME
|
178
169
|
|
179
170
|
<dd class="description"><p>The maximum amount of time that we should sleep to, when waiting for output
|
180
|
-
job requests.</p
|
171
|
+
job requests.</p>
|
181
172
|
|
182
173
|
|
183
|
-
<dt
|
174
|
+
<dt id="NEEDS_BUCKET_INFO">NEEDS_BUCKET_INFO
|
184
175
|
|
185
|
-
<dd class="description"><p>
|
176
|
+
<dd class="description"><p>If the user doesn’t give us enough info to infer what bucket we should
|
177
|
+
place their code in, this message is displayed and execution aborts.</p>
|
186
178
|
|
187
179
|
|
188
|
-
<dt
|
180
|
+
<dt id="NEED_PREPROCESSING">NEED_PREPROCESSING
|
189
181
|
|
190
|
-
<dd class="description"><p>
|
182
|
+
<dd class="description"><p>A list of jobs that require some kind of work to be done before the actual
|
183
|
+
computation can be performed.</p>
|
191
184
|
|
192
185
|
|
193
|
-
<dt
|
186
|
+
<dt id="NO_NODES_NEEDED">NO_NODES_NEEDED
|
194
187
|
|
195
188
|
<dd class="description"><p>A list of Neptune jobs that do not require nodes to be spawned up for
|
196
|
-
computation</p
|
189
|
+
computation</p>
|
197
190
|
|
198
191
|
|
199
|
-
<dt
|
192
|
+
<dt id="NO_OUTPUT_NEEDED">NO_OUTPUT_NEEDED
|
200
193
|
|
201
194
|
<dd class="description"><p>A list of Neptune jobs that do not require the output to be specified
|
202
|
-
beforehand</p
|
195
|
+
beforehand</p>
|
203
196
|
|
204
197
|
|
205
|
-
<dt
|
198
|
+
<dt id="NO_TIMEOUT">NO_TIMEOUT
|
206
199
|
|
207
|
-
<dd class="description"><p>
|
208
|
-
for
|
200
|
+
<dd class="description"><p>Sometimes SOAP calls take a long time if large amounts of data are being
|
201
|
+
sent over the network: for this first version we don’t want these calls
|
202
|
+
to endlessly timeout and retry, so as a hack, just don’t let them
|
203
|
+
timeout. The next version should replace this and properly timeout and not
|
204
|
+
use long calls unless necessary.</p>
|
209
205
|
|
210
206
|
|
211
|
-
<dt
|
207
|
+
<dt id="SLEEP_TIME">SLEEP_TIME
|
212
208
|
|
213
|
-
<dd class="description"><p>
|
214
|
-
|
209
|
+
<dd class="description"><p>The initial amount of time, in seconds, to sleep between output job
|
210
|
+
requests. An exponential backoff is used with this value as the starting
|
211
|
+
sleep time.</p>
|
215
212
|
|
216
213
|
|
217
214
|
</dl>
|
218
|
-
</
|
215
|
+
</section>
|
219
216
|
|
220
217
|
|
221
|
-
<!-- Attributes -->
|
222
218
|
|
223
219
|
|
224
220
|
<!-- Methods -->
|
225
221
|
|
226
|
-
|
222
|
+
<section id="public-instance-5Buntitled-5D-method-details" class="method-section section">
|
227
223
|
<h3 class="section-header">Public Instance Methods</h3>
|
228
224
|
|
229
225
|
|
230
|
-
<div id="babel
|
231
|
-
<a name="method-i-babel"></a>
|
232
|
-
|
226
|
+
<div id="method-i-babel" class="method-detail ">
|
233
227
|
|
234
228
|
<div class="method-heading">
|
235
229
|
<span class="method-name">babel</span><span
|
236
|
-
class="method-args">(
|
230
|
+
class="method-args">(jobs)</span>
|
237
231
|
<span class="method-click-advice">click to toggle source</span>
|
238
232
|
</div>
|
239
233
|
|
@@ -247,61 +241,141 @@ handles this.</p>
|
|
247
241
|
|
248
242
|
|
249
243
|
|
250
|
-
<div class="method-source-code"
|
251
|
-
|
252
|
-
<
|
253
|
-
<span class="ruby-comment"># File lib/babel.rb, line 43</span>
|
254
|
-
def babel(params)
|
244
|
+
<div class="method-source-code" id="babel-source">
|
245
|
+
<pre><span class="ruby-comment"># File lib/babel.rb, line 44</span>
|
246
|
+
<span class="ruby-keyword">def</span> <span class="ruby-identifier">babel</span>(<span class="ruby-identifier">jobs</span>)
|
255
247
|
<span class="ruby-comment"># Since this whole function should run asynchronously, we run it as a future.</span>
|
256
248
|
<span class="ruby-comment"># It automatically starts running in a new thread, and attempting to get the</span>
|
257
249
|
<span class="ruby-comment"># value of what this returns causes it to block until the job completes.</span>
|
258
|
-
future {
|
259
|
-
if
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
<span class="ruby-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
<span class="ruby-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
250
|
+
<span class="ruby-comment">#future {</span>
|
251
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Hash</span>
|
252
|
+
<span class="ruby-identifier">was_one_job</span> = <span class="ruby-keyword">true</span>
|
253
|
+
<span class="ruby-identifier">jobs</span> = [<span class="ruby-identifier">jobs</span>]
|
254
|
+
<span class="ruby-keyword">else</span>
|
255
|
+
<span class="ruby-identifier">was_one_job</span> = <span class="ruby-keyword">false</span>
|
256
|
+
<span class="ruby-keyword">end</span>
|
257
|
+
|
258
|
+
<span class="ruby-identifier">job_data_list</span> = []
|
259
|
+
<span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">params</span><span class="ruby-operator">|</span>
|
260
|
+
<span class="ruby-identifier">job_data</span> = <span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">convert_from_neptune_params</span>(<span class="ruby-identifier">params</span>)
|
261
|
+
<span class="ruby-identifier">job_data</span>[<span class="ruby-string">'@metadata_info'</span>] = {<span class="ruby-string">'time_to_store_inputs'</span> =<span class="ruby-operator">></span> <span class="ruby-value">0.0</span>}
|
262
|
+
|
263
|
+
<span class="ruby-comment"># Add in S3 storage parameters</span>
|
264
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">validate_storage_params</span>(<span class="ruby-identifier">job_data</span>)
|
265
|
+
|
266
|
+
<span class="ruby-comment"># :code is the only required parameter</span>
|
267
|
+
<span class="ruby-comment"># everything else can use default vals</span>
|
268
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">require_param</span>(<span class="ruby-string">"@code"</span>, <span class="ruby-identifier">job_data</span>)
|
269
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">check_output_files</span>(<span class="ruby-identifier">job_data</span>)
|
270
|
+
|
271
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@is_remote"</span>]
|
272
|
+
<span class="ruby-comment">#BabelHelper.validate_inputs(job_data)</span>
|
273
|
+
<span class="ruby-keyword">else</span>
|
274
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">put_code</span>(<span class="ruby-identifier">job_data</span>)
|
275
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">put_inputs</span>(<span class="ruby-identifier">job_data</span>)
|
276
|
+
<span class="ruby-keyword">end</span>
|
277
|
+
|
278
|
+
<span class="ruby-identifier">job_data_list</span> <span class="ruby-operator"><<</span> <span class="ruby-identifier">job_data</span>
|
279
|
+
}
|
280
|
+
|
281
|
+
<span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">job_data_list</span>)
|
282
|
+
|
283
|
+
<span class="ruby-comment"># Return an object to the user that has all the information about their</span>
|
284
|
+
<span class="ruby-comment"># task, including its standard out, err, debugging info, and profiling</span>
|
285
|
+
<span class="ruby-comment"># info. Since the job may not be done when the user asks for this info,</span>
|
286
|
+
<span class="ruby-comment"># its the responsibility of TaskInfo objects to block until that info</span>
|
287
|
+
<span class="ruby-comment"># is ready. We don't explicitly return the TaskInfo object, because it's</span>
|
288
|
+
<span class="ruby-comment"># in a Future block - it will automatically return whatever the last</span>
|
289
|
+
<span class="ruby-comment"># statement returns.</span>
|
290
|
+
<span class="ruby-identifier">tasks</span> = []
|
291
|
+
<span class="ruby-identifier">job_data_list</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">job_data</span><span class="ruby-operator">|</span>
|
292
|
+
<span class="ruby-identifier">tasks</span> <span class="ruby-operator"><<</span> <span class="ruby-constant">TaskInfo</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">job_data</span>)
|
293
|
+
}
|
294
|
+
|
295
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">was_one_job</span>
|
296
|
+
<span class="ruby-identifier">tasks</span>[<span class="ruby-value">0</span>]
|
297
|
+
<span class="ruby-keyword">else</span>
|
298
|
+
<span class="ruby-identifier">tasks</span>
|
299
|
+
<span class="ruby-keyword">end</span>
|
300
|
+
<span class="ruby-comment">#}</span>
|
301
|
+
<span class="ruby-keyword">end</span></pre>
|
302
|
+
</div><!-- babel-source -->
|
289
303
|
|
290
304
|
</div>
|
291
305
|
|
292
306
|
|
293
307
|
|
294
308
|
|
295
|
-
</div
|
309
|
+
</div><!-- babel-method -->
|
296
310
|
|
297
311
|
|
298
|
-
<div id="
|
299
|
-
|
312
|
+
<div id="method-i-exodus" class="method-detail ">
|
313
|
+
|
314
|
+
<div class="method-heading">
|
315
|
+
<span class="method-name">exodus</span><span
|
316
|
+
class="method-args">(jobs)</span>
|
317
|
+
<span class="method-click-advice">click to toggle source</span>
|
318
|
+
</div>
|
319
|
+
|
320
|
+
|
321
|
+
<div class="method-description">
|
322
|
+
|
323
|
+
<p>Exodus provides further improvements to Babel. Instead of making users tell
|
324
|
+
us what compute, storage, and queue services they want to use (required for
|
325
|
+
babel calls), Exodus will automatically handle this for us. Callers need to
|
326
|
+
specify what clouds their job can run over, and Exodus will automatically
|
327
|
+
select the best cloud for their job and run it there.</p>
|
328
|
+
|
329
|
+
|
330
|
+
|
331
|
+
<div class="method-source-code" id="exodus-source">
|
332
|
+
<pre><span class="ruby-comment"># File lib/exodus.rb, line 15</span>
|
333
|
+
<span class="ruby-keyword">def</span> <span class="ruby-identifier">exodus</span>(<span class="ruby-identifier">jobs</span>)
|
334
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Hash</span>
|
335
|
+
<span class="ruby-identifier">job_given_as_hash</span> = <span class="ruby-keyword">true</span>
|
336
|
+
<span class="ruby-identifier">jobs</span> = [<span class="ruby-identifier">jobs</span>]
|
337
|
+
<span class="ruby-keyword">elsif</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Array</span>
|
338
|
+
<span class="ruby-identifier">job_given_as_hash</span> = <span class="ruby-keyword">false</span>
|
339
|
+
<span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">ensure_all_jobs_are_hashes</span>(<span class="ruby-identifier">jobs</span>)
|
340
|
+
<span class="ruby-keyword">else</span>
|
341
|
+
<span class="ruby-identifier">raise</span> <span class="ruby-constant">BadConfigurationException</span>.<span class="ruby-identifier">new</span>(<span class="ruby-node">"jobs was a #{jobs.class}, which "</span> <span class="ruby-operator">+</span>
|
342
|
+
<span class="ruby-string">"is not an acceptable class type"</span>)
|
343
|
+
<span class="ruby-keyword">end</span>
|
344
|
+
|
345
|
+
<span class="ruby-identifier">tasks</span> = []
|
346
|
+
|
347
|
+
<span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">job</span><span class="ruby-operator">|</span>
|
348
|
+
<span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">ensure_all_params_are_present</span>(<span class="ruby-identifier">job</span>)
|
349
|
+
<span class="ruby-identifier">profiling_info</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">get_profiling_info</span>(<span class="ruby-identifier">job</span>)
|
350
|
+
<span class="ruby-identifier">clouds_to_run_task_on</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">get_clouds_to_run_task_on</span>(<span class="ruby-identifier">job</span>,
|
351
|
+
<span class="ruby-identifier">profiling_info</span>)
|
352
|
+
<span class="ruby-identifier">babel_tasks_to_run</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">generate_babel_tasks</span>(<span class="ruby-identifier">job</span>,
|
353
|
+
<span class="ruby-identifier">clouds_to_run_task_on</span>)
|
354
|
+
<span class="ruby-identifier">dispatched_tasks</span> = <span class="ruby-constant">ExodusHelper</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">babel_tasks_to_run</span>)
|
355
|
+
<span class="ruby-identifier">tasks</span> <span class="ruby-operator"><<</span> <span class="ruby-constant">ExodusTaskInfo</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">dispatched_tasks</span>)
|
356
|
+
}
|
357
|
+
|
358
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_given_as_hash</span>
|
359
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">tasks</span>[<span class="ruby-value">0</span>]
|
360
|
+
<span class="ruby-keyword">else</span>
|
361
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">tasks</span>
|
362
|
+
<span class="ruby-keyword">end</span>
|
363
|
+
<span class="ruby-keyword">end</span></pre>
|
364
|
+
</div><!-- exodus-source -->
|
365
|
+
|
366
|
+
</div>
|
367
|
+
|
368
|
+
|
300
369
|
|
301
370
|
|
371
|
+
</div><!-- exodus-method -->
|
372
|
+
|
373
|
+
|
374
|
+
<div id="method-i-neptune" class="method-detail ">
|
375
|
+
|
302
376
|
<div class="method-heading">
|
303
377
|
<span class="method-name">neptune</span><span
|
304
|
-
class="method-args">(
|
378
|
+
class="method-args">(jobs)</span>
|
305
379
|
<span class="method-click-advice">click to toggle source</span>
|
306
380
|
</div>
|
307
381
|
|
@@ -317,49 +391,73 @@ vice-versa).</p>
|
|
317
391
|
|
318
392
|
|
319
393
|
|
320
|
-
<div class="method-source-code"
|
321
|
-
|
322
|
-
<
|
323
|
-
<span class="ruby-comment"># File lib/neptune.rb, line 58</span>
|
324
|
-
def neptune(params)
|
394
|
+
<div class="method-source-code" id="neptune-source">
|
395
|
+
<pre><span class="ruby-comment"># File lib/neptune.rb, line 58</span>
|
396
|
+
<span class="ruby-keyword">def</span> <span class="ruby-identifier">neptune</span>(<span class="ruby-identifier">jobs</span>)
|
325
397
|
<span class="ruby-comment"># Kernel.puts "Received a request to run a job."</span>
|
326
398
|
<span class="ruby-comment"># Kernel.puts params[:type]</span>
|
399
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Hash</span>
|
400
|
+
<span class="ruby-identifier">jobs</span> = [<span class="ruby-identifier">jobs</span>]
|
401
|
+
<span class="ruby-keyword">end</span>
|
402
|
+
|
403
|
+
<span class="ruby-identifier">job_data_list</span> = []
|
404
|
+
<span class="ruby-identifier">shadow_ip</span> = <span class="ruby-keyword">nil</span>
|
405
|
+
<span class="ruby-identifier">ssh_args</span> = <span class="ruby-string">""</span>
|
406
|
+
<span class="ruby-identifier">secret</span> = <span class="ruby-string">""</span>
|
407
|
+
<span class="ruby-identifier">controller</span> = <span class="ruby-keyword">nil</span>
|
408
|
+
|
409
|
+
<span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">params</span><span class="ruby-operator">|</span>
|
410
|
+
<span class="ruby-identifier">job_data</span> = <span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">get_job_data</span>(<span class="ruby-identifier">params</span>)
|
411
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">validate_storage_params</span>(<span class="ruby-identifier">job_data</span>)
|
412
|
+
<span class="ruby-comment"># Kernel.puts "job data = #{job_data.inspect}"</span>
|
413
|
+
<span class="ruby-identifier">keyname</span> = <span class="ruby-identifier">job_data</span>[<span class="ruby-string">"@keyname"</span>]
|
414
|
+
|
415
|
+
<span class="ruby-identifier">shadow_ip</span> = <span class="ruby-constant">CommonFunctions</span>.<span class="ruby-identifier">get_from_yaml</span>(<span class="ruby-identifier">keyname</span>, <span class="ruby-value">:shadow</span>)
|
416
|
+
<span class="ruby-identifier">secret</span> = <span class="ruby-constant">CommonFunctions</span>.<span class="ruby-identifier">get_secret_key</span>(<span class="ruby-identifier">keyname</span>)
|
417
|
+
<span class="ruby-identifier">ssh_key</span> = <span class="ruby-constant">File</span>.<span class="ruby-identifier">expand_path</span>(<span class="ruby-node">"~/.appscale/#{keyname}.key"</span>)
|
418
|
+
<span class="ruby-identifier">ssh_args</span> = <span class="ruby-node">"-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "</span>
|
419
|
+
|
420
|
+
<span class="ruby-identifier">controller</span> = <span class="ruby-constant">NeptuneManagerClient</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">shadow_ip</span>, <span class="ruby-identifier">secret</span>)
|
421
|
+
<span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">do_preprocessing</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
|
422
|
+
<span class="ruby-identifier">job_data_list</span> <span class="ruby-operator"><<</span> <span class="ruby-identifier">job_data</span>
|
423
|
+
}
|
327
424
|
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
425
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">job_data_list</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">==</span> <span class="ruby-value">1</span>
|
426
|
+
<span class="ruby-keyword">return</span> <span class="ruby-constant">NeptuneHelper</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">job_data_list</span>[<span class="ruby-value">0</span>], <span class="ruby-identifier">ssh_args</span>, <span class="ruby-identifier">shadow_ip</span>,
|
427
|
+
<span class="ruby-identifier">secret</span>)
|
428
|
+
<span class="ruby-keyword">else</span> <span class="ruby-comment"># right now we only support batch run_job operations</span>
|
429
|
+
<span class="ruby-identifier">msg</span> = <span class="ruby-identifier">controller</span>.<span class="ruby-identifier">start_neptune_job</span>(<span class="ruby-identifier">job_data_list</span>)
|
430
|
+
<span class="ruby-identifier">result</span> = {}
|
431
|
+
<span class="ruby-identifier">result</span>[<span class="ruby-value">:msg</span>] = <span class="ruby-identifier">msg</span>
|
432
|
+
<span class="ruby-keyword">if</span> <span class="ruby-identifier">result</span>[<span class="ruby-value">:msg</span>] <span class="ruby-operator">=~</span> <span class="ruby-regexp">%rjob is now running\Z/</span>
|
433
|
+
<span class="ruby-identifier">result</span>[<span class="ruby-value">:result</span>] = <span class="ruby-value">:success</span>
|
434
|
+
<span class="ruby-keyword">else</span>
|
435
|
+
<span class="ruby-identifier">result</span>[<span class="ruby-value">:result</span>] = <span class="ruby-value">:failure</span>
|
436
|
+
<span class="ruby-keyword">end</span>
|
437
|
+
|
438
|
+
<span class="ruby-keyword">return</span> <span class="ruby-identifier">result</span>
|
439
|
+
<span class="ruby-keyword">end</span>
|
440
|
+
<span class="ruby-keyword">end</span></pre>
|
441
|
+
</div><!-- neptune-source -->
|
343
442
|
|
344
443
|
</div>
|
345
444
|
|
346
445
|
|
347
446
|
|
348
447
|
|
349
|
-
</div
|
448
|
+
</div><!-- neptune-method -->
|
350
449
|
|
351
450
|
|
352
|
-
</
|
451
|
+
</section><!-- public-instance-method-details -->
|
353
452
|
|
453
|
+
</section><!-- 5Buntitled-5D -->
|
354
454
|
|
355
|
-
|
455
|
+
</div><!-- documentation -->
|
356
456
|
|
357
|
-
<div id="validator-badges">
|
358
|
-
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
359
|
-
<p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
|
360
|
-
Rdoc Generator</a> 2</small>.</p>
|
361
|
-
</div>
|
362
457
|
|
363
|
-
|
364
|
-
</
|
458
|
+
<footer id="validator-badges">
|
459
|
+
<p><a href="http://validator.w3.org/check/referer">[Validate]</a>
|
460
|
+
<p>Generated by <a href="https://github.com/rdoc/rdoc">RDoc</a> 3.12.
|
461
|
+
<p>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish Rdoc Generator</a> 3.
|
462
|
+
</footer>
|
365
463
|
|