neptune 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +4 -0
- data/bin/neptune +7 -0
- data/doc/AppControllerClient.html +113 -22
- data/doc/{Kernel.html → AppControllerException.html} +23 -55
- data/doc/BabelHelper.html +707 -0
- data/doc/BadConfigurationException.html +142 -0
- data/doc/CommonFunctions.html +121 -33
- data/doc/FileNotFoundException.html +142 -0
- data/doc/NeptuneHelper.html +1102 -0
- data/doc/Object.html +94 -749
- data/doc/bin/neptune.html +3 -1
- data/doc/created.rid +7 -5
- data/doc/index.html +73 -25
- data/doc/lib/app_controller_client_rb.html +1 -1
- data/doc/lib/babel_rb.html +68 -0
- data/doc/lib/common_functions_rb.html +3 -1
- data/doc/lib/custom_exceptions_rb.html +54 -0
- data/doc/lib/neptune_rb.html +3 -1
- data/lib/app_controller_client.rb +28 -10
- data/lib/babel.rb +260 -0
- data/lib/common_functions.rb +42 -28
- data/lib/custom_exceptions.rb +10 -0
- data/lib/neptune.rb +371 -304
- data/test/unit/test_app_controller_client.rb +9 -9
- data/test/unit/test_babel.rb +154 -0
- data/test/unit/test_common_functions.rb +39 -75
- data/test/unit/test_neptune.rb +168 -76
- data/test/unit/ts_all.rb +5 -0
- metadata +32 -8
|
@@ -0,0 +1,1102 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
|
3
|
+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
|
4
|
+
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
|
5
|
+
<head>
|
|
6
|
+
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
|
|
7
|
+
|
|
8
|
+
<title>Module: NeptuneHelper</title>
|
|
9
|
+
|
|
10
|
+
<link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
|
|
11
|
+
|
|
12
|
+
<script src="./js/jquery.js" type="text/javascript"
|
|
13
|
+
charset="utf-8"></script>
|
|
14
|
+
<script src="./js/thickbox-compressed.js" type="text/javascript"
|
|
15
|
+
charset="utf-8"></script>
|
|
16
|
+
<script src="./js/quicksearch.js" type="text/javascript"
|
|
17
|
+
charset="utf-8"></script>
|
|
18
|
+
<script src="./js/darkfish.js" type="text/javascript"
|
|
19
|
+
charset="utf-8"></script>
|
|
20
|
+
|
|
21
|
+
</head>
|
|
22
|
+
<body class="module">
|
|
23
|
+
|
|
24
|
+
<div id="metadata">
|
|
25
|
+
<div id="home-metadata">
|
|
26
|
+
<div id="home-section" class="section">
|
|
27
|
+
<h3 class="section-header">
|
|
28
|
+
<a href="./index.html">Home</a>
|
|
29
|
+
<a href="./index.html#classes">Classes</a>
|
|
30
|
+
<a href="./index.html#methods">Methods</a>
|
|
31
|
+
</h3>
|
|
32
|
+
</div>
|
|
33
|
+
</div>
|
|
34
|
+
|
|
35
|
+
<div id="file-metadata">
|
|
36
|
+
<div id="file-list-section" class="section">
|
|
37
|
+
<h3 class="section-header">In Files</h3>
|
|
38
|
+
<div class="section-body">
|
|
39
|
+
<ul>
|
|
40
|
+
|
|
41
|
+
<li><a href="./lib/neptune_rb.html?TB_iframe=true&height=550&width=785"
|
|
42
|
+
class="thickbox" title="lib/neptune.rb">lib/neptune.rb</a></li>
|
|
43
|
+
|
|
44
|
+
</ul>
|
|
45
|
+
</div>
|
|
46
|
+
</div>
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
</div>
|
|
50
|
+
|
|
51
|
+
<div id="class-metadata">
|
|
52
|
+
|
|
53
|
+
<!-- Parent Class -->
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
<!-- Namespace Contents -->
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
<!-- Method Quickref -->
|
|
60
|
+
|
|
61
|
+
<div id="method-list-section" class="section">
|
|
62
|
+
<h3 class="section-header">Methods</h3>
|
|
63
|
+
<ul class="link-list">
|
|
64
|
+
|
|
65
|
+
<li><a href="#method-c-compile_code">::compile_code</a></li>
|
|
66
|
+
|
|
67
|
+
<li><a href="#method-c-do_preprocessing">::do_preprocessing</a></li>
|
|
68
|
+
|
|
69
|
+
<li><a href="#method-c-get_input">::get_input</a></li>
|
|
70
|
+
|
|
71
|
+
<li><a href="#method-c-get_job_data">::get_job_data</a></li>
|
|
72
|
+
|
|
73
|
+
<li><a href="#method-c-get_std_out_and_err">::get_std_out_and_err</a></li>
|
|
74
|
+
|
|
75
|
+
<li><a href="#method-c-preprocess_babel">::preprocess_babel</a></li>
|
|
76
|
+
|
|
77
|
+
<li><a href="#method-c-preprocess_compile">::preprocess_compile</a></li>
|
|
78
|
+
|
|
79
|
+
<li><a href="#method-c-preprocess_erlang">::preprocess_erlang</a></li>
|
|
80
|
+
|
|
81
|
+
<li><a href="#method-c-preprocess_mpi">::preprocess_mpi</a></li>
|
|
82
|
+
|
|
83
|
+
<li><a href="#method-c-preprocess_ssa">::preprocess_ssa</a></li>
|
|
84
|
+
|
|
85
|
+
<li><a href="#method-c-require_file_to_exist">::require_file_to_exist</a></li>
|
|
86
|
+
|
|
87
|
+
<li><a href="#method-c-require_file_to_not_exist">::require_file_to_not_exist</a></li>
|
|
88
|
+
|
|
89
|
+
<li><a href="#method-c-require_param">::require_param</a></li>
|
|
90
|
+
|
|
91
|
+
<li><a href="#method-c-run_job">::run_job</a></li>
|
|
92
|
+
|
|
93
|
+
<li><a href="#method-c-upload_app_for_cicero">::upload_app_for_cicero</a></li>
|
|
94
|
+
|
|
95
|
+
<li><a href="#method-c-validate_storage_params">::validate_storage_params</a></li>
|
|
96
|
+
|
|
97
|
+
<li><a href="#method-c-wait_for_compilation_to_finish">::wait_for_compilation_to_finish</a></li>
|
|
98
|
+
|
|
99
|
+
</ul>
|
|
100
|
+
</div>
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
<!-- Included Modules -->
|
|
104
|
+
|
|
105
|
+
</div>
|
|
106
|
+
|
|
107
|
+
<div id="project-metadata">
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
<div id="classindex-section" class="section project-section">
|
|
112
|
+
<h3 class="section-header">Class/Module Index
|
|
113
|
+
<span class="search-toggle"><img src="./images/find.png"
|
|
114
|
+
height="16" width="16" alt="[+]"
|
|
115
|
+
title="show/hide quicksearch" /></span></h3>
|
|
116
|
+
<form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
|
|
117
|
+
<fieldset>
|
|
118
|
+
<legend>Quicksearch</legend>
|
|
119
|
+
<input type="text" name="quicksearch" value=""
|
|
120
|
+
class="quicksearch-field" />
|
|
121
|
+
</fieldset>
|
|
122
|
+
</form>
|
|
123
|
+
|
|
124
|
+
<ul class="link-list">
|
|
125
|
+
|
|
126
|
+
<li><a href="./AppControllerClient.html">AppControllerClient</a></li>
|
|
127
|
+
|
|
128
|
+
<li><a href="./AppControllerException.html">AppControllerException</a></li>
|
|
129
|
+
|
|
130
|
+
<li><a href="./BabelHelper.html">BabelHelper</a></li>
|
|
131
|
+
|
|
132
|
+
<li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
|
|
133
|
+
|
|
134
|
+
<li><a href="./CommonFunctions.html">CommonFunctions</a></li>
|
|
135
|
+
|
|
136
|
+
<li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
|
|
137
|
+
|
|
138
|
+
<li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
|
|
139
|
+
|
|
140
|
+
<li><a href="./Object.html">Object</a></li>
|
|
141
|
+
|
|
142
|
+
</ul>
|
|
143
|
+
<div id="no-class-search-results" style="display: none;">No matching classes.</div>
|
|
144
|
+
</div>
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
</div>
|
|
148
|
+
</div>
|
|
149
|
+
|
|
150
|
+
<div id="documentation">
|
|
151
|
+
<h1 class="module">NeptuneHelper</h1>
|
|
152
|
+
|
|
153
|
+
<div id="description">
|
|
154
|
+
|
|
155
|
+
</div>
|
|
156
|
+
|
|
157
|
+
<!-- Constants -->
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
<!-- Attributes -->
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
<!-- Methods -->
|
|
164
|
+
|
|
165
|
+
<div id="public-class-method-details" class="method-section section">
|
|
166
|
+
<h3 class="section-header">Public Class Methods</h3>
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
<div id="compile_code-method" class="method-detail ">
|
|
170
|
+
<a name="method-c-compile_code"></a>
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
<div class="method-heading">
|
|
174
|
+
<span class="method-name">compile_code</span><span
|
|
175
|
+
class="method-args">(job_data, ssh_args, shadow_ip)</span>
|
|
176
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
177
|
+
</div>
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
<div class="method-description">
|
|
181
|
+
|
|
182
|
+
<p>This method sends out a request to compile code, waits for it to finish,
|
|
183
|
+
and gets the standard out and error returned from the compilation. This
|
|
184
|
+
method returns a hash containing the standard out, error, and a result that
|
|
185
|
+
indicates whether or not the compilation was successful.</p>
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
<div class="method-source-code"
|
|
190
|
+
id="compile_code-source">
|
|
191
|
+
<pre>
|
|
192
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 369</span>
|
|
193
|
+
def self.compile_code(job_data, ssh_args, shadow_ip)
|
|
194
|
+
compiled_location = controller.compile_code(job_data)
|
|
195
|
+
copy_to = job_data[<span class="ruby-string">"@copy_to"</span>]
|
|
196
|
+
self.wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
|
|
197
|
+
|
|
198
|
+
<span class="ruby-constant">FileUtils</span>.rm_rf(copy_to)
|
|
199
|
+
|
|
200
|
+
scp_command = "scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2>&1"
|
|
201
|
+
<span class="ruby-constant">Kernel</span>.puts scp_command
|
|
202
|
+
<span class="ruby-constant">CommonFunctions</span>.shell(scp_command)
|
|
203
|
+
|
|
204
|
+
code = job_data[<span class="ruby-string">"@code"</span>]
|
|
205
|
+
dirs = code.split(<span class="ruby-regexp">/\//</span>)
|
|
206
|
+
remote_dir = <span class="ruby-string">"/tmp/"</span> + dirs[-1]
|
|
207
|
+
|
|
208
|
+
[remote_dir, compiled_location].each { |remote_files|
|
|
209
|
+
ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_files}' 2>&1"
|
|
210
|
+
<span class="ruby-constant">Kernel</span>.puts ssh_command
|
|
211
|
+
<span class="ruby-constant">CommonFunctions</span>.shell(ssh_command)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return get_std_out_and_err(copy_to)
|
|
215
|
+
end</pre>
|
|
216
|
+
</div>
|
|
217
|
+
|
|
218
|
+
</div>
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
</div>
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
<div id="do_preprocessing-method" class="method-detail ">
|
|
227
|
+
<a name="method-c-do_preprocessing"></a>
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
<div class="method-heading">
|
|
231
|
+
<span class="method-name">do_preprocessing</span><span
|
|
232
|
+
class="method-args">(job_data, controller)</span>
|
|
233
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
234
|
+
</div>
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
<div class="method-description">
|
|
238
|
+
|
|
239
|
+
<p>Certain types of jobs need steps to be taken before they can be started
|
|
240
|
+
(e.g., copying input data or code over). This method dispatches the right
|
|
241
|
+
method to use based on the type of the job that the user has asked to run.</p>
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
<div class="method-source-code"
|
|
246
|
+
id="do_preprocessing-source">
|
|
247
|
+
<pre>
|
|
248
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 60</span>
|
|
249
|
+
def self.do_preprocessing(job_data, controller)
|
|
250
|
+
job_type = job_data[<span class="ruby-string">"@type"</span>]
|
|
251
|
+
if !<span class="ruby-constant">NEED_PREPROCESSING</span>.include?(job_type)
|
|
252
|
+
return
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
<span class="ruby-comment"># Don't worry about adding on the self. prefix - send will resolve</span>
|
|
256
|
+
<span class="ruby-comment"># it the right way</span>
|
|
257
|
+
preprocess = "preprocess_#{job_type}".to_sym
|
|
258
|
+
send(preprocess, job_data, controller)
|
|
259
|
+
end</pre>
|
|
260
|
+
</div>
|
|
261
|
+
|
|
262
|
+
</div>
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
</div>
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
<div id="get_input-method" class="method-detail ">
|
|
271
|
+
<a name="method-c-get_input"></a>
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
<div class="method-heading">
|
|
275
|
+
<span class="method-name">get_input</span><span
|
|
276
|
+
class="method-args">(job_data, ssh_args, shadow_ip, controller)</span>
|
|
277
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
278
|
+
</div>
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
<div class="method-description">
|
|
282
|
+
|
|
283
|
+
<p>This method takes a file on the local user’s computer and stores it
|
|
284
|
+
remotely via AppScale. It returns a hash map indicating whether or not the
|
|
285
|
+
job succeeded and if it failed, the reason for it.</p>
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
<div class="method-source-code"
|
|
290
|
+
id="get_input-source">
|
|
291
|
+
<pre>
|
|
292
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 319</span>
|
|
293
|
+
def self.get_input(job_data, ssh_args, shadow_ip, controller)
|
|
294
|
+
result = {:result => :success}
|
|
295
|
+
|
|
296
|
+
self.require_param(<span class="ruby-string">"@local"</span>, job_data)
|
|
297
|
+
|
|
298
|
+
local_file = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@local"</span>])
|
|
299
|
+
if !<span class="ruby-constant">File</span>.exists?(local_file)
|
|
300
|
+
reason = "the file you specified to copy, #{local_file}, doesn't exist." +
|
|
301
|
+
<span class="ruby-string">" Please specify a file that exists and try again."</span>
|
|
302
|
+
return {:result => :failure, :reason => reason}
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
remote = "/tmp/neptune-input-#{rand(100000)}"
|
|
306
|
+
scp_cmd = "scp -r #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
|
|
307
|
+
<span class="ruby-constant">Kernel</span>.puts scp_cmd
|
|
308
|
+
<span class="ruby-constant">CommonFunctions</span>.shell(scp_cmd)
|
|
309
|
+
|
|
310
|
+
job_data[<span class="ruby-string">"@local"</span>] = remote
|
|
311
|
+
<span class="ruby-constant">Kernel</span>.puts "job data = #{job_data.inspect}"
|
|
312
|
+
response = controller.put_input(job_data)
|
|
313
|
+
if response
|
|
314
|
+
return {:result => :success}
|
|
315
|
+
else
|
|
316
|
+
<span class="ruby-comment"># TODO - expand this to include the reason why it failed</span>
|
|
317
|
+
return {:result => :failure}
|
|
318
|
+
end
|
|
319
|
+
end</pre>
|
|
320
|
+
</div>
|
|
321
|
+
|
|
322
|
+
</div>
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
</div>
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
<div id="get_job_data-method" class="method-detail ">
|
|
331
|
+
<a name="method-c-get_job_data"></a>
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
<div class="method-heading">
|
|
335
|
+
<span class="method-name">get_job_data</span><span
|
|
336
|
+
class="method-args">(params)</span>
|
|
337
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
338
|
+
</div>
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
<div class="method-description">
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
<div class="method-source-code"
|
|
348
|
+
id="get_job_data-source">
|
|
349
|
+
<pre>
|
|
350
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 232</span>
|
|
351
|
+
def self.get_job_data(params)
|
|
352
|
+
job_data = {}
|
|
353
|
+
params.each { |k, v|
|
|
354
|
+
key = "@#{k}"
|
|
355
|
+
job_data[key] = v
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
job_data.delete(<span class="ruby-string">"@job"</span>)
|
|
359
|
+
job_data[<span class="ruby-string">"@keyname"</span>] = params[:keyname] || <span class="ruby-string">"appscale"</span>
|
|
360
|
+
|
|
361
|
+
job_data[<span class="ruby-string">"@type"</span>] = job_data[<span class="ruby-string">"@type"</span>].to_s
|
|
362
|
+
type = job_data[<span class="ruby-string">"@type"</span>]
|
|
363
|
+
|
|
364
|
+
if !<span class="ruby-constant">ALLOWED_JOB_TYPES</span>.include?(type)
|
|
365
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-constant">JOB_TYPE_NOT_ALLOWED</span>)
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
if type == <span class="ruby-string">"upc"</span> or type == <span class="ruby-string">"x10"</span>
|
|
369
|
+
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"mpi"</span>
|
|
370
|
+
type = <span class="ruby-string">"mpi"</span>
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
<span class="ruby-comment"># kdt jobs also run as mpi jobs, but need to pass along an executable</span>
|
|
374
|
+
<span class="ruby-comment"># parameter to let mpiexec know to use python to exec it</span>
|
|
375
|
+
if type == <span class="ruby-string">"kdt"</span>
|
|
376
|
+
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"mpi"</span>
|
|
377
|
+
type = <span class="ruby-string">"mpi"</span>
|
|
378
|
+
|
|
379
|
+
job_data[<span class="ruby-string">"@executable"</span>] = <span class="ruby-string">"python"</span>
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
if job_data[<span class="ruby-string">"@nodes_to_use"</span>].class == <span class="ruby-constant">Hash</span>
|
|
383
|
+
job_data[<span class="ruby-string">"@nodes_to_use"</span>] = job_data[<span class="ruby-string">"@nodes_to_use"</span>].to_a.flatten
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
if !<span class="ruby-constant">NO_OUTPUT_NEEDED</span>.include?(type)
|
|
387
|
+
if (job_data[<span class="ruby-string">"@output"</span>].nil? or job_data[<span class="ruby-string">"@output"</span>].empty?)
|
|
388
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"Job output must be specified"</span>)
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
if job_data[<span class="ruby-string">"@output"</span>][0].chr != <span class="ruby-string">"/"</span>
|
|
392
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"Job output must begin with a slash ('/')"</span>)
|
|
393
|
+
end
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
return job_data
|
|
397
|
+
end</pre>
|
|
398
|
+
</div>
|
|
399
|
+
|
|
400
|
+
</div>
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
</div>
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
<div id="get_std_out_and_err-method" class="method-detail ">
|
|
409
|
+
<a name="method-c-get_std_out_and_err"></a>
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
<div class="method-heading">
|
|
413
|
+
<span class="method-name">get_std_out_and_err</span><span
|
|
414
|
+
class="method-args">(location)</span>
|
|
415
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
416
|
+
</div>
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
<div class="method-description">
|
|
420
|
+
|
|
421
|
+
<p>This method returns a hash containing the standard out and standard error
|
|
422
|
+
from a completed job, as well as a result field that indicates whether or
|
|
423
|
+
not the job completed successfully (success = no errors).</p>
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
<div class="method-source-code"
|
|
428
|
+
id="get_std_out_and_err-source">
|
|
429
|
+
<pre>
|
|
430
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 396</span>
|
|
431
|
+
def self.get_std_out_and_err(location)
|
|
432
|
+
result = {}
|
|
433
|
+
|
|
434
|
+
out = <span class="ruby-constant">File</span>.open("#{location}/compile_out") { |f| f.read.chomp! }
|
|
435
|
+
result[:out] = out
|
|
436
|
+
|
|
437
|
+
err = <span class="ruby-constant">File</span>.open("#{location}/compile_err") { |f| f.read.chomp! }
|
|
438
|
+
result[:err] = err
|
|
439
|
+
|
|
440
|
+
if result[:err]
|
|
441
|
+
result[:result] = :failure
|
|
442
|
+
else
|
|
443
|
+
result[:result] = :success
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
return result
|
|
447
|
+
end</pre>
|
|
448
|
+
</div>
|
|
449
|
+
|
|
450
|
+
</div>
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
</div>
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
<div id="preprocess_babel-method" class="method-detail ">
|
|
459
|
+
<a name="method-c-preprocess_babel"></a>
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
<div class="method-heading">
|
|
463
|
+
<span class="method-name">preprocess_babel</span><span
|
|
464
|
+
class="method-args">(job_data, controller)</span>
|
|
465
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
466
|
+
</div>
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
<div class="method-description">
|
|
470
|
+
|
|
471
|
+
<p>This preprocessing method verifies that the user specified code that should
|
|
472
|
+
be run, where the output should be placed, and an engine to run over. It
|
|
473
|
+
also verifies that all files to be used are actually reachable. Supported
|
|
474
|
+
engines can be found by contacting an AppScale node.</p>
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
<div class="method-source-code"
|
|
479
|
+
id="preprocess_babel-source">
|
|
480
|
+
<pre>
|
|
481
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 188</span>
|
|
482
|
+
def self.preprocess_babel(job_data, controller)
|
|
483
|
+
self.require_param(<span class="ruby-string">"@code"</span>, job_data)
|
|
484
|
+
self.require_param(<span class="ruby-string">"@engine"</span>, job_data)
|
|
485
|
+
self.require_param(<span class="ruby-string">"@output"</span>, job_data)
|
|
486
|
+
|
|
487
|
+
<span class="ruby-comment"># For most code types, the file's name given is the thing to exec.</span>
|
|
488
|
+
<span class="ruby-comment"># For Java, the actual file to search for is whatever the user gives</span>
|
|
489
|
+
<span class="ruby-comment"># us, with a .class extension.</span>
|
|
490
|
+
code_file_name = job_data[<span class="ruby-string">"@code"</span>]
|
|
491
|
+
if !job_data[<span class="ruby-string">"@executable"</span>].nil? and job_data[<span class="ruby-string">"@executable"</span>] == <span class="ruby-string">"java"</span>
|
|
492
|
+
code_file_name += <span class="ruby-string">".class"</span>
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
self.require_file_to_exist(code_file_name, job_data, controller)
|
|
496
|
+
self.require_file_to_not_exist(job_data[<span class="ruby-string">"@output"</span>], job_data, controller)
|
|
497
|
+
|
|
498
|
+
if job_data[<span class="ruby-string">"@argv"</span>]
|
|
499
|
+
argv = job_data[<span class="ruby-string">"@argv"</span>]
|
|
500
|
+
if argv.class != <span class="ruby-constant">Array</span>
|
|
501
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"argv must be an array"</span>)
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
argv.each { |arg|
|
|
505
|
+
if arg =~ <span class="ruby-regexp">/\/.*\/.*/</span>
|
|
506
|
+
self.require_file_to_exist(arg, job_data, controller)
|
|
507
|
+
end
|
|
508
|
+
}
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
if job_data[<span class="ruby-string">"@appcfg_cookies"</span>]
|
|
512
|
+
self.require_file_to_exist(job_data[<span class="ruby-string">"@appcfg_cookies"</span>], job_data, controller)
|
|
513
|
+
end
|
|
514
|
+
|
|
515
|
+
user_specified_engine = job_data[<span class="ruby-string">"@engine"</span>]
|
|
516
|
+
|
|
517
|
+
<span class="ruby-comment"># validate the engine here</span>
|
|
518
|
+
engines = controller.get_supported_babel_engines(job_data)
|
|
519
|
+
if !engines.include?(user_specified_engine)
|
|
520
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"The engine you specified, "</span> +
|
|
521
|
+
"#{user_specified_engine}, is not a supported engine. Supported engines" +
|
|
522
|
+
" are: #{engines.join(', ')}")
|
|
523
|
+
end
|
|
524
|
+
end</pre>
|
|
525
|
+
</div>
|
|
526
|
+
|
|
527
|
+
</div>
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
</div>
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
<div id="preprocess_compile-method" class="method-detail ">
|
|
536
|
+
<a name="method-c-preprocess_compile"></a>
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
<div class="method-heading">
|
|
540
|
+
<span class="method-name">preprocess_compile</span><span
|
|
541
|
+
class="method-args">(job_data, controller)</span>
|
|
542
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
543
|
+
</div>
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
<div class="method-description">
|
|
547
|
+
|
|
548
|
+
<p>This preprocessing method copies over the user’s code to the Shadow node
|
|
549
|
+
so that it can be compiled there. A future version of this method may also
|
|
550
|
+
copy over libraries as well.</p>
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
<div class="method-source-code"
|
|
555
|
+
id="preprocess_compile-source">
|
|
556
|
+
<pre>
|
|
557
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 75</span>
|
|
558
|
+
def self.preprocess_compile(job_data, controller)
|
|
559
|
+
code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@code"</span>])
|
|
560
|
+
if !<span class="ruby-constant">File</span>.exists?(code)
|
|
561
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new("The source file #{code} does not exist.")
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
suffix = code.split(<span class="ruby-string">'/'</span>)[-1]
|
|
565
|
+
dest = "/tmp/#{suffix}"
|
|
566
|
+
keyname = job_data[<span class="ruby-string">"@keyname"</span>]
|
|
567
|
+
shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
|
|
568
|
+
|
|
569
|
+
ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}"
|
|
570
|
+
remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}' 2>&1"
|
|
571
|
+
<span class="ruby-constant">Kernel</span>.puts remove_dir
|
|
572
|
+
<span class="ruby-constant">CommonFunctions</span>.shell(remove_dir)
|
|
573
|
+
<span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(code, dest, keyname, is_dir=true)
|
|
574
|
+
|
|
575
|
+
job_data[<span class="ruby-string">"@code"</span>] = dest
|
|
576
|
+
end</pre>
|
|
577
|
+
</div>
|
|
578
|
+
|
|
579
|
+
</div>
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
</div>
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
<div id="preprocess_erlang-method" class="method-detail ">
|
|
588
|
+
<a name="method-c-preprocess_erlang"></a>
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
<div class="method-heading">
|
|
592
|
+
<span class="method-name">preprocess_erlang</span><span
|
|
593
|
+
class="method-args">(job_data, controller)</span>
|
|
594
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
595
|
+
</div>
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
<div class="method-description">
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
<div class="method-source-code"
|
|
605
|
+
id="preprocess_erlang-source">
|
|
606
|
+
<pre>
|
|
607
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 95</span>
|
|
608
|
+
def self.preprocess_erlang(job_data, controller)
|
|
609
|
+
self.require_param(<span class="ruby-string">"@code"</span>, job_data)
|
|
610
|
+
|
|
611
|
+
source_code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@code"</span>])
|
|
612
|
+
if !<span class="ruby-constant">File</span>.exists?(source_code)
|
|
613
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new("The specified code, #{job_data['@code']}," +
|
|
614
|
+
<span class="ruby-string">" didn't exist. Please specify one that exists and try again"</span>)
|
|
615
|
+
end
|
|
616
|
+
dest_code = <span class="ruby-string">"/tmp/"</span>
|
|
617
|
+
|
|
618
|
+
keyname = job_data[<span class="ruby-string">"@keyname"</span>]
|
|
619
|
+
<span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(source_code, dest_code, keyname)
|
|
620
|
+
end</pre>
|
|
621
|
+
</div>
|
|
622
|
+
|
|
623
|
+
</div>
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
</div>
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
<div id="preprocess_mpi-method" class="method-detail ">
|
|
632
|
+
<a name="method-c-preprocess_mpi"></a>
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
<div class="method-heading">
|
|
636
|
+
<span class="method-name">preprocess_mpi</span><span
|
|
637
|
+
class="method-args">(job_data, controller)</span>
|
|
638
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
639
|
+
</div>
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
<div class="method-description">
|
|
643
|
+
|
|
644
|
+
<p>This preprocessing method verifies that the user specified the number of
|
|
645
|
+
nodes to use. If they also specified the number of processes to use, we
|
|
646
|
+
also verify that this value is at least as many as the number of nodes
|
|
647
|
+
(that is, nodes can’t be underprovisioned in MPI).</p>
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
<div class="method-source-code"
|
|
652
|
+
id="preprocess_mpi-source">
|
|
653
|
+
<pre>
|
|
654
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 113</span>
|
|
655
|
+
def self.preprocess_mpi(job_data, controller)
|
|
656
|
+
self.require_param(<span class="ruby-string">"@nodes_to_use"</span>, job_data)
|
|
657
|
+
self.require_param(<span class="ruby-string">"@procs_to_use"</span>, job_data)
|
|
658
|
+
|
|
659
|
+
if job_data[<span class="ruby-string">"@procs_to_use"</span>]
|
|
660
|
+
p = job_data[<span class="ruby-string">"@procs_to_use"</span>]
|
|
661
|
+
n = job_data[<span class="ruby-string">"@nodes_to_use"</span>]
|
|
662
|
+
if p < n
|
|
663
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">":procs_to_use must be at least as "</span> +
|
|
664
|
+
<span class="ruby-string">"large as :nodes_to_use."</span>)
|
|
665
|
+
end
|
|
666
|
+
end
|
|
667
|
+
|
|
668
|
+
if job_data[<span class="ruby-string">"@argv"</span>]
|
|
669
|
+
argv = job_data[<span class="ruby-string">"@argv"</span>]
|
|
670
|
+
|
|
671
|
+
if argv.class == <span class="ruby-constant">String</span>
|
|
672
|
+
job_data[<span class="ruby-string">"@argv"</span>] = argv
|
|
673
|
+
elsif argv.class == <span class="ruby-constant">Array</span>
|
|
674
|
+
job_data[<span class="ruby-string">"@argv"</span>] = argv.join(<span class="ruby-string">' '</span>)
|
|
675
|
+
else
|
|
676
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">":argv must be either a String or Array"</span>)
|
|
677
|
+
end
|
|
678
|
+
end
|
|
679
|
+
|
|
680
|
+
return job_data
|
|
681
|
+
end</pre>
|
|
682
|
+
</div>
|
|
683
|
+
|
|
684
|
+
</div>
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
</div>
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
<div id="preprocess_ssa-method" class="method-detail ">
|
|
693
|
+
<a name="method-c-preprocess_ssa"></a>
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
<div class="method-heading">
|
|
697
|
+
<span class="method-name">preprocess_ssa</span><span
|
|
698
|
+
class="method-args">(job_data, controller)</span>
|
|
699
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
700
|
+
</div>
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
<div class="method-description">
|
|
704
|
+
|
|
705
|
+
<p>This preprocessing method verifies that the user specified the number of
|
|
706
|
+
trajectories to run, via either :trajectories or :simulations. Both should
|
|
707
|
+
not be specified - only one or the other, and regardless of which they
|
|
708
|
+
specify, convert it to be :trajectories.</p>
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
<div class="method-source-code"
|
|
713
|
+
id="preprocess_ssa-source">
|
|
714
|
+
<pre>
|
|
715
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 145</span>
|
|
716
|
+
def self.preprocess_ssa(job_data, controller)
|
|
717
|
+
if job_data[<span class="ruby-string">"@simulations"</span>] and job_data[<span class="ruby-string">"@trajectories"</span>]
|
|
718
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">":simulations and :trajectories "</span> +
|
|
719
|
+
<span class="ruby-string">"not both be specified."</span>)
|
|
720
|
+
end
|
|
721
|
+
|
|
722
|
+
if job_data[<span class="ruby-string">"@simulations"</span>]
|
|
723
|
+
job_data[<span class="ruby-string">"@trajectories"</span>] = job_data[<span class="ruby-string">"@simulations"</span>]
|
|
724
|
+
job_data.delete(<span class="ruby-string">"@simulations"</span>)
|
|
725
|
+
end
|
|
726
|
+
|
|
727
|
+
self.require_param(<span class="ruby-string">"@trajectories"</span>, job_data)
|
|
728
|
+
return job_data
|
|
729
|
+
end</pre>
|
|
730
|
+
</div>
|
|
731
|
+
|
|
732
|
+
</div>
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
</div>
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
<div id="require_file_to_exist-method" class="method-detail ">
|
|
741
|
+
<a name="method-c-require_file_to_exist"></a>
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
<div class="method-heading">
|
|
745
|
+
<span class="method-name">require_file_to_exist</span><span
|
|
746
|
+
class="method-args">(file, job_data, controller)</span>
|
|
747
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
748
|
+
</div>
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
<div class="method-description">
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
<div class="method-source-code"
|
|
758
|
+
id="require_file_to_exist-source">
|
|
759
|
+
<pre>
|
|
760
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 166</span>
|
|
761
|
+
def self.require_file_to_exist(file, job_data, controller)
|
|
762
|
+
if controller.does_file_exist?(file, job_data)
|
|
763
|
+
return
|
|
764
|
+
else
|
|
765
|
+
raise <span class="ruby-constant">FileNotFoundException</span>
|
|
766
|
+
end
|
|
767
|
+
end</pre>
|
|
768
|
+
</div>
|
|
769
|
+
|
|
770
|
+
</div>
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
</div>
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
<div id="require_file_to_not_exist-method" class="method-detail ">
|
|
779
|
+
<a name="method-c-require_file_to_not_exist"></a>
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
<div class="method-heading">
|
|
783
|
+
<span class="method-name">require_file_to_not_exist</span><span
|
|
784
|
+
class="method-args">(file, job_data, controller)</span>
|
|
785
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
786
|
+
</div>
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
<div class="method-description">
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
<div class="method-source-code"
|
|
796
|
+
id="require_file_to_not_exist-source">
|
|
797
|
+
<pre>
|
|
798
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 174</span>
|
|
799
|
+
def self.require_file_to_not_exist(file, job_data, controller)
|
|
800
|
+
begin
|
|
801
|
+
self.require_file_to_exist(file, job_data, controller)
|
|
802
|
+
<span class="ruby-comment"># no exception thrown previously means that the output file exists</span>
|
|
803
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">'Output specified already exists'</span>)
|
|
804
|
+
rescue <span class="ruby-constant">FileNotFoundException</span>
|
|
805
|
+
return
|
|
806
|
+
end
|
|
807
|
+
end</pre>
|
|
808
|
+
</div>
|
|
809
|
+
|
|
810
|
+
</div>
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
</div>
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
<div id="require_param-method" class="method-detail ">
|
|
819
|
+
<a name="method-c-require_param"></a>
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
<div class="method-heading">
|
|
823
|
+
<span class="method-name">require_param</span><span
|
|
824
|
+
class="method-args">(param, job_data)</span>
|
|
825
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
826
|
+
</div>
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
<div class="method-description">
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
<div class="method-source-code"
|
|
836
|
+
id="require_param-source">
|
|
837
|
+
<pre>
|
|
838
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 160</span>
|
|
839
|
+
def self.require_param(param, job_data)
|
|
840
|
+
if !job_data[param]
|
|
841
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new("#{param} must be specified")
|
|
842
|
+
end
|
|
843
|
+
end</pre>
|
|
844
|
+
</div>
|
|
845
|
+
|
|
846
|
+
</div>
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
|
|
851
|
+
</div>
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
<div id="run_job-method" class="method-detail ">
|
|
855
|
+
<a name="method-c-run_job"></a>
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
<div class="method-heading">
|
|
859
|
+
<span class="method-name">run_job</span><span
|
|
860
|
+
class="method-args">(job_data, ssh_args, shadow_ip, secret)</span>
|
|
861
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
862
|
+
</div>
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
<div class="method-description">
|
|
866
|
+
|
|
867
|
+
<p>This method actually runs the Neptune job, given information about the job
|
|
868
|
+
as well as information about the node to send the request to.</p>
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
<div class="method-source-code"
|
|
873
|
+
id="run_job-source">
|
|
874
|
+
<pre>
|
|
875
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 442</span>
|
|
876
|
+
def self.run_job(job_data, ssh_args, shadow_ip, secret)
|
|
877
|
+
controller = <span class="ruby-constant">AppControllerClient</span>.new(shadow_ip, secret)
|
|
878
|
+
|
|
879
|
+
<span class="ruby-comment"># TODO - right now the job is assumed to succeed in many cases</span>
|
|
880
|
+
<span class="ruby-comment"># need to investigate the various failure scenarios</span>
|
|
881
|
+
result = { :result => :success }
|
|
882
|
+
|
|
883
|
+
case job_data[<span class="ruby-string">"@type"</span>]
|
|
884
|
+
when <span class="ruby-string">"input"</span>
|
|
885
|
+
result = self.get_input(job_data, ssh_args, shadow_ip, controller)
|
|
886
|
+
when <span class="ruby-string">"output"</span>
|
|
887
|
+
result[:output] = controller.get_output(job_data)
|
|
888
|
+
when <span class="ruby-string">"get-acl"</span>
|
|
889
|
+
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"acl"</span>
|
|
890
|
+
result[:acl] = controller.get_acl(job_data)
|
|
891
|
+
when <span class="ruby-string">"set-acl"</span>
|
|
892
|
+
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"acl"</span>
|
|
893
|
+
result[:acl] = controller.set_acl(job_data)
|
|
894
|
+
when <span class="ruby-string">"compile"</span>
|
|
895
|
+
result = self.compile_code(job_data, ssh_args, shadow_ip)
|
|
896
|
+
when <span class="ruby-string">"cicero"</span>
|
|
897
|
+
self.upload_app_for_cicero(job_data)
|
|
898
|
+
msg = controller.start_neptune_job(job_data)
|
|
899
|
+
result[:msg] = msg
|
|
900
|
+
result[:result] = :failure if result[:msg] !~ <span class="ruby-regexp">/job is now running\Z/</span>
|
|
901
|
+
else
|
|
902
|
+
msg = controller.start_neptune_job(job_data)
|
|
903
|
+
result[:msg] = msg
|
|
904
|
+
result[:result] = :failure if result[:msg] !~ <span class="ruby-regexp">/job is now running\Z/</span>
|
|
905
|
+
end
|
|
906
|
+
|
|
907
|
+
return result
|
|
908
|
+
end</pre>
|
|
909
|
+
</div>
|
|
910
|
+
|
|
911
|
+
</div>
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
</div>
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
<div id="upload_app_for_cicero-method" class="method-detail ">
|
|
920
|
+
<a name="method-c-upload_app_for_cicero"></a>
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
<div class="method-heading">
|
|
924
|
+
<span class="method-name">upload_app_for_cicero</span><span
|
|
925
|
+
class="method-args">(job_data)</span>
|
|
926
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
927
|
+
</div>
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
<div class="method-description">
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
<div class="method-source-code"
|
|
937
|
+
id="upload_app_for_cicero-source">
|
|
938
|
+
<pre>
|
|
939
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 414</span>
|
|
940
|
+
def self.upload_app_for_cicero(job_data)
|
|
941
|
+
if !job_data[<span class="ruby-string">"@app"</span>]
|
|
942
|
+
<span class="ruby-constant">Kernel</span>.puts <span class="ruby-string">"No app specified, not uploading..."</span>
|
|
943
|
+
return
|
|
944
|
+
end
|
|
945
|
+
|
|
946
|
+
app_location = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@app"</span>])
|
|
947
|
+
if !<span class="ruby-constant">File</span>.exists?(app_location)
|
|
948
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new("The app you specified, #{app_location}, does not exist." +
|
|
949
|
+
<span class="ruby-string">"Please specify one that does and try again."</span>)
|
|
950
|
+
end
|
|
951
|
+
|
|
952
|
+
keyname = job_data[<span class="ruby-string">"@keyname"</span>] || <span class="ruby-string">"appscale"</span>
|
|
953
|
+
if job_data[<span class="ruby-string">"@appscale_tools"</span>]
|
|
954
|
+
upload_app = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@appscale_tools"</span>]) +
|
|
955
|
+
<span class="ruby-constant">File</span>::<span class="ruby-constant">SEPARATOR</span> + <span class="ruby-string">"bin"</span> + <span class="ruby-constant">File</span>::<span class="ruby-constant">SEPARATOR</span> + <span class="ruby-string">"appscale-upload-app"</span>
|
|
956
|
+
else
|
|
957
|
+
upload_app = <span class="ruby-string">"appscale-upload-app"</span>
|
|
958
|
+
end
|
|
959
|
+
|
|
960
|
+
<span class="ruby-constant">Kernel</span>.puts "Uploading AppEngine app at #{app_location}"
|
|
961
|
+
upload_command = "#{upload_app} --file #{app_location} --test --keyname #{keyname}"
|
|
962
|
+
<span class="ruby-constant">Kernel</span>.puts upload_command
|
|
963
|
+
<span class="ruby-constant">Kernel</span>.puts `#{upload_command}`
|
|
964
|
+
end</pre>
|
|
965
|
+
</div>
|
|
966
|
+
|
|
967
|
+
</div>
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
</div>
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
<div id="validate_storage_params-method" class="method-detail ">
|
|
976
|
+
<a name="method-c-validate_storage_params"></a>
|
|
977
|
+
|
|
978
|
+
|
|
979
|
+
<div class="method-heading">
|
|
980
|
+
<span class="method-name">validate_storage_params</span><span
|
|
981
|
+
class="method-args">(job_data)</span>
|
|
982
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
983
|
+
</div>
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
<div class="method-description">
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
<div class="method-source-code"
|
|
993
|
+
id="validate_storage_params-source">
|
|
994
|
+
<pre>
|
|
995
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 280</span>
|
|
996
|
+
def self.validate_storage_params(job_data)
|
|
997
|
+
job_data[<span class="ruby-string">"@storage"</span>] ||= <span class="ruby-string">"appdb"</span>
|
|
998
|
+
|
|
999
|
+
storage = job_data[<span class="ruby-string">"@storage"</span>]
|
|
1000
|
+
if !<span class="ruby-constant">ALLOWED_STORAGE_TYPES</span>.include?(storage)
|
|
1001
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-string">"Supported storage types are "</span> +
|
|
1002
|
+
"#{ALLOWED_STORAGE_TYPES.join(', ')} - #{storage} is not supported.")
|
|
1003
|
+
end
|
|
1004
|
+
|
|
1005
|
+
<span class="ruby-comment"># Our implementation for storing / retrieving via Google Storage</span>
|
|
1006
|
+
<span class="ruby-comment"># and Walrus uses</span>
|
|
1007
|
+
<span class="ruby-comment"># the same library as we do for S3 - so just tell it that it's S3</span>
|
|
1008
|
+
if storage == <span class="ruby-string">"gstorage"</span> or storage == <span class="ruby-string">"walrus"</span>
|
|
1009
|
+
storage = <span class="ruby-string">"s3"</span>
|
|
1010
|
+
job_data[<span class="ruby-string">"@storage"</span>] = <span class="ruby-string">"s3"</span>
|
|
1011
|
+
end
|
|
1012
|
+
|
|
1013
|
+
if storage == <span class="ruby-string">"s3"</span>
|
|
1014
|
+
[<span class="ruby-string">"EC2_ACCESS_KEY"</span>, <span class="ruby-string">"EC2_SECRET_KEY"</span>, <span class="ruby-string">"S3_URL"</span>].each { |item|
|
|
1015
|
+
if job_data["@#{item}"]
|
|
1016
|
+
<span class="ruby-constant">Kernel</span>.puts "Using specified #{item}"
|
|
1017
|
+
else
|
|
1018
|
+
if <span class="ruby-constant">ENV</span>[item]
|
|
1019
|
+
<span class="ruby-constant">Kernel</span>.puts "Using #{item} from environment"
|
|
1020
|
+
job_data["@#{item}"] = <span class="ruby-constant">ENV</span>[item]
|
|
1021
|
+
else
|
|
1022
|
+
raise <span class="ruby-constant">BadConfigurationException</span>.new("When storing data to S3, #{item} must be specified or be in " +
|
|
1023
|
+
<span class="ruby-string">"your environment. Please do so and try again."</span>)
|
|
1024
|
+
end
|
|
1025
|
+
end
|
|
1026
|
+
}
|
|
1027
|
+
end
|
|
1028
|
+
|
|
1029
|
+
return job_data
|
|
1030
|
+
end</pre>
|
|
1031
|
+
</div>
|
|
1032
|
+
|
|
1033
|
+
</div>
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
</div>
|
|
1039
|
+
|
|
1040
|
+
|
|
1041
|
+
<div id="wait_for_compilation_to_finish-method" class="method-detail ">
|
|
1042
|
+
<a name="method-c-wait_for_compilation_to_finish"></a>
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
<div class="method-heading">
|
|
1046
|
+
<span class="method-name">wait_for_compilation_to_finish</span><span
|
|
1047
|
+
class="method-args">(ssh_args, shadow_ip, compiled_location)</span>
|
|
1048
|
+
<span class="method-click-advice">click to toggle source</span>
|
|
1049
|
+
</div>
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
<div class="method-description">
|
|
1053
|
+
|
|
1054
|
+
<p>This method waits for AppScale to finish compiling the user’s code,
|
|
1055
|
+
indicated by AppScale copying the finished code to a pre-determined
|
|
1056
|
+
location.</p>
|
|
1057
|
+
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
<div class="method-source-code"
|
|
1061
|
+
id="wait_for_compilation_to_finish-source">
|
|
1062
|
+
<pre>
|
|
1063
|
+
<span class="ruby-comment"># File lib/neptune.rb, line 349</span>
|
|
1064
|
+
def self.wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
|
|
1065
|
+
loop {
|
|
1066
|
+
ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2>&1"
|
|
1067
|
+
<span class="ruby-constant">Kernel</span>.puts ssh_command
|
|
1068
|
+
ssh_result = <span class="ruby-constant">CommonFunctions</span>.shell(ssh_command)
|
|
1069
|
+
<span class="ruby-constant">Kernel</span>.puts "result was [#{ssh_result}]"
|
|
1070
|
+
if ssh_result =~ <span class="ruby-regexp">/No such file or directory/</span>
|
|
1071
|
+
<span class="ruby-constant">Kernel</span>.puts <span class="ruby-string">"Still waiting for code to be compiled..."</span>
|
|
1072
|
+
else
|
|
1073
|
+
<span class="ruby-constant">Kernel</span>.puts "compilation complete! Copying compiled code to #{copy_to}"
|
|
1074
|
+
return
|
|
1075
|
+
end
|
|
1076
|
+
sleep(5)
|
|
1077
|
+
}
|
|
1078
|
+
end</pre>
|
|
1079
|
+
</div>
|
|
1080
|
+
|
|
1081
|
+
</div>
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
|
|
1086
|
+
</div>
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
</div>
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
</div>
|
|
1093
|
+
|
|
1094
|
+
<div id="validator-badges">
|
|
1095
|
+
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
|
1096
|
+
<p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
|
|
1097
|
+
Rdoc Generator</a> 2</small>.</p>
|
|
1098
|
+
</div>
|
|
1099
|
+
|
|
1100
|
+
</body>
|
|
1101
|
+
</html>
|
|
1102
|
+
|