neptune 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +15 -2
- data/doc/AppControllerClient.html +98 -5
- data/doc/Object.html +81 -12
- data/doc/README.html +8 -1
- data/doc/bin/neptune.html +1 -1
- data/doc/created.rid +5 -5
- data/doc/index.html +4 -0
- data/doc/lib/app_controller_client_rb.html +1 -1
- data/doc/lib/job_rb.html +1 -1
- data/lib/app_controller_client.rb +10 -0
- data/lib/common_functions.rb +11 -7
- data/lib/job.rb +73 -21
- metadata +4 -4
data/README
CHANGED
@@ -21,10 +21,15 @@ it also can be used to deploy other types of software. For example,
|
|
21
21
|
Neptune has support for user-specified scaling of the underlying
|
22
22
|
cloud platform: users can write Neptune jobs that manually add
|
23
23
|
load balancers, application servers, or database servers to a running
|
24
|
-
AppScale deployment.
|
24
|
+
AppScale deployment. Additionally, as of version 0.0.2, remote
|
25
|
+
compiling can also be performed: just give Neptune the path to
|
26
|
+
the directory you want to compile and be sure to include a Makefile
|
27
|
+
in it! Neptune will run 'make' on it (you can specify which target
|
28
|
+
to make as well) and return to you a folder containing the standard
|
29
|
+
out and standard error of the make command.
|
25
30
|
|
26
31
|
Sample Neptune job scripts can be found in samples. Test scripts will
|
27
|
-
be added to test
|
32
|
+
be added to the 'test' folder soon.
|
28
33
|
|
29
34
|
Developed by Chris Bunch as part of the AppScale project.
|
30
35
|
See LICENSE for the specifics of the New BSD License
|
@@ -46,5 +51,13 @@ will likely appear in a future release as well.
|
|
46
51
|
An academic paper describing Neptune is in the works - check
|
47
52
|
in for a link to that as it becomes available.
|
48
53
|
|
54
|
+
Version History:
|
55
|
+
|
56
|
+
February 4, 2011 - 0.0.2 released, adding support for remote
|
57
|
+
compiling
|
58
|
+
|
59
|
+
January 27, 2011 - 0.0.1 released, with initial support for
|
60
|
+
MPI, X10, and MapReduce.
|
61
|
+
|
49
62
|
=end
|
50
63
|
|
@@ -71,6 +71,8 @@
|
|
71
71
|
|
72
72
|
<li><a href="#method-c-new">::new</a></li>
|
73
73
|
|
74
|
+
<li><a href="#method-i-compile_code">#compile_code</a></li>
|
75
|
+
|
74
76
|
<li><a href="#method-i-get_acl">#get_acl</a></li>
|
75
77
|
|
76
78
|
<li><a href="#method-i-get_output">#get_output</a></li>
|
@@ -149,6 +151,57 @@ AppScale Tools, but with non-Neptune SOAP calls removed.</p>
|
|
149
151
|
|
150
152
|
<!-- Attributes -->
|
151
153
|
|
154
|
+
<div id="attribute-method-details" class="method-section section">
|
155
|
+
<h3 class="section-header">Attributes</h3>
|
156
|
+
|
157
|
+
|
158
|
+
<div id="conn-attribute-method" class="method-detail">
|
159
|
+
<a name="conn"></a>
|
160
|
+
|
161
|
+
<div class="method-heading attribute-method-heading">
|
162
|
+
<span class="method-name">conn</span><span
|
163
|
+
class="attribute-access-type">[R]</span>
|
164
|
+
</div>
|
165
|
+
|
166
|
+
<div class="method-description">
|
167
|
+
|
168
|
+
|
169
|
+
|
170
|
+
</div>
|
171
|
+
</div>
|
172
|
+
|
173
|
+
<div id="ip-attribute-method" class="method-detail">
|
174
|
+
<a name="ip"></a>
|
175
|
+
|
176
|
+
<div class="method-heading attribute-method-heading">
|
177
|
+
<span class="method-name">ip</span><span
|
178
|
+
class="attribute-access-type">[R]</span>
|
179
|
+
</div>
|
180
|
+
|
181
|
+
<div class="method-description">
|
182
|
+
|
183
|
+
|
184
|
+
|
185
|
+
</div>
|
186
|
+
</div>
|
187
|
+
|
188
|
+
<div id="secret-attribute-method" class="method-detail">
|
189
|
+
<a name="secret"></a>
|
190
|
+
|
191
|
+
<div class="method-heading attribute-method-heading">
|
192
|
+
<span class="method-name">secret</span><span
|
193
|
+
class="attribute-access-type">[R]</span>
|
194
|
+
</div>
|
195
|
+
|
196
|
+
<div class="method-description">
|
197
|
+
|
198
|
+
|
199
|
+
|
200
|
+
</div>
|
201
|
+
</div>
|
202
|
+
|
203
|
+
</div>
|
204
|
+
|
152
205
|
|
153
206
|
<!-- Methods -->
|
154
207
|
|
@@ -191,6 +244,7 @@ def initialize(ip, secret)
|
|
191
244
|
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_get_output"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
192
245
|
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_get_acl"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
193
246
|
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_set_acl"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
247
|
+
<span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">"neptune_compile_code"</span>, <span class="ruby-string">"job_data"</span>, <span class="ruby-string">"secret"</span>)
|
194
248
|
end</pre>
|
195
249
|
</div>
|
196
250
|
|
@@ -208,6 +262,45 @@ end</pre>
|
|
208
262
|
<h3 class="section-header">Public Instance Methods</h3>
|
209
263
|
|
210
264
|
|
265
|
+
<div id="compile_code-method" class="method-detail ">
|
266
|
+
<a name="method-i-compile_code"></a>
|
267
|
+
|
268
|
+
|
269
|
+
<div class="method-heading">
|
270
|
+
<span class="method-name">compile_code</span><span
|
271
|
+
class="method-args">(job_data)</span>
|
272
|
+
<span class="method-click-advice">click to toggle source</span>
|
273
|
+
</div>
|
274
|
+
|
275
|
+
|
276
|
+
<div class="method-description">
|
277
|
+
|
278
|
+
|
279
|
+
|
280
|
+
|
281
|
+
|
282
|
+
<div class="method-source-code"
|
283
|
+
id="compile_code-source">
|
284
|
+
<pre>
|
285
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 137</span>
|
286
|
+
def compile_code(job_data)
|
287
|
+
result = <span class="ruby-string">""</span>
|
288
|
+
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
289
|
+
result = conn.neptune_compile_code(job_data, <span class="ruby-ivar">@secret</span>)
|
290
|
+
}
|
291
|
+
abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
|
292
|
+
return result
|
293
|
+
end</pre>
|
294
|
+
</div>
|
295
|
+
|
296
|
+
</div>
|
297
|
+
|
298
|
+
|
299
|
+
|
300
|
+
|
301
|
+
</div>
|
302
|
+
|
303
|
+
|
211
304
|
<div id="get_acl-method" class="method-detail ">
|
212
305
|
<a name="method-i-get_acl"></a>
|
213
306
|
|
@@ -231,7 +324,7 @@ exceptions mirror that of start_neptune_job.</p>
|
|
231
324
|
<div class="method-source-code"
|
232
325
|
id="get_acl-source">
|
233
326
|
<pre>
|
234
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
327
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 114</span>
|
235
328
|
def get_acl(job_data)
|
236
329
|
result = <span class="ruby-string">""</span>
|
237
330
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -280,7 +373,7 @@ used for job_data.</p>
|
|
280
373
|
<div class="method-source-code"
|
281
374
|
id="get_output-source">
|
282
375
|
<pre>
|
283
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
376
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 100</span>
|
284
377
|
def get_output(job_data)
|
285
378
|
result = <span class="ruby-string">""</span>
|
286
379
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -330,7 +423,7 @@ The result of the block is returned to the caller.</p>
|
|
330
423
|
<div class="method-source-code"
|
331
424
|
id="make_call-source">
|
332
425
|
<pre>
|
333
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
426
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 51</span>
|
334
427
|
def make_call(time, retry_on_except)
|
335
428
|
begin
|
336
429
|
<span class="ruby-constant">Timeout</span>::timeout(time) {
|
@@ -387,7 +480,7 @@ that of start_neptune_job.</p>
|
|
387
480
|
<div class="method-source-code"
|
388
481
|
id="set_acl-source">
|
389
482
|
<pre>
|
390
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
483
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 128</span>
|
391
484
|
def set_acl(job_data)
|
392
485
|
result = <span class="ruby-string">""</span>
|
393
486
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
@@ -433,7 +526,7 @@ is the result returned from the AppController.</p>
|
|
433
526
|
<div class="method-source-code"
|
434
527
|
id="start_neptune_job-source">
|
435
528
|
<pre>
|
436
|
-
<span class="ruby-comment"># File lib/app_controller_client.rb, line
|
529
|
+
<span class="ruby-comment"># File lib/app_controller_client.rb, line 81</span>
|
437
530
|
def start_neptune_job(job_data)
|
438
531
|
result = <span class="ruby-string">""</span>
|
439
532
|
make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
|
data/doc/Object.html
CHANGED
@@ -76,6 +76,8 @@
|
|
76
76
|
|
77
77
|
<li><a href="#method-i-job">#job</a></li>
|
78
78
|
|
79
|
+
<li><a href="#method-i-preprocess_compile">#preprocess_compile</a></li>
|
80
|
+
|
79
81
|
<li><a href="#method-i-preprocess_mapreduce">#preprocess_mapreduce</a></li>
|
80
82
|
|
81
83
|
<li><a href="#method-i-preprocess_mpi">#preprocess_mpi</a></li>
|
@@ -244,17 +246,13 @@ vice-versa).</p>
|
|
244
246
|
<div class="method-source-code"
|
245
247
|
id="job-source">
|
246
248
|
<pre>
|
247
|
-
<span class="ruby-comment"># File lib/job.rb, line
|
249
|
+
<span class="ruby-comment"># File lib/job.rb, line 119</span>
|
248
250
|
def job(name, &block)
|
249
251
|
puts <span class="ruby-string">"Received a request to run a job."</span>
|
250
252
|
puts name
|
251
253
|
block.call()
|
252
254
|
|
253
|
-
|
254
|
-
keyname = <span class="ruby-string">"appscale"</span>
|
255
|
-
else
|
256
|
-
keyname = <span class="ruby-ivar">@keyname</span>
|
257
|
-
end
|
255
|
+
keyname = <span class="ruby-ivar">@keyname</span> || <span class="ruby-string">"appscale"</span>
|
258
256
|
|
259
257
|
shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
|
260
258
|
secret = <span class="ruby-constant">CommonFunctions</span>.get_secret_key(keyname)
|
@@ -270,7 +268,7 @@ def job(name, &block)
|
|
270
268
|
job_data[<span class="ruby-string">"@type"</span>] = name
|
271
269
|
job_data[<span class="ruby-string">"@keyname"</span>] = keyname
|
272
270
|
|
273
|
-
if job_data[<span class="ruby-string">"@output"</span>].nil? or job_data[<span class="ruby-string">"@output"</span>] == <span class="ruby-string">""</span>
|
271
|
+
if (job_data[<span class="ruby-string">"@output"</span>].nil? or job_data[<span class="ruby-string">"@output"</span>] == <span class="ruby-string">""</span>)
|
274
272
|
abort(<span class="ruby-string">"Job output must be specified"</span>)
|
275
273
|
end
|
276
274
|
|
@@ -288,14 +286,38 @@ def job(name, &block)
|
|
288
286
|
|
289
287
|
do_preprocessing(job_data)
|
290
288
|
|
291
|
-
|
289
|
+
type = job_data[<span class="ruby-string">"@type"</span>]
|
290
|
+
if type == <span class="ruby-string">"output"</span>
|
292
291
|
return controller.get_output(job_data)
|
293
|
-
elsif
|
292
|
+
elsif type == <span class="ruby-string">"get-acl"</span>
|
294
293
|
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"acl"</span>
|
295
294
|
return controller.get_acl(job_data)
|
296
|
-
elsif
|
295
|
+
elsif type == <span class="ruby-string">"set-acl"</span>
|
297
296
|
job_data[<span class="ruby-string">"@type"</span>] = <span class="ruby-string">"acl"</span>
|
298
297
|
return controller.set_acl(job_data)
|
298
|
+
elsif type == <span class="ruby-string">"compile"</span>
|
299
|
+
compiled_location = controller.compile_code(job_data)
|
300
|
+
|
301
|
+
ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2>&1 root@#{shadow_ip}"
|
302
|
+
copy_to = job_data[<span class="ruby-string">"@copy_to"</span>]
|
303
|
+
|
304
|
+
loop {
|
305
|
+
ssh_command = "ssh #{ssh_args} 'ls #{compiled_location}'"
|
306
|
+
puts ssh_command
|
307
|
+
result = `#{ssh_command}`
|
308
|
+
puts "result was [#{result}]"
|
309
|
+
if result =~ <span class="ruby-regexp">/No such file or directory/</span>
|
310
|
+
puts <span class="ruby-string">"Still waiting for code to be compiled..."</span>
|
311
|
+
else
|
312
|
+
puts "compilation complete! Copying compiled code to #{copy_to}"
|
313
|
+
break
|
314
|
+
end
|
315
|
+
sleep(5)
|
316
|
+
}
|
317
|
+
|
318
|
+
scp_command = "scp #{ssh_args}:#{compiled_location} #{copy_to}"
|
319
|
+
puts scp_command
|
320
|
+
`#{scp_command}`
|
299
321
|
else
|
300
322
|
result = controller.start_neptune_job(job_data)
|
301
323
|
if result =~ <span class="ruby-regexp">/job is now running\Z/</span>
|
@@ -312,6 +334,53 @@ end</pre>
|
|
312
334
|
|
313
335
|
|
314
336
|
|
337
|
+
</div>
|
338
|
+
|
339
|
+
|
340
|
+
<div id="preprocess_compile-method" class="method-detail ">
|
341
|
+
<a name="method-i-preprocess_compile"></a>
|
342
|
+
|
343
|
+
|
344
|
+
<div class="method-heading">
|
345
|
+
<span class="method-name">preprocess_compile</span><span
|
346
|
+
class="method-args">(job_data)</span>
|
347
|
+
<span class="method-click-advice">click to toggle source</span>
|
348
|
+
</div>
|
349
|
+
|
350
|
+
|
351
|
+
<div class="method-description">
|
352
|
+
|
353
|
+
<p>This preprocessing method copies over the user’s code to the Shadow node
|
354
|
+
so that it can be compiled there. A future version of this method may also
|
355
|
+
copy over libraries as well.</p>
|
356
|
+
|
357
|
+
|
358
|
+
|
359
|
+
<div class="method-source-code"
|
360
|
+
id="preprocess_compile-source">
|
361
|
+
<pre>
|
362
|
+
<span class="ruby-comment"># File lib/job.rb, line 45</span>
|
363
|
+
def preprocess_compile(job_data)
|
364
|
+
code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@code"</span>])
|
365
|
+
unless <span class="ruby-constant">File</span>.exists?(code)
|
366
|
+
abort("The source file #{code} does not exist.")
|
367
|
+
end
|
368
|
+
|
369
|
+
suffix = code.split(<span class="ruby-string">'/'</span>)[-1]
|
370
|
+
dest = "/tmp/#{suffix}"
|
371
|
+
|
372
|
+
keyname = job_data[<span class="ruby-string">"@keyname"</span>]
|
373
|
+
<span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(code, dest, keyname)
|
374
|
+
|
375
|
+
job_data[<span class="ruby-string">"@code"</span>] = dest
|
376
|
+
end</pre>
|
377
|
+
</div>
|
378
|
+
|
379
|
+
</div>
|
380
|
+
|
381
|
+
|
382
|
+
|
383
|
+
|
315
384
|
</div>
|
316
385
|
|
317
386
|
|
@@ -340,7 +409,7 @@ will copy it into HDFS for us.</p>
|
|
340
409
|
<div class="method-source-code"
|
341
410
|
id="preprocess_mapreduce-source">
|
342
411
|
<pre>
|
343
|
-
<span class="ruby-comment"># File lib/job.rb, line
|
412
|
+
<span class="ruby-comment"># File lib/job.rb, line 67</span>
|
344
413
|
def preprocess_mapreduce(job_data)
|
345
414
|
items_to_copy = [<span class="ruby-string">"@map"</span>, <span class="ruby-string">"@reduce"</span>] if job_data[<span class="ruby-string">"@map"</span>] and job_data[<span class="ruby-string">"@reduce"</span>]
|
346
415
|
items_to_copy = [<span class="ruby-string">"@mapreducejar"</span>] if job_data[<span class="ruby-string">"@mapreducejar"</span>]
|
@@ -392,7 +461,7 @@ job.</p>
|
|
392
461
|
<div class="method-source-code"
|
393
462
|
id="preprocess_mpi-source">
|
394
463
|
<pre>
|
395
|
-
<span class="ruby-comment"># File lib/job.rb, line
|
464
|
+
<span class="ruby-comment"># File lib/job.rb, line 90</span>
|
396
465
|
def preprocess_mpi(job_data)
|
397
466
|
source_code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">"@code"</span>])
|
398
467
|
unless <span class="ruby-constant">File</span>.exists?(source_code)
|
data/doc/README.html
CHANGED
@@ -116,11 +116,18 @@ which Neptune is released.</p>
|
|
116
116
|
|
117
117
|
<p>Contributors welcome! We would love to add support for other cloud
|
118
118
|
platforms and test out Neptune more on non-virtualized deployments, as well
|
119
|
-
as adding capabilities for other types of computation
|
119
|
+
as adding capabilities for other types of computation. We would also like
|
120
|
+
to refactor Neptune to use symbols instead of instance variables for
|
121
|
+
running jobs: this will likely appear in a future release as well.</p>
|
120
122
|
|
121
123
|
<p>An academic paper describing Neptune is in the works - check in for a link
|
122
124
|
to that as it becomes available.</p>
|
123
125
|
|
126
|
+
<p>Version History:</p>
|
127
|
+
|
128
|
+
<p>January 27, 20111 - 0.0.1 released, with initial support for MPI, X10, and
|
129
|
+
MapReduce.</p>
|
130
|
+
|
124
131
|
<h1>end</h1>
|
125
132
|
|
126
133
|
</div>
|
data/doc/bin/neptune.html
CHANGED
data/doc/created.rid
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
|
1
|
+
Tue, 01 Feb 2011 12:02:35 -0800
|
2
2
|
./lib/common_functions.rb Thu, 27 Jan 2011 12:52:03 -0800
|
3
3
|
./LICENSE Thu, 27 Jan 2011 13:24:30 -0800
|
4
|
-
./lib/job.rb
|
5
|
-
./lib/app_controller_client.rb
|
6
|
-
./README
|
7
|
-
./bin/neptune Thu, 27 Jan 2011
|
4
|
+
./lib/job.rb Mon, 31 Jan 2011 19:32:41 -0800
|
5
|
+
./lib/app_controller_client.rb Mon, 31 Jan 2011 17:47:55 -0800
|
6
|
+
./README Mon, 31 Jan 2011 19:32:13 -0800
|
7
|
+
./bin/neptune Thu, 27 Jan 2011 22:04:31 -0800
|
data/doc/index.html
CHANGED
@@ -61,6 +61,8 @@
|
|
61
61
|
|
62
62
|
<li><a href="CommonFunctions.html#method-c-scp_to_shadow">::scp_to_shadow — CommonFunctions</a></li>
|
63
63
|
|
64
|
+
<li><a href="AppControllerClient.html#method-i-compile_code">#compile_code — AppControllerClient</a></li>
|
65
|
+
|
64
66
|
<li><a href="Object.html#method-i-do_preprocessing">#do_preprocessing — Object</a></li>
|
65
67
|
|
66
68
|
<li><a href="AppControllerClient.html#method-i-get_acl">#get_acl — AppControllerClient</a></li>
|
@@ -71,6 +73,8 @@
|
|
71
73
|
|
72
74
|
<li><a href="AppControllerClient.html#method-i-make_call">#make_call — AppControllerClient</a></li>
|
73
75
|
|
76
|
+
<li><a href="Object.html#method-i-preprocess_compile">#preprocess_compile — Object</a></li>
|
77
|
+
|
74
78
|
<li><a href="Object.html#method-i-preprocess_mapreduce">#preprocess_mapreduce — Object</a></li>
|
75
79
|
|
76
80
|
<li><a href="Object.html#method-i-preprocess_mpi">#preprocess_mpi — Object</a></li>
|
data/doc/lib/job_rb.html
CHANGED
@@ -32,6 +32,7 @@ class AppControllerClient
|
|
32
32
|
@conn.add_method("neptune_get_output", "job_data", "secret")
|
33
33
|
@conn.add_method("neptune_get_acl", "job_data", "secret")
|
34
34
|
@conn.add_method("neptune_set_acl", "job_data", "secret")
|
35
|
+
@conn.add_method("neptune_compile_code", "job_data", "secret")
|
35
36
|
end
|
36
37
|
|
37
38
|
# A helper method to make SOAP calls for us. This method is mainly here to
|
@@ -132,4 +133,13 @@ class AppControllerClient
|
|
132
133
|
abort(result) if result =~ /Error:/
|
133
134
|
return result
|
134
135
|
end
|
136
|
+
|
137
|
+
def compile_code(job_data)
|
138
|
+
result = ""
|
139
|
+
make_call(NO_TIMEOUT, false) {
|
140
|
+
result = conn.neptune_compile_code(job_data, @secret)
|
141
|
+
}
|
142
|
+
abort(result) if result =~ /Error:/
|
143
|
+
return result
|
144
|
+
end
|
135
145
|
end
|
data/lib/common_functions.rb
CHANGED
@@ -22,31 +22,35 @@ module CommonFunctions
|
|
22
22
|
# placed, and the name of the key to use. The keyname is typically
|
23
23
|
# specified by the Neptune job given, but defaults to ''appscale''
|
24
24
|
# if not provided.
|
25
|
-
def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname)
|
25
|
+
def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname, is_dir=false)
|
26
26
|
shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
|
27
27
|
ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
|
28
28
|
|
29
|
-
self.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key)
|
29
|
+
self.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key, is_dir)
|
30
30
|
end
|
31
31
|
|
32
32
|
# Performs the actual remote copying of files: given the IP address
|
33
33
|
# and other information from scp_to_shadow, attempts to use scp
|
34
34
|
# to copy the file over. Aborts if the scp fails, which can occur
|
35
35
|
# if the network is down, if a bad keyname is provided, or if the
|
36
|
-
# wrong IP is given.
|
37
|
-
|
36
|
+
# wrong IP is given. If the user specifies that the file to copy is
|
37
|
+
# actually a directory, we append the -r flag to scp as well.
|
38
|
+
def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc, is_dir=false)
|
38
39
|
cmd = ""
|
39
40
|
local_file_loc = File.expand_path(local_file_loc)
|
40
|
-
|
41
|
+
|
42
|
+
ssh_args = "-o StrictHostkeyChecking=no 2>&1"
|
43
|
+
ssh_args << " -r " if is_dir
|
44
|
+
|
41
45
|
if public_key_loc.class == Array
|
42
46
|
public_key_loc.each { |key|
|
43
47
|
key = File.expand_path(key)
|
44
48
|
}
|
45
49
|
|
46
|
-
cmd = "scp -i #{public_key_loc.join(' -i ')}
|
50
|
+
cmd = "scp -i #{public_key_loc.join(' -i ')} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
|
47
51
|
else
|
48
52
|
public_key_loc = File.expand_path(public_key_loc)
|
49
|
-
cmd = "scp -i #{public_key_loc}
|
53
|
+
cmd = "scp -i #{public_key_loc} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
|
50
54
|
end
|
51
55
|
|
52
56
|
cmd << "; echo $? >> ~/.appscale/retval"
|
data/lib/job.rb
CHANGED
@@ -18,7 +18,7 @@ $VERBOSE = nil
|
|
18
18
|
|
19
19
|
# A list of jobs that require some kind of work to be done before
|
20
20
|
# the actual computation can be performed.
|
21
|
-
NEED_PREPROCESSING =
|
21
|
+
NEED_PREPROCESSING = ["compile", "mapreduce", "mpi"]
|
22
22
|
|
23
23
|
# A set of methods and constants that we've monkey-patched to enable Neptune
|
24
24
|
# support. In the future, it is likely that the only exposed / monkey-patched
|
@@ -39,20 +39,28 @@ def do_preprocessing(job_data)
|
|
39
39
|
send(preprocess, job_data)
|
40
40
|
end
|
41
41
|
|
42
|
-
# This preprocessing method copies over the user's
|
43
|
-
#
|
44
|
-
#
|
45
|
-
def
|
46
|
-
|
47
|
-
unless File.exists?(
|
48
|
-
|
49
|
-
" didn't exist. Please specify one that exists and try again"
|
50
|
-
abort(file_not_found)
|
42
|
+
# This preprocessing method copies over the user's code to the
|
43
|
+
# Shadow node so that it can be compiled there. A future version
|
44
|
+
# of this method may also copy over libraries as well.
|
45
|
+
def preprocess_compile(job_data)
|
46
|
+
code = File.expand_path(job_data["@code"])
|
47
|
+
unless File.exists?(code)
|
48
|
+
abort("The source file #{code} does not exist.")
|
51
49
|
end
|
52
|
-
dest_code = "/tmp/thempicode"
|
53
50
|
|
51
|
+
suffix = code.split('/')[-1]
|
52
|
+
dest = "/tmp/#{suffix}"
|
54
53
|
keyname = job_data["@keyname"]
|
55
|
-
CommonFunctions.
|
54
|
+
shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
|
55
|
+
|
56
|
+
ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2>&1 root@#{shadow_ip}"
|
57
|
+
remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}'"
|
58
|
+
#puts remove_dir
|
59
|
+
`#{remove_dir}`
|
60
|
+
|
61
|
+
CommonFunctions.scp_to_shadow(code, dest, keyname, is_dir=true)
|
62
|
+
|
63
|
+
job_data["@code"] = dest
|
56
64
|
end
|
57
65
|
|
58
66
|
# This preprocessing method handles copying data for regular
|
@@ -82,6 +90,22 @@ def preprocess_mapreduce(job_data)
|
|
82
90
|
}
|
83
91
|
end
|
84
92
|
|
93
|
+
# This preprocessing method copies over the user's MPI
|
94
|
+
# code to the master node in AppScale - this node will
|
95
|
+
# then copy it to whoever will run the MPI job.
|
96
|
+
def preprocess_mpi(job_data)
|
97
|
+
source_code = File.expand_path(job_data["@code"])
|
98
|
+
unless File.exists?(source_code)
|
99
|
+
file_not_found = "The specified code, #{job_data['@code']}," +
|
100
|
+
" didn't exist. Please specify one that exists and try again"
|
101
|
+
abort(file_not_found)
|
102
|
+
end
|
103
|
+
dest_code = "/tmp/thempicode"
|
104
|
+
|
105
|
+
keyname = job_data["@keyname"]
|
106
|
+
CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
|
107
|
+
end
|
108
|
+
|
85
109
|
# TODO: actually use me!
|
86
110
|
#def validate_args(list)
|
87
111
|
# list.each do |item|
|
@@ -103,11 +127,7 @@ def job(name, &block)
|
|
103
127
|
puts name
|
104
128
|
block.call()
|
105
129
|
|
106
|
-
|
107
|
-
keyname = "appscale"
|
108
|
-
else
|
109
|
-
keyname = @keyname
|
110
|
-
end
|
130
|
+
keyname = @keyname || "appscale"
|
111
131
|
|
112
132
|
shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
|
113
133
|
secret = CommonFunctions.get_secret_key(keyname)
|
@@ -123,7 +143,7 @@ def job(name, &block)
|
|
123
143
|
job_data["@type"] = name
|
124
144
|
job_data["@keyname"] = keyname
|
125
145
|
|
126
|
-
if job_data["@output"].nil? or job_data["@output"] == ""
|
146
|
+
if (job_data["@output"].nil? or job_data["@output"] == "")
|
127
147
|
abort("Job output must be specified")
|
128
148
|
end
|
129
149
|
|
@@ -141,14 +161,46 @@ def job(name, &block)
|
|
141
161
|
|
142
162
|
do_preprocessing(job_data)
|
143
163
|
|
144
|
-
|
164
|
+
type = job_data["@type"]
|
165
|
+
if type == "output"
|
145
166
|
return controller.get_output(job_data)
|
146
|
-
elsif
|
167
|
+
elsif type == "get-acl"
|
147
168
|
job_data["@type"] = "acl"
|
148
169
|
return controller.get_acl(job_data)
|
149
|
-
elsif
|
170
|
+
elsif type == "set-acl"
|
150
171
|
job_data["@type"] = "acl"
|
151
172
|
return controller.set_acl(job_data)
|
173
|
+
elsif type == "compile"
|
174
|
+
compiled_location = controller.compile_code(job_data)
|
175
|
+
|
176
|
+
ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2>&1 root@#{shadow_ip}"
|
177
|
+
copy_to = job_data["@copy_to"]
|
178
|
+
|
179
|
+
loop {
|
180
|
+
ssh_command = "ssh #{ssh_args} 'ls #{compiled_location}'"
|
181
|
+
#puts ssh_command
|
182
|
+
result = `#{ssh_command}`
|
183
|
+
#puts "result was [#{result}]"
|
184
|
+
if result =~ /No such file or directory/
|
185
|
+
puts "Still waiting for code to be compiled..."
|
186
|
+
else
|
187
|
+
puts "compilation complete! Copying compiled code to #{copy_to}"
|
188
|
+
break
|
189
|
+
end
|
190
|
+
sleep(5)
|
191
|
+
}
|
192
|
+
|
193
|
+
rm_local = "rm -rf #{copy_to}"
|
194
|
+
#puts rm_local
|
195
|
+
`#{rm_local}`
|
196
|
+
|
197
|
+
scp_command = "scp -r #{ssh_args}:#{compiled_location} #{copy_to}"
|
198
|
+
#puts scp_command
|
199
|
+
`#{scp_command}`
|
200
|
+
|
201
|
+
out = File.open("#{copy_to}/compile_out") { |f| f.read.chomp! }
|
202
|
+
err = File.open("#{copy_to}/compile_err") { |f| f.read.chomp! }
|
203
|
+
return {:out => out, :err => err }
|
152
204
|
else
|
153
205
|
result = controller.start_neptune_job(job_data)
|
154
206
|
if result =~ /job is now running\Z/
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: neptune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 27
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
9
|
+
- 2
|
10
|
+
version: 0.0.2
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Chris Bunch
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-02-04 00:00:00 -08:00
|
19
19
|
default_executable: neptune
|
20
20
|
dependencies: []
|
21
21
|
|