neptune 0.0.1 → 0.0.2

data/README

@@ -21,10 +21,15 @@ it also can be used to deploy other types of software. For example,
 Neptune has support for user-specified scaling of the underlying
 cloud platform: users can write Neptune jobs that manually add
 load balancers, application servers, or database servers to a running
-AppScale deployment.
+AppScale deployment. Additionally, as of version 0.0.2, remote
+compiling can also be performed: just give Neptune the path to
+the directory you want to compile and be sure to include a Makefile
+in it! Neptune will run 'make' on it (you can specify which target
+to make as well) and return to you a folder containing the standard
+out and standard error of the make command.
 
 Sample Neptune job scripts can be found in samples. Test scripts will
-be added to test in the next release.
+be added to the 'test' folder soon.
 
 Developed by Chris Bunch as part of the AppScale project.
 See LICENSE for the specifics of the New BSD License
@@ -46,5 +51,13 @@ will likely appear in a future release as well.
 An academic paper describing Neptune is in the works - check
 in for a link to that as it becomes available.
 
+Version History:
+
+February 4, 2011 - 0.0.2 released, adding support for remote
+compiling
+
+January 27, 2011 - 0.0.1 released, with initial support for
+MPI, X10, and MapReduce.
+
 =end
 

data/doc/AppControllerClient.html

@@ -71,6 +71,8 @@
           
           <li><a href="#method-c-new">::new</a></li>
           
+          <li><a href="#method-i-compile_code">#compile_code</a></li>
+          
           <li><a href="#method-i-get_acl">#get_acl</a></li>
           
           <li><a href="#method-i-get_output">#get_output</a></li>
@@ -149,6 +151,57 @@ AppScale Tools, but with non-Neptune SOAP calls removed.</p>
 
     <!-- Attributes -->
     
+    <div id="attribute-method-details" class="method-section section">
+      <h3 class="section-header">Attributes</h3>
+
+      
+      <div id="conn-attribute-method" class="method-detail">
+        <a name="conn"></a>
+        
+        <div class="method-heading attribute-method-heading">
+          <span class="method-name">conn</span><span
+            class="attribute-access-type">[R]</span>
+        </div>
+
+        <div class="method-description">
+        
+        
+        
+        </div>
+      </div>
+      
+      <div id="ip-attribute-method" class="method-detail">
+        <a name="ip"></a>
+        
+        <div class="method-heading attribute-method-heading">
+          <span class="method-name">ip</span><span
+            class="attribute-access-type">[R]</span>
+        </div>
+
+        <div class="method-description">
+        
+        
+        
+        </div>
+      </div>
+      
+      <div id="secret-attribute-method" class="method-detail">
+        <a name="secret"></a>
+        
+        <div class="method-heading attribute-method-heading">
+          <span class="method-name">secret</span><span
+            class="attribute-access-type">[R]</span>
+        </div>
+
+        <div class="method-description">
+        
+        
+        
+        </div>
+      </div>
+      
+    </div>
+    
 
     <!-- Methods -->
     
@@ -191,6 +244,7 @@ def initialize(ip, secret)
   <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_get_output&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
   <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_get_acl&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
   <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_set_acl&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
+  <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_compile_code&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
 end</pre>
           </div>
           
@@ -208,6 +262,45 @@ end</pre>
       <h3 class="section-header">Public Instance Methods</h3>
 
     
+      <div id="compile_code-method" class="method-detail ">
+        <a name="method-i-compile_code"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">compile_code</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          
+          
+
+          
+          <div class="method-source-code"
+            id="compile_code-source">
+<pre>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 137</span>
+def compile_code(job_data)
+  result = <span class="ruby-string">&quot;&quot;</span>
+  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
+    result = conn.neptune_compile_code(job_data, <span class="ruby-ivar">@secret</span>)
+  }  
+  abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  return result
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
       <div id="get_acl-method" class="method-detail ">
         <a name="method-i-get_acl"></a>
 
@@ -231,7 +324,7 @@ exceptions mirror that of start_neptune_job.</p>
           <div class="method-source-code"
             id="get_acl-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 113</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 114</span>
 def get_acl(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
@@ -280,7 +373,7 @@ used for job_data.</p>
           <div class="method-source-code"
             id="get_output-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 99</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 100</span>
 def get_output(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
@@ -330,7 +423,7 @@ The result of the block is returned to the caller.</p>
           <div class="method-source-code"
             id="make_call-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 50</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 51</span>
 def make_call(time, retry_on_except)
   begin
     <span class="ruby-constant">Timeout</span>::timeout(time) {
@@ -387,7 +480,7 @@ that of start_neptune_job.</p>
           <div class="method-source-code"
             id="set_acl-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 127</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 128</span>
 def set_acl(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
@@ -433,7 +526,7 @@ is the result returned from the AppController.</p>
           <div class="method-source-code"
             id="start_neptune_job-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 80</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 81</span>
 def start_neptune_job(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 

data/doc/Object.html

@@ -76,6 +76,8 @@
           
           <li><a href="#method-i-job">#job</a></li>
           
+          <li><a href="#method-i-preprocess_compile">#preprocess_compile</a></li>
+          
           <li><a href="#method-i-preprocess_mapreduce">#preprocess_mapreduce</a></li>
           
           <li><a href="#method-i-preprocess_mpi">#preprocess_mpi</a></li>
@@ -244,17 +246,13 @@ vice-versa).</p>
           <div class="method-source-code"
             id="job-source">
 <pre>
-<span class="ruby-comment"># File lib/job.rb, line 101</span>
+<span class="ruby-comment"># File lib/job.rb, line 119</span>
 def job(name, &amp;block)
   puts <span class="ruby-string">&quot;Received a request to run a job.&quot;</span>
   puts name
   block.call()
 
-  if <span class="ruby-ivar">@keyname</span>.nil?
-    keyname = <span class="ruby-string">&quot;appscale&quot;</span>
-  else
-    keyname = <span class="ruby-ivar">@keyname</span>
-  end
+  keyname = <span class="ruby-ivar">@keyname</span> || <span class="ruby-string">&quot;appscale&quot;</span>
 
   shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
   secret = <span class="ruby-constant">CommonFunctions</span>.get_secret_key(keyname)
@@ -270,7 +268,7 @@ def job(name, &amp;block)
   job_data[<span class="ruby-string">&quot;@type&quot;</span>] = name
   job_data[<span class="ruby-string">&quot;@keyname&quot;</span>] = keyname
 
-  if job_data[<span class="ruby-string">&quot;@output&quot;</span>].nil? or job_data[<span class="ruby-string">&quot;@output&quot;</span>] == <span class="ruby-string">&quot;&quot;</span>
+  if (job_data[<span class="ruby-string">&quot;@output&quot;</span>].nil? or job_data[<span class="ruby-string">&quot;@output&quot;</span>] == <span class="ruby-string">&quot;&quot;</span>)
     abort(<span class="ruby-string">&quot;Job output must be specified&quot;</span>)
   end
 
@@ -288,14 +286,38 @@ def job(name, &amp;block)
 
   do_preprocessing(job_data) 
 
-  if job_data[<span class="ruby-string">&quot;@type&quot;</span>] == <span class="ruby-string">&quot;output&quot;</span>
+  type = job_data[<span class="ruby-string">&quot;@type&quot;</span>]
+  if type == <span class="ruby-string">&quot;output&quot;</span>
     return controller.get_output(job_data)
-  elsif job_data[<span class="ruby-string">&quot;@type&quot;</span>] == <span class="ruby-string">&quot;get-acl&quot;</span>
+  elsif type == <span class="ruby-string">&quot;get-acl&quot;</span>
     job_data[<span class="ruby-string">&quot;@type&quot;</span>] = <span class="ruby-string">&quot;acl&quot;</span>
     return controller.get_acl(job_data)
-  elsif job_data[<span class="ruby-string">&quot;@type&quot;</span>] == <span class="ruby-string">&quot;set-acl&quot;</span>
+  elsif type == <span class="ruby-string">&quot;set-acl&quot;</span>
     job_data[<span class="ruby-string">&quot;@type&quot;</span>] = <span class="ruby-string">&quot;acl&quot;</span>
     return controller.set_acl(job_data)
+  elsif type == <span class="ruby-string">&quot;compile&quot;</span>
+    compiled_location = controller.compile_code(job_data)
+
+    ssh_args = &quot;-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2&gt;&amp;1 root@#{shadow_ip}&quot;
+    copy_to = job_data[<span class="ruby-string">&quot;@copy_to&quot;</span>]
+
+    loop {
+      ssh_command = &quot;ssh #{ssh_args} 'ls #{compiled_location}'&quot;
+      puts ssh_command
+      result = `#{ssh_command}`
+      puts &quot;result was [#{result}]&quot;
+      if result =~ <span class="ruby-regexp">/No such file or directory/</span>
+        puts <span class="ruby-string">&quot;Still waiting for code to be compiled...&quot;</span>
+      else
+        puts &quot;compilation complete! Copying compiled code to #{copy_to}&quot;
+        break
+      end
+      sleep(5)
+    }
+
+    scp_command = &quot;scp #{ssh_args}:#{compiled_location} #{copy_to}&quot;
+    puts scp_command
+    `#{scp_command}`
   else
     result = controller.start_neptune_job(job_data)
     if result =~ <span class="ruby-regexp">/job is now running\Z/</span>
@@ -312,6 +334,53 @@ end</pre>
         
 
         
+      </div>
+
+    
+      <div id="preprocess_compile-method" class="method-detail ">
+        <a name="method-i-preprocess_compile"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">preprocess_compile</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>This preprocessing method copies over the user’s code to the Shadow node
+so that it can be compiled there. A future version of this method may also
+copy over libraries as well.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="preprocess_compile-source">
+<pre>
+<span class="ruby-comment"># File lib/job.rb, line 45</span>
+def preprocess_compile(job_data)
+  code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
+  unless <span class="ruby-constant">File</span>.exists?(code)
+    abort(&quot;The source file #{code} does not exist.&quot;)
+  end
+
+  suffix = code.split(<span class="ruby-string">'/'</span>)[-1]
+  dest = &quot;/tmp/#{suffix}&quot;
+
+  keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>]
+  <span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(code, dest, keyname)
+
+  job_data[<span class="ruby-string">&quot;@code&quot;</span>] = dest
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
       </div>
 
     
@@ -340,7 +409,7 @@ will copy it into HDFS for us.</p>
           <div class="method-source-code"
             id="preprocess_mapreduce-source">
 <pre>
-<span class="ruby-comment"># File lib/job.rb, line 65</span>
+<span class="ruby-comment"># File lib/job.rb, line 67</span>
 def preprocess_mapreduce(job_data)
   items_to_copy = [<span class="ruby-string">&quot;@map&quot;</span>, <span class="ruby-string">&quot;@reduce&quot;</span>] if job_data[<span class="ruby-string">&quot;@map&quot;</span>] and job_data[<span class="ruby-string">&quot;@reduce&quot;</span>]
   items_to_copy = [<span class="ruby-string">&quot;@mapreducejar&quot;</span>] if job_data[<span class="ruby-string">&quot;@mapreducejar&quot;</span>]
@@ -392,7 +461,7 @@ job.</p>
           <div class="method-source-code"
             id="preprocess_mpi-source">
 <pre>
-<span class="ruby-comment"># File lib/job.rb, line 45</span>
+<span class="ruby-comment"># File lib/job.rb, line 90</span>
 def preprocess_mpi(job_data)
   source_code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
   unless <span class="ruby-constant">File</span>.exists?(source_code)

data/doc/README.html

@@ -116,11 +116,18 @@ which Neptune is released.</p>
 
 <p>Contributors welcome! We would love to add support for other cloud
 platforms and test out Neptune more on non-virtualized deployments, as well
-as adding capabilities for other types of computation.</p>
+as adding capabilities for other types of computation. We would also like
+to refactor Neptune to use symbols instead of instance variables for
+running jobs: this will likely appear in a future release as well.</p>
 
 <p>An academic paper describing Neptune is in the works - check in for a link
 to that as it becomes available.</p>
 
+<p>Version History:</p>
+
+<p>January 27, 20111 - 0.0.1 released, with initial support for MPI, X10, and
+MapReduce.</p>
+
 <h1>end</h1>
 
   </div>

data/doc/bin/neptune.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Thu Jan 27 03:28:28 -0800 2011</dd>
+      <dd class="modified-date">Thu Jan 27 22:04:31 -0800 2011</dd>
 
       
       <dt class="requires">Requires</dt>

data/doc/created.rid

@@ -1,7 +1,7 @@
-Thu, 27 Jan 2011 13:44:54 -0800
+Tue, 01 Feb 2011 12:02:35 -0800
 ./lib/common_functions.rb	Thu, 27 Jan 2011 12:52:03 -0800
 ./LICENSE	Thu, 27 Jan 2011 13:24:30 -0800
-./lib/job.rb	Thu, 27 Jan 2011 13:28:13 -0800
-./lib/app_controller_client.rb	Thu, 27 Jan 2011 13:22:15 -0800
-./README	Thu, 27 Jan 2011 13:44:46 -0800
-./bin/neptune	Thu, 27 Jan 2011 03:28:28 -0800
+./lib/job.rb	Mon, 31 Jan 2011 19:32:41 -0800
+./lib/app_controller_client.rb	Mon, 31 Jan 2011 17:47:55 -0800
+./README	Mon, 31 Jan 2011 19:32:13 -0800
+./bin/neptune	Thu, 27 Jan 2011 22:04:31 -0800

data/doc/index.html

@@ -61,6 +61,8 @@
     
       <li><a href="CommonFunctions.html#method-c-scp_to_shadow">::scp_to_shadow &mdash; CommonFunctions</a></li>
     
+      <li><a href="AppControllerClient.html#method-i-compile_code">#compile_code &mdash; AppControllerClient</a></li>
+    
       <li><a href="Object.html#method-i-do_preprocessing">#do_preprocessing &mdash; Object</a></li>
     
       <li><a href="AppControllerClient.html#method-i-get_acl">#get_acl &mdash; AppControllerClient</a></li>
@@ -71,6 +73,8 @@
     
       <li><a href="AppControllerClient.html#method-i-make_call">#make_call &mdash; AppControllerClient</a></li>
     
+      <li><a href="Object.html#method-i-preprocess_compile">#preprocess_compile &mdash; Object</a></li>
+    
       <li><a href="Object.html#method-i-preprocess_mapreduce">#preprocess_mapreduce &mdash; Object</a></li>
     
       <li><a href="Object.html#method-i-preprocess_mpi">#preprocess_mpi &mdash; Object</a></li>

data/doc/lib/app_controller_client_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Thu Jan 27 13:22:15 -0800 2011</dd>
+      <dd class="modified-date">Mon Jan 31 17:47:55 -0800 2011</dd>
 
       
       <dt class="requires">Requires</dt>

data/doc/lib/job_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Thu Jan 27 13:28:13 -0800 2011</dd>
+      <dd class="modified-date">Mon Jan 31 19:32:41 -0800 2011</dd>
 
       
       <dt class="requires">Requires</dt>

data/lib/app_controller_client.rb

@@ -32,6 +32,7 @@ class AppControllerClient
     @conn.add_method("neptune_get_output", "job_data", "secret")
     @conn.add_method("neptune_get_acl", "job_data", "secret")
     @conn.add_method("neptune_set_acl", "job_data", "secret")
+    @conn.add_method("neptune_compile_code", "job_data", "secret")
   end
 
   # A helper method to make SOAP calls for us. This method is mainly here to
@@ -132,4 +133,13 @@ class AppControllerClient
     abort(result) if result =~ /Error:/
     return result
   end
+
+  def compile_code(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) { 
+      result = conn.neptune_compile_code(job_data, @secret)
+    }  
+    abort(result) if result =~ /Error:/
+    return result
+  end
 end

data/lib/common_functions.rb

@@ -22,31 +22,35 @@ module CommonFunctions
   # placed, and the name of the key to use. The keyname is typically
   # specified by the Neptune job given, but defaults to ''appscale''
   # if not provided.
-  def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname)
+  def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname, is_dir=false)
     shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
     ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
 
-    self.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key)
+    self.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key, is_dir)
   end 
  
   # Performs the actual remote copying of files: given the IP address
   # and other information from scp_to_shadow, attempts to use scp
   # to copy the file over. Aborts if the scp fails, which can occur
   # if the network is down, if a bad keyname is provided, or if the 
-  # wrong IP is given.
-  def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc)
+  # wrong IP is given. If the user specifies that the file to copy is
+  # actually a directory, we append the -r flag to scp as well.
+  def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc, is_dir=false)
     cmd = ""
     local_file_loc = File.expand_path(local_file_loc)
-    
+
+    ssh_args = "-o StrictHostkeyChecking=no 2>&1"
+    ssh_args << " -r " if is_dir
+
     if public_key_loc.class == Array
       public_key_loc.each { |key|
         key = File.expand_path(key)
       }
       
-      cmd = "scp -i #{public_key_loc.join(' -i ')} -o StrictHostkeyChecking=no 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
+      cmd = "scp -i #{public_key_loc.join(' -i ')} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
     else
       public_key_loc = File.expand_path(public_key_loc)
-      cmd = "scp -i #{public_key_loc} -o StrictHostkeyChecking=no 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
+      cmd = "scp -i #{public_key_loc} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
     end
 
     cmd << "; echo $? >> ~/.appscale/retval"

data/lib/job.rb

@@ -18,7 +18,7 @@ $VERBOSE = nil
 
 # A list of jobs that require some kind of work to be done before
 # the actual computation can be performed.
-NEED_PREPROCESSING = %w{ mpi mapreduce }
+NEED_PREPROCESSING = ["compile", "mapreduce", "mpi"]
 
 # A set of methods and constants that we've monkey-patched to enable Neptune
 # support. In the future, it is likely that the only exposed / monkey-patched
@@ -39,20 +39,28 @@ def do_preprocessing(job_data)
   send(preprocess, job_data)
 end
 
-# This preprocessing method copies over the user's MPI
-# code to the master node in AppScale - this node will
-# then copy it to whoever will run the MPI job.
-def preprocess_mpi(job_data)
-  source_code = File.expand_path(job_data["@code"])
-  unless File.exists?(source_code)
-    file_not_found = "The specified code, #{job_data['@code']}," +
-      " didn't exist. Please specify one that exists and try again"
-    abort(file_not_found)
+# This preprocessing method copies over the user's code to the
+# Shadow node so that it can be compiled there. A future version
+# of this method may also copy over libraries as well.
+def preprocess_compile(job_data)
+  code = File.expand_path(job_data["@code"])
+  unless File.exists?(code)
+    abort("The source file #{code} does not exist.")
   end
-  dest_code = "/tmp/thempicode"
 
+  suffix = code.split('/')[-1]
+  dest = "/tmp/#{suffix}"
   keyname = job_data["@keyname"]
-  CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
+  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+
+  ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2>&1 root@#{shadow_ip}"
+  remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}'"
+  #puts remove_dir
+  `#{remove_dir}`
+
+  CommonFunctions.scp_to_shadow(code, dest, keyname, is_dir=true)
+
+  job_data["@code"] = dest
 end
 
 # This preprocessing method handles copying data for regular
@@ -82,6 +90,22 @@ def preprocess_mapreduce(job_data)
   }
 end
 
+# This preprocessing method copies over the user's MPI
+# code to the master node in AppScale - this node will
+# then copy it to whoever will run the MPI job.
+def preprocess_mpi(job_data)
+  source_code = File.expand_path(job_data["@code"])
+  unless File.exists?(source_code)
+    file_not_found = "The specified code, #{job_data['@code']}," +
+      " didn't exist. Please specify one that exists and try again"
+    abort(file_not_found)
+  end
+  dest_code = "/tmp/thempicode"
+
+  keyname = job_data["@keyname"]
+  CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
+end
+
 # TODO: actually use me!
 #def validate_args(list)
 #  list.each do |item|
@@ -103,11 +127,7 @@ def job(name, &block)
   puts name
   block.call()
 
-  if @keyname.nil?
-    keyname = "appscale"
-  else
-    keyname = @keyname
-  end
+  keyname = @keyname || "appscale"
 
   shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
   secret = CommonFunctions.get_secret_key(keyname)
@@ -123,7 +143,7 @@ def job(name, &block)
   job_data["@type"] = name
   job_data["@keyname"] = keyname
 
-  if job_data["@output"].nil? or job_data["@output"] == ""
+  if (job_data["@output"].nil? or job_data["@output"] == "")
     abort("Job output must be specified")
   end
 
@@ -141,14 +161,46 @@ def job(name, &block)
 
   do_preprocessing(job_data) 
 
-  if job_data["@type"] == "output"
+  type = job_data["@type"]
+  if type == "output"
     return controller.get_output(job_data)
-  elsif job_data["@type"] == "get-acl"
+  elsif type == "get-acl"
     job_data["@type"] = "acl"
     return controller.get_acl(job_data)
-  elsif job_data["@type"] == "set-acl"
+  elsif type == "set-acl"
     job_data["@type"] = "acl"
     return controller.set_acl(job_data)
+  elsif type == "compile"
+    compiled_location = controller.compile_code(job_data)
+
+    ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2>&1 root@#{shadow_ip}"
+    copy_to = job_data["@copy_to"]
+
+    loop {
+      ssh_command = "ssh #{ssh_args} 'ls #{compiled_location}'"
+      #puts ssh_command
+      result = `#{ssh_command}`
+      #puts "result was [#{result}]"
+      if result =~ /No such file or directory/
+        puts "Still waiting for code to be compiled..."
+      else
+        puts "compilation complete! Copying compiled code to #{copy_to}"
+        break
+      end
+      sleep(5)
+    }
+
+    rm_local = "rm -rf #{copy_to}"
+    #puts rm_local
+    `#{rm_local}`
+
+    scp_command = "scp -r #{ssh_args}:#{compiled_location} #{copy_to}"
+    #puts scp_command
+    `#{scp_command}`
+
+    out = File.open("#{copy_to}/compile_out") { |f| f.read.chomp! }
+    err = File.open("#{copy_to}/compile_err") { |f| f.read.chomp! }
+    return {:out => out, :err => err }
   else
     result = controller.start_neptune_job(job_data)
     if result =~ /job is now running\Z/

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: neptune
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: 
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Chris Bunch
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-27 00:00:00 -08:00
+date: 2011-02-04 00:00:00 -08:00
 default_executable: neptune
 dependencies: []