neptune 0.1.1 → 0.1.2

data/doc/LICENSE.html

@@ -66,6 +66,8 @@
         
           <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
         
+          <li><a href="./Kernel.html">Kernel</a></li>
+        
           <li><a href="./Object.html">Object</a></li>
         
         </ul>

data/doc/Object.html

@@ -72,18 +72,32 @@
         <h3 class="section-header">Methods</h3>
         <ul class="link-list">
           
+          <li><a href="#method-i-compile_code">#compile_code</a></li>
+          
           <li><a href="#method-i-do_preprocessing">#do_preprocessing</a></li>
           
+          <li><a href="#method-i-get_input">#get_input</a></li>
+          
+          <li><a href="#method-i-get_job_data">#get_job_data</a></li>
+          
+          <li><a href="#method-i-get_std_out_and_err">#get_std_out_and_err</a></li>
+          
           <li><a href="#method-i-neptune">#neptune</a></li>
           
           <li><a href="#method-i-preprocess_compile">#preprocess_compile</a></li>
           
           <li><a href="#method-i-preprocess_erlang">#preprocess_erlang</a></li>
           
-          <li><a href="#method-i-preprocess_mapreduce">#preprocess_mapreduce</a></li>
-          
           <li><a href="#method-i-preprocess_mpi">#preprocess_mpi</a></li>
           
+          <li><a href="#method-i-preprocess_ssa">#preprocess_ssa</a></li>
+          
+          <li><a href="#method-i-run_job">#run_job</a></li>
+          
+          <li><a href="#method-i-validate_storage_params">#validate_storage_params</a></li>
+          
+          <li><a href="#method-i-wait_for_compilation_to_finish">#wait_for_compilation_to_finish</a></li>
+          
         </ul>
       </div>
       
@@ -126,6 +140,8 @@
         
           <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
         
+          <li><a href="./Kernel.html">Kernel</a></li>
+        
           <li><a href="./Object.html">Object</a></li>
         
         </ul>
@@ -145,7 +161,9 @@
 Neptune support. In the future, it is likely that the only exposed /
 monkey-patched method should be job, while the others could probably be
 folded into either a Neptune-specific class or into <a
-href="CommonFunctions.html">CommonFunctions</a>.</p>
+href="CommonFunctions.html">CommonFunctions</a>. TODO(cbunch): This
+doesn’t look like it does anything - run the integration test and confirm
+one way or the other.</p>
 
     </div>
 
@@ -201,6 +219,65 @@ computation can be performed.</p></dd>
       <h3 class="section-header">Public Instance Methods</h3>
 
     
+      <div id="compile_code-method" class="method-detail ">
+        <a name="method-i-compile_code"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">compile_code</span><span
+            class="method-args">(job_data, ssh_args, shadow_ip, shell=Kernel.method(:`))</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>This method sends out a request to compile code, waits for it to finish,
+and gets the standard out and error returned from the compilation. This
+method returns a hash containing the standard out, error, and a result that
+indicates whether or not the compilation was successful.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="compile_code-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 281</span>
+def compile_code(job_data, ssh_args, shadow_ip, shell=<span class="ruby-constant">Kernel</span>.method(:`))
+  compiled_location = controller.compile_code(job_data)
+
+  copy_to = job_data[<span class="ruby-string">&quot;@copy_to&quot;</span>]
+
+  wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
+
+  <span class="ruby-constant">FileUtils</span>.rm_rf(copy_to)
+
+  scp_command = &quot;scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2&gt;&amp;1&quot;
+  puts scp_command
+  shell.call(scp_command)
+
+  code = job_data[<span class="ruby-string">&quot;@code&quot;</span>]
+  dirs = code.split(<span class="ruby-regexp">/\//</span>)
+  remote_dir = <span class="ruby-string">&quot;/tmp/&quot;</span> + dirs[-1] 
+
+  [remote_dir, compiled_location].each { |remote_files|
+    ssh_command = &quot;ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_files}' 2&gt;&amp;1&quot;
+    puts ssh_command
+    shell.call(ssh_command)
+  }
+
+  return get_std_out_and_err(copy_to)
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
       <div id="do_preprocessing-method" class="method-detail ">
         <a name="method-i-do_preprocessing"></a>
 
@@ -223,10 +300,12 @@ method to use based on the type of the job that the user has asked to run.</p>
           <div class="method-source-code"
             id="do_preprocessing-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 49</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 52</span>
 def do_preprocessing(job_data)
   job_type = job_data[<span class="ruby-string">&quot;@type&quot;</span>]
-  return unless <span class="ruby-constant">NEED_PREPROCESSING</span>.include?(job_type)
+  if !<span class="ruby-constant">NEED_PREPROCESSING</span>.include?(job_type)
+    return
+  end
 
   preprocess = &quot;preprocess_#{job_type}&quot;.to_sym
   send(preprocess, job_data)
@@ -241,51 +320,101 @@ end</pre>
       </div>
 
     
-      <div id="neptune-method" class="method-detail ">
-        <a name="method-i-neptune"></a>
+      <div id="get_input-method" class="method-detail ">
+        <a name="method-i-get_input"></a>
 
         
         <div class="method-heading">
-          <span class="method-name">neptune</span><span
-            class="method-args">(params)</span>
+          <span class="method-name">get_input</span><span
+            class="method-args">(job_data, ssh_args, shadow_ip, controller, file=File, shell=Kernel.method(:`))</span>
           <span class="method-click-advice">click to toggle source</span>
         </div>
         
 
         <div class="method-description">
           
-          <p>This method is the heart of Neptune - here, we take blocks of code that the
-user has written and convert them into HPC job requests. At a high level,
-the user can request to run a job, retrieve a job’s output, or modify the
-access policy (ACL) for the output of a job. By default, job data is
-private, but a Neptune job can be used to set it to public later (and
-vice-versa).</p>
+          <p>This method takes a file on the local user’s computer and stores it
+remotely via AppScale. It returns a hash map indicating whether or not the
+job succeeded and if it failed, the reason for it.</p>
           
 
           
           <div class="method-source-code"
-            id="neptune-source">
+            id="get_input-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 175</span>
-def neptune(params)
-  puts <span class="ruby-string">&quot;Received a request to run a job.&quot;</span>
-  puts params[:type]
+<span class="ruby-comment"># File lib/neptune.rb, line 227</span>
+def get_input(job_data, ssh_args, shadow_ip, controller, file=<span class="ruby-constant">File</span>,
+  shell=<span class="ruby-constant">Kernel</span>.method(:`))
+  result = {:result =&gt; :success}
 
-  keyname = params[:keyname] || <span class="ruby-string">&quot;appscale&quot;</span>
+  if !job_data[<span class="ruby-string">&quot;@local&quot;</span>]
+    abort(<span class="ruby-string">&quot;You failed to specify a file to copy over via the :local flag.&quot;</span>)
+  end
 
-  shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
-  secret = <span class="ruby-constant">CommonFunctions</span>.get_secret_key(keyname)
-  controller = <span class="ruby-constant">AppControllerClient</span>.new(shadow_ip, secret)
-  ssh_key = <span class="ruby-constant">File</span>.expand_path(&quot;~/.appscale/#{keyname}.key&quot;)
+  local_file = file.expand_path(job_data[<span class="ruby-string">&quot;@local&quot;</span>])
+  if !file.exists?(local_file)
+    reason = &quot;the file you specified to copy, #{local_file}, doesn't exist.&quot; + 
+        <span class="ruby-string">&quot; Please specify a file that exists and try again.&quot;</span>
+    return {:result =&gt; :failure, :reason =&gt; reason}  
+  end
+
+  remote = &quot;/tmp/neptune-input-#{rand(100000)}&quot;
+  scp_cmd = &quot;scp -r #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}&quot;
+  puts scp_cmd
+  shell.call(scp_cmd)
+
+  job_data[<span class="ruby-string">&quot;@local&quot;</span>] = remote
+  puts &quot;job data = #{job_data.inspect}&quot;
+  response = controller.put_input(job_data)
+  if response
+    return {:result =&gt; :success}
+  else
+    <span class="ruby-comment"># TODO - expand this to include the reason why it failed</span>
+    return {:result =&gt; :failure}
+  end
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
 
+    
+      <div id="get_job_data-method" class="method-detail ">
+        <a name="method-i-get_job_data"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">get_job_data</span><span
+            class="method-args">(params)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          
+          
+
+          
+          <div class="method-source-code"
+            id="get_job_data-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 151</span>
+def get_job_data(params)
   job_data = {}
   params.each { |k, v|
     key = &quot;@#{k}&quot;
     job_data[key] = v
   }
 
-  job_data[<span class="ruby-string">&quot;@job&quot;</span>] = nil
-  job_data[<span class="ruby-string">&quot;@keyname&quot;</span>] = keyname || <span class="ruby-string">&quot;appscale&quot;</span>
+  job_data.delete(<span class="ruby-string">&quot;@job&quot;</span>)
+  job_data[<span class="ruby-string">&quot;@keyname&quot;</span>] = params[:keyname] || <span class="ruby-string">&quot;appscale&quot;</span>
+
+  job_data[<span class="ruby-string">&quot;@type&quot;</span>] = job_data[<span class="ruby-string">&quot;@type&quot;</span>].to_s
   type = job_data[<span class="ruby-string">&quot;@type&quot;</span>]
 
   if type == <span class="ruby-string">&quot;upc&quot;</span> or type == <span class="ruby-string">&quot;x10&quot;</span>
@@ -307,115 +436,110 @@ def neptune(params)
     end
   end
 
-  if job_data[<span class="ruby-string">&quot;@storage&quot;</span>]
-    storage = job_data[<span class="ruby-string">&quot;@storage&quot;</span>]
-    unless <span class="ruby-constant">ALLOWED_STORAGE_TYPES</span>.include?(storage)
-      msg = &quot;Supported storage types are #{ALLOWED_STORAGE_TYPES.join(', ')}&quot; +
-        &quot; - we do not support #{storage}.&quot;
-      abort(msg)
-    end
+  return job_data
+end</pre>
+          </div>
+          
+        </div>
 
-    <span class="ruby-comment"># Our implementation for storing / retrieving via Google Storage uses</span>
-    <span class="ruby-comment"># the same library as we do for S3 - so just tell it that it's S3</span>
-    if storage == <span class="ruby-string">&quot;gstorage&quot;</span>
-      storage = <span class="ruby-string">&quot;s3&quot;</span>
-      job_data[<span class="ruby-string">&quot;@storage&quot;</span>] = <span class="ruby-string">&quot;s3&quot;</span>
-    end
+        
 
-    if storage == <span class="ruby-string">&quot;s3&quot;</span>
-      [<span class="ruby-string">&quot;EC2_ACCESS_KEY&quot;</span>, <span class="ruby-string">&quot;EC2_SECRET_KEY&quot;</span>, <span class="ruby-string">&quot;S3_URL&quot;</span>].each { |item|
-        unless job_data[&quot;@#{item}&quot;]
-          if <span class="ruby-constant">ENV</span>[item]
-            puts &quot;Using #{item} from environment&quot;
-            job_data[&quot;@#{item}&quot;] = <span class="ruby-constant">ENV</span>[item]
-          else
-            msg = &quot;When storing data to S3, #{item} must be specified or be in &quot; + 
-              <span class="ruby-string">&quot;your environment. Please do so and try again.&quot;</span>
-            abort(msg)
-          end
-        end
-      }
-    end
+        
+      </div>
+
+    
+      <div id="get_std_out_and_err-method" class="method-detail ">
+        <a name="method-i-get_std_out_and_err"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">get_std_out_and_err</span><span
+            class="method-args">(location)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>This method returns a hash containing the standard out and standard error
+from a completed job, as well as a result field that indicates whether or
+not the job completed successfully (success = no errors).</p>
+          
+
+          
+          <div class="method-source-code"
+            id="get_std_out_and_err-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 310</span>
+def get_std_out_and_err(location)
+  result = {}
+
+  out = <span class="ruby-constant">File</span>.open(&quot;#{location}/compile_out&quot;) { |f| f.read.chomp! }
+  result[:out] = out
+
+  err = <span class="ruby-constant">File</span>.open(&quot;#{location}/compile_err&quot;) { |f| f.read.chomp! }
+  result[:err] = err
+
+  if result[:err]
+    result[:result] = :failure
   else
-    job_data[<span class="ruby-string">&quot;@storage&quot;</span>] = <span class="ruby-string">&quot;appdb&quot;</span>
-  end
+    result[:result] = :success
+  end    
 
-  <span class="ruby-comment">#if job_data[&quot;@can_run_on&quot;].class == Range</span>
-  <span class="ruby-comment">#  job_data[&quot;@can_run_on&quot;] = job_data[&quot;@can_run_on&quot;].to_a</span>
-  <span class="ruby-comment">#elsif job_data[&quot;@can_run_on&quot;].class == Fixnum</span>
-  <span class="ruby-comment">#  job_data[&quot;@can_run_on&quot;] = [job_data[&quot;@can_run_on&quot;]]</span>
-  <span class="ruby-comment">#end</span>
+  return result
+end</pre>
+          </div>
+          
+        </div>
 
-  puts &quot;job data = #{job_data.inspect}&quot;
+        
 
-  do_preprocessing(job_data) 
+        
+      </div>
 
-  ssh_args = &quot;-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no &quot;
+    
+      <div id="neptune-method" class="method-detail ">
+        <a name="method-i-neptune"></a>
 
-  if type == <span class="ruby-string">&quot;input&quot;</span>
-    <span class="ruby-comment"># copy file to remote</span>
-    <span class="ruby-comment"># set location</span>
-    local_file = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@local&quot;</span>])
-    if !<span class="ruby-constant">File</span>.exists?(local_file)
-      msg = &quot;the file you specified to copy, #{local_file}, doesn't exist.&quot; + 
-        <span class="ruby-string">&quot; Please specify a file that exists and try again.&quot;</span>
-      abort(msg)
-    end
+        
+        <div class="method-heading">
+          <span class="method-name">neptune</span><span
+            class="method-args">(params)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
 
-    remote = &quot;/tmp/neptune-input-#{rand(100000)}&quot;
-    scp_cmd = &quot;scp #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}&quot;
-    puts scp_cmd
-    `#{scp_cmd}`
-
-    job_data[<span class="ruby-string">&quot;@local&quot;</span>] = remote
-    puts &quot;job data = #{job_data.inspect}&quot;
-    return controller.put_input(job_data)
-  elsif type == <span class="ruby-string">&quot;output&quot;</span>
-    return controller.get_output(job_data)
-  elsif type == <span class="ruby-string">&quot;get-acl&quot;</span>
-    job_data[<span class="ruby-string">&quot;@type&quot;</span>] = <span class="ruby-string">&quot;acl&quot;</span>
-    return controller.get_acl(job_data)
-  elsif type == <span class="ruby-string">&quot;set-acl&quot;</span>
-    job_data[<span class="ruby-string">&quot;@type&quot;</span>] = <span class="ruby-string">&quot;acl&quot;</span>
-    return controller.set_acl(job_data)
-  elsif type == <span class="ruby-string">&quot;compile&quot;</span>
-    compiled_location = controller.compile_code(job_data)
-
-    copy_to = job_data[<span class="ruby-string">&quot;@copy_to&quot;</span>]
-
-    loop {
-      ssh_command = &quot;ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2&gt;&amp;1&quot;
-      <span class="ruby-comment">#puts ssh_command</span>
-      result = `#{ssh_command}`
-      <span class="ruby-comment">#puts &quot;result was [#{result}]&quot;</span>
-      if result =~ <span class="ruby-regexp">/No such file or directory/</span>
-        puts <span class="ruby-string">&quot;Still waiting for code to be compiled...&quot;</span>
-      else
-        puts &quot;compilation complete! Copying compiled code to #{copy_to}&quot;
-        break
-      end
-      sleep(5)
-    }
+        <div class="method-description">
+          
+          <p>This method is the heart of Neptune - here, we take blocks of code that the
+user has written and convert them into HPC job requests. At a high level,
+the user can request to run a job, retrieve a job’s output, or modify the
+access policy (ACL) for the output of a job. By default, job data is
+private, but a Neptune job can be used to set it to public later (and
+vice-versa).</p>
+          
 
-    rm_local = &quot;rm -rf #{copy_to}&quot;
-    <span class="ruby-comment">#puts rm_local</span>
-    `#{rm_local}`
+          
+          <div class="method-source-code"
+            id="neptune-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 368</span>
+def neptune(params)
+  puts <span class="ruby-string">&quot;Received a request to run a job.&quot;</span>
+  puts params[:type]
 
-    scp_command = &quot;scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2&gt;&amp;1&quot;
-    puts scp_command
-    `#{scp_command}`
+  job_data = get_job_data(params)
+  validate_storage_params(job_data)
+  puts &quot;job data = #{job_data.inspect}&quot;
+  do_preprocessing(job_data) 
+  keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>]
 
-    out = <span class="ruby-constant">File</span>.open(&quot;#{copy_to}/compile_out&quot;) { |f| f.read.chomp! }
-    err = <span class="ruby-constant">File</span>.open(&quot;#{copy_to}/compile_err&quot;) { |f| f.read.chomp! }
-    return {:out =&gt; out, :err =&gt; err }
-  else
-    result = controller.start_neptune_job(job_data)
-    if result =~ <span class="ruby-regexp">/job is now running\Z/</span>
-      return {:result =&gt; :success, :msg =&gt; result}
-    else
-      return {:result =&gt; :failure, :msg =&gt; result}
-    end
-  end
+  shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
+  secret = <span class="ruby-constant">CommonFunctions</span>.get_secret_key(keyname)
+  ssh_key = <span class="ruby-constant">File</span>.expand_path(&quot;~/.appscale/#{keyname}.key&quot;)
+  ssh_args = &quot;-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no &quot;
+
+  return run_job(job_data, ssh_args, shadow_ip, secret)
 end</pre>
           </div>
           
@@ -433,7 +557,7 @@ end</pre>
         
         <div class="method-heading">
           <span class="method-name">preprocess_compile</span><span
-            class="method-args">(job_data)</span>
+            class="method-args">(job_data, shell=Kernel.method(:`))</span>
           <span class="method-click-advice">click to toggle source</span>
         </div>
         
@@ -449,10 +573,10 @@ copy over libraries as well.</p>
           <div class="method-source-code"
             id="preprocess_compile-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 60</span>
-def preprocess_compile(job_data)
+<span class="ruby-comment"># File lib/neptune.rb, line 65</span>
+def preprocess_compile(job_data, shell=<span class="ruby-constant">Kernel</span>.method(:`))
   code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
-  unless <span class="ruby-constant">File</span>.exists?(code)
+  if !<span class="ruby-constant">File</span>.exists?(code)
     abort(&quot;The source file #{code} does not exist.&quot;)
   end
 
@@ -463,8 +587,8 @@ def preprocess_compile(job_data)
 
   ssh_args = &quot;-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}&quot;
   remove_dir = &quot;ssh #{ssh_args} 'rm -rf #{dest}' 2&gt;&amp;1&quot;
-  <span class="ruby-comment">#puts remove_dir</span>
-  `#{remove_dir}`
+  puts remove_dir
+  shell.call(remove_dir)
 
   <span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(code, dest, keyname, is_dir=true)
 
@@ -486,7 +610,7 @@ end</pre>
         
         <div class="method-heading">
           <span class="method-name">preprocess_erlang</span><span
-            class="method-args">(job_data)</span>
+            class="method-args">(job_data, file=File, common_functions=CommonFunctions)</span>
           <span class="method-click-advice">click to toggle source</span>
         </div>
         
@@ -500,18 +624,21 @@ end</pre>
           <div class="method-source-code"
             id="preprocess_erlang-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 81</span>
-def preprocess_erlang(job_data)
-  source_code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
-  unless <span class="ruby-constant">File</span>.exists?(source_code)
-    file_not_found = &quot;The specified code, #{job_data['@code']},&quot; +
-      <span class="ruby-string">&quot; didn't exist. Please specify one that exists and try again&quot;</span>
-    abort(file_not_found)
+<span class="ruby-comment"># File lib/neptune.rb, line 86</span>
+def preprocess_erlang(job_data, file=<span class="ruby-constant">File</span>, common_functions=<span class="ruby-constant">CommonFunctions</span>)
+  if !job_data[<span class="ruby-string">&quot;@code&quot;</span>]
+    abort(<span class="ruby-string">&quot;When running Erlang jobs, :code must be specified.&quot;</span>)
+  end
+
+  source_code = file.expand_path(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
+  if !file.exists?(source_code)
+    abort(&quot;The specified code, #{job_data['@code']},&quot; +
+      <span class="ruby-string">&quot; didn't exist. Please specify one that exists and try again&quot;</span>)
   end
   dest_code = <span class="ruby-string">&quot;/tmp/&quot;</span>
 
   keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>]
-  <span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(source_code, dest_code, keyname)
+  common_functions.scp_to_shadow(source_code, dest_code, keyname)
 end</pre>
           </div>
           
@@ -523,12 +650,12 @@ end</pre>
       </div>
 
     
-      <div id="preprocess_mapreduce-method" class="method-detail ">
-        <a name="method-i-preprocess_mapreduce"></a>
+      <div id="preprocess_mpi-method" class="method-detail ">
+        <a name="method-i-preprocess_mpi"></a>
 
         
         <div class="method-heading">
-          <span class="method-name">preprocess_mapreduce</span><span
+          <span class="method-name">preprocess_mpi</span><span
             class="method-args">(job_data)</span>
           <span class="method-click-advice">click to toggle source</span>
         </div>
@@ -536,38 +663,89 @@ end</pre>
 
         <div class="method-description">
           
-          <p>This preprocessing method handles copying data for regular Hadoop MapReduce
-and Hadoop MapReduce Streaming. For the former case, we copy over just the
-JAR the user has given us, and in the latter case, we copy over the Map and
-Reduce files that have been specified. In either case, if the user has
-specified to us to copy over an input file, we do that as well: AppScale
-will copy it into HDFS for us.</p>
+          <p>This preprocessing method verifies that the user specified the number of
+nodes to use. If they also specified the number of processes to use, we
+also verify that this value is at least as many as the number of nodes
+(that is, nodes can’t be underprovisioned in MPI).</p>
           
 
           
           <div class="method-source-code"
-            id="preprocess_mapreduce-source">
+            id="preprocess_mpi-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 101</span>
-def preprocess_mapreduce(job_data)
-  return
-  <span class="ruby-comment">#items_to_copy = [&quot;@map&quot;, &quot;@reduce&quot;] if job_data[&quot;@map&quot;] and job_data[&quot;@reduce&quot;]</span>
-  items_to_copy = [<span class="ruby-string">&quot;@mapreducejar&quot;</span>] if job_data[<span class="ruby-string">&quot;@mapreducejar&quot;</span>]
-  <span class="ruby-comment">#items_to_copy &lt;&lt; &quot;@input&quot; if job_data[&quot;@copy_input&quot;]</span>
-  items_to_copy.each { |item|
-    source = <span class="ruby-constant">File</span>.expand_path(job_data[item])
-    unless <span class="ruby-constant">File</span>.exists?(source)
-      abort(&quot;The #{item} file #{source} does not exist.&quot;)
+<span class="ruby-comment"># File lib/neptune.rb, line 106</span>
+def preprocess_mpi(job_data)
+  if !job_data[<span class="ruby-string">&quot;@nodes_to_use&quot;</span>]
+    abort(<span class="ruby-string">&quot;When running MPI jobs, :nodes_to_use must be specified.&quot;</span>)
+  end
+
+  if !job_data[<span class="ruby-string">&quot;@procs_to_use&quot;</span>]
+    abort(<span class="ruby-string">&quot;When running MPI jobs, :procs_to_use must be specified.&quot;</span>)
+  end
+
+  if job_data[<span class="ruby-string">&quot;@procs_to_use&quot;</span>]
+    p = job_data[<span class="ruby-string">&quot;@procs_to_use&quot;</span>]
+    n = job_data[<span class="ruby-string">&quot;@nodes_to_use&quot;</span>]
+    if p &lt; n
+      abort(<span class="ruby-string">&quot;When specifying both :procs_to_use and :nodes_to_use&quot;</span> +
+        <span class="ruby-string">&quot;, :procs_to_use must be at least as large as :nodes_to_use. Please &quot;</span> +
+        &quot;change this and try again. You specified :procs_to_use = #{p} and&quot; +
+        &quot;:nodes_to_use = #{n}.&quot;)
     end
+  end
+
+  return job_data
+end</pre>
+          </div>
+          
+        </div>
+
+        
 
-    suffix = source.split(<span class="ruby-string">'/'</span>)[-1]
-    dest = &quot;/tmp/#{suffix}&quot;
+        
+      </div>
 
-    keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>]
-    <span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(source, dest, keyname)
+    
+      <div id="preprocess_ssa-method" class="method-detail ">
+        <a name="method-i-preprocess_ssa"></a>
 
-    job_data[item] = dest
-  }
+        
+        <div class="method-heading">
+          <span class="method-name">preprocess_ssa</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>This preprocessing method verifies that the user specified the number of
+trajectories to run, via either :trajectories or :simulations. Both should
+not be specified - only one or the other, and regardless of which they
+specify, convert it to be :trajectories.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="preprocess_ssa-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 133</span>
+def preprocess_ssa(job_data)
+  if job_data[<span class="ruby-string">&quot;@simulations&quot;</span>] and job_data[<span class="ruby-string">&quot;@trajectories&quot;</span>]
+    abort(<span class="ruby-string">&quot;Both :simulations and :trajectories cannot be specified - use one&quot;</span> +
+      <span class="ruby-string">&quot; or the other.&quot;</span>)
+  end
+
+  if job_data[<span class="ruby-string">&quot;@simulations&quot;</span>]
+    job_data[<span class="ruby-string">&quot;@trajectories&quot;</span>] = job_data[<span class="ruby-string">&quot;@simulations&quot;</span>]
+    job_data.delete(<span class="ruby-string">&quot;@simulations&quot;</span>)
+  end
+
+  if !job_data[<span class="ruby-string">&quot;@trajectories&quot;</span>]
+    abort(<span class="ruby-string">&quot;:trajectories needs to be specified when running ssa jobs&quot;</span>)
+  end
+
+  return job_data
 end</pre>
           </div>
           
@@ -579,12 +757,73 @@ end</pre>
       </div>
 
     
-      <div id="preprocess_mpi-method" class="method-detail ">
-        <a name="method-i-preprocess_mpi"></a>
+      <div id="run_job-method" class="method-detail ">
+        <a name="method-i-run_job"></a>
 
         
         <div class="method-heading">
-          <span class="method-name">preprocess_mpi</span><span
+          <span class="method-name">run_job</span><span
+            class="method-args">(job_data, ssh_args, shadow_ip, secret, controller=AppControllerClient, file=File)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>This method actually runs the Neptune job, given information about the job
+as well as information about the node to send the request to.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="run_job-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 330</span>
+def run_job(job_data, ssh_args, shadow_ip, secret,
+  controller=<span class="ruby-constant">AppControllerClient</span>, file=<span class="ruby-constant">File</span>)
+  controller = controller.new(shadow_ip, secret)
+
+  <span class="ruby-comment"># TODO - right now the job is assumed to succeed in many cases</span>
+  <span class="ruby-comment"># need to investigate the various failure scenarios</span>
+  result = { :result =&gt; :success }
+
+  case job_data[<span class="ruby-string">&quot;@type&quot;</span>]
+  when <span class="ruby-string">&quot;input&quot;</span>
+    result = get_input(job_data, ssh_args, shadow_ip, controller, file)
+  when <span class="ruby-string">&quot;output&quot;</span>
+    result[:output] = controller.get_output(job_data)
+  when <span class="ruby-string">&quot;get-acl&quot;</span>
+    job_data[<span class="ruby-string">&quot;@type&quot;</span>] = <span class="ruby-string">&quot;acl&quot;</span>
+    result[:acl] = controller.get_acl(job_data)
+  when <span class="ruby-string">&quot;set-acl&quot;</span>
+    job_data[<span class="ruby-string">&quot;@type&quot;</span>] = <span class="ruby-string">&quot;acl&quot;</span>
+    result[:acl] = controller.set_acl(job_data)
+  when <span class="ruby-string">&quot;compile&quot;</span>
+    result = compile_code(job_data, ssh_args, shadow_ip)
+  else
+    msg = controller.start_neptune_job(job_data)
+    result[:msg] = msg
+    result[:result] = :failure if result[:msg] !~ <span class="ruby-regexp">/job is now running\Z/</span>
+  end
+
+  return result
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="validate_storage_params-method" class="method-detail ">
+        <a name="method-i-validate_storage_params"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">validate_storage_params</span><span
             class="method-args">(job_data)</span>
           <span class="method-click-advice">click to toggle source</span>
         </div>
@@ -592,48 +831,99 @@ end</pre>
 
         <div class="method-description">
           
-          <p>This preprocessing method copies over the user’s MPI code to the master
-node in AppScale - this node will then copy it to whoever will run the MPI
-job.</p>
+          
           
 
           
           <div class="method-source-code"
-            id="preprocess_mpi-source">
+            id="validate_storage_params-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 125</span>
-def preprocess_mpi(job_data)
-  if job_data[<span class="ruby-string">&quot;@procs_to_use&quot;</span>]
-    p = job_data[<span class="ruby-string">&quot;@procs_to_use&quot;</span>]
-    n = job_data[<span class="ruby-string">&quot;@nodes_to_use&quot;</span>]
-    if p &lt; n
-      not_enough_procs = <span class="ruby-string">&quot;When specifying both :procs_to_use and :nodes_to_use&quot;</span> +
-        <span class="ruby-string">&quot;, :procs_to_use must be at least as large as :nodes_to_use. Please &quot;</span> +
-        &quot;change this and try again. You specified :procs_to_use = #{p} and&quot; +
-        &quot;:nodes_to_use = #{n}.&quot;
-      abort(not_enough_procs)
-    end
+<span class="ruby-comment"># File lib/neptune.rb, line 186</span>
+def validate_storage_params(job_data)
+  if !job_data[<span class="ruby-string">&quot;@storage&quot;</span>]
+    job_data[<span class="ruby-string">&quot;@storage&quot;</span>] = <span class="ruby-string">&quot;appdb&quot;</span>
   end
 
-  source_code = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
-  unless <span class="ruby-constant">File</span>.exists?(source_code)
-    file_not_found = &quot;The specified code, #{source_code},&quot; +
-      <span class="ruby-string">&quot; didn't exist. Please specify one that exists and try again&quot;</span>
-    abort(file_not_found)
+  storage = job_data[<span class="ruby-string">&quot;@storage&quot;</span>]
+  if !<span class="ruby-constant">ALLOWED_STORAGE_TYPES</span>.include?(storage)
+    abort(&quot;Supported storage types are #{ALLOWED_STORAGE_TYPES.join(', ')}&quot; +
+      &quot; - we do not support #{storage}.&quot;)
   end
 
-  unless <span class="ruby-constant">File</span>.file?(source_code)
-    should_be_file = &quot;The specified code, #{source_code}, was not a file - &quot; +
-      <span class="ruby-string">&quot; it was a directory or symbolic link. Please specify a file and try again.&quot;</span>
-    abort(should_be_file)
+  <span class="ruby-comment"># Our implementation for storing / retrieving via Google Storage</span>
+  <span class="ruby-comment"># and Walrus uses</span>
+  <span class="ruby-comment"># the same library as we do for S3 - so just tell it that it's S3</span>
+  if storage == <span class="ruby-string">&quot;gstorage&quot;</span> or storage == <span class="ruby-string">&quot;walrus&quot;</span>
+    storage = <span class="ruby-string">&quot;s3&quot;</span>
+    job_data[<span class="ruby-string">&quot;@storage&quot;</span>] = <span class="ruby-string">&quot;s3&quot;</span>
+  end
+
+  if storage == <span class="ruby-string">&quot;s3&quot;</span>
+    [<span class="ruby-string">&quot;EC2_ACCESS_KEY&quot;</span>, <span class="ruby-string">&quot;EC2_SECRET_KEY&quot;</span>, <span class="ruby-string">&quot;S3_URL&quot;</span>].each { |item|
+      if job_data[&quot;@#{item}&quot;]
+        puts &quot;Using specified #{item}&quot;
+      else
+        if <span class="ruby-constant">ENV</span>[item]
+          puts &quot;Using #{item} from environment&quot;
+          job_data[&quot;@#{item}&quot;] = <span class="ruby-constant">ENV</span>[item]
+        else
+          abort(&quot;When storing data to S3, #{item} must be specified or be in &quot; + 
+            <span class="ruby-string">&quot;your environment. Please do so and try again.&quot;</span>)
+        end
+      end
+    }
   end
 
-  dest_code = <span class="ruby-string">&quot;/tmp/thempicode&quot;</span>
+  return job_data
+end</pre>
+          </div>
+          
+        </div>
+
+        
 
-  keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>]
-  puts <span class="ruby-string">&quot;Copying over code...&quot;</span>
-  <span class="ruby-constant">CommonFunctions</span>.scp_to_shadow(source_code, dest_code, keyname)
-  puts <span class="ruby-string">&quot;Done copying code!&quot;</span>
+        
+      </div>
+
+    
+      <div id="wait_for_compilation_to_finish-method" class="method-detail ">
+        <a name="method-i-wait_for_compilation_to_finish"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">wait_for_compilation_to_finish</span><span
+            class="method-args">(ssh_args, shadow_ip, compiled_location, shell=Kernel.method(:`))</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>This method waits for AppScale to finish compiling the user’s code,
+indicated by AppScale copying the finished code to a pre-determined
+location.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="wait_for_compilation_to_finish-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 260</span>
+def wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location,
+  shell=<span class="ruby-constant">Kernel</span>.method(:`))
+  loop {
+    ssh_command = &quot;ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2&gt;&amp;1&quot;
+    puts ssh_command
+    ssh_result = shell.call(ssh_command)
+    puts &quot;result was [#{ssh_result}]&quot;
+    if ssh_result =~ <span class="ruby-regexp">/No such file or directory/</span>
+      puts <span class="ruby-string">&quot;Still waiting for code to be compiled...&quot;</span>
+    else
+      puts &quot;compilation complete! Copying compiled code to #{copy_to}&quot;
+      return
+    end
+    sleep(5)
+  }
 end</pre>
           </div>