neptune 0.1.4 → 0.2.0

data/README

@@ -67,6 +67,10 @@ http://www.neptune-lang.org/2011/6/Neptune-Picks-up-Best-Paper-at-ScienceCloud-2
 
 Version History:
 
+February 11, 2012 - 0.2.0 released, adding support for Babel jobs (code that
+runs over multiple clouds, using their queues automatically) and the babel()
+method, to automatically upload code, run it, and retrieve the output.
+
 December 6, 2011 - 0.1.4 released, allowing MPI jobs (and by extension,
 UPC/X10/KDT jobs) to specify command-line arguments via :argv that should be
 passed to the executable.

data/bin/neptune

@@ -3,8 +3,15 @@
 $VERBOSE = nil # to surpress excessive SSL cert warnings
 
 $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'babel'
 require 'neptune'
 
+
+# If the user gives us only one argument, then we assume it's the location of a
+# Neptune script and execute it accordingly. In all other conditions, we assume
+# the user is trying to generate a Neptune script on-the-fly, and (if an even
+# number of params are given) we generate a hash from the params and exec it
+# as a Neptune job.
 if ARGV.length == 1
   file = ARGV[0]
   unless File.exists?(file)

data/doc/AppControllerClient.html

@@ -73,10 +73,14 @@
           
           <li><a href="#method-i-compile_code">#compile_code</a></li>
           
+          <li><a href="#method-i-does_file_exist-3F">#does_file_exist?</a></li>
+          
           <li><a href="#method-i-get_acl">#get_acl</a></li>
           
           <li><a href="#method-i-get_output">#get_output</a></li>
           
+          <li><a href="#method-i-get_supported_babel_engines">#get_supported_babel_engines</a></li>
+          
           <li><a href="#method-i-make_call">#make_call</a></li>
           
           <li><a href="#method-i-put_input">#put_input</a></li>
@@ -114,9 +118,17 @@
         
           <li><a href="./AppControllerClient.html">AppControllerClient</a></li>
         
+          <li><a href="./AppControllerException.html">AppControllerException</a></li>
+        
+          <li><a href="./BabelHelper.html">BabelHelper</a></li>
+        
+          <li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
+        
           <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
         
-          <li><a href="./Kernel.html">Kernel</a></li>
+          <li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
+        
+          <li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
         
           <li><a href="./Object.html">Object</a></li>
         
@@ -245,6 +257,8 @@ def initialize(ip, secret)
   <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_get_acl&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
   <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_set_acl&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
   <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_compile_code&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
+  <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_get_supported_babel_engines&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
+  <span class="ruby-ivar">@conn</span>.add_method(<span class="ruby-string">&quot;neptune_does_file_exist&quot;</span>, <span class="ruby-string">&quot;file&quot;</span>, <span class="ruby-string">&quot;job_data&quot;</span>, <span class="ruby-string">&quot;secret&quot;</span>)
 end</pre>
           </div>
           
@@ -282,13 +296,51 @@ end</pre>
           <div class="method-source-code"
             id="compile_code-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 152</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 154</span>
 def compile_code(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
     result = conn.neptune_compile_code(job_data, <span class="ruby-ivar">@secret</span>)
   }  
-  abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  return result
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="does_file_exist-3F-method" class="method-detail ">
+        <a name="method-i-does_file_exist-3F"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">does_file_exist?</span><span
+            class="method-args">(file, job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          
+          
+
+          
+          <div class="method-source-code"
+            id="does_file_exist-3F-source">
+<pre>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 171</span>
+def does_file_exist?(file, job_data)
+  result = false
+  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
+    result = conn.neptune_does_file_exist(file, job_data, <span class="ruby-ivar">@secret</span>)
+  }
   return result
 end</pre>
           </div>
@@ -324,13 +376,13 @@ exceptions mirror that of start_neptune_job.</p>
           <div class="method-source-code"
             id="get_acl-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 129</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 131</span>
 def get_acl(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
     result = conn.neptune_get_acl(job_data, <span class="ruby-ivar">@secret</span>)
   }  
-  abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
   return result
 end</pre>
           </div>
@@ -365,21 +417,59 @@ string to the caller, but as this may be inefficient for non-trivial output
 jobs, the next version of Neptune will add an additional call to directly
 copy the output to a file on the local filesystem. See <a
 href="AppControllerClient.html#method-i-start_neptune_job">start_neptune_job</a>
-for conditions by which this method can abort as well as the input format
-used for job_data.</p>
+for conditions by which this method can raise AppControllerException.new as
+well as the input format used for job_data.</p>
           
 
           
           <div class="method-source-code"
             id="get_output-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 115</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 117</span>
 def get_output(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
     result = conn.neptune_get_output(job_data, <span class="ruby-ivar">@secret</span>)
   }  
-  abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  return result
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="get_supported_babel_engines-method" class="method-detail ">
+        <a name="method-i-get_supported_babel_engines"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">get_supported_babel_engines</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          
+          
+
+          
+          <div class="method-source-code"
+            id="get_supported_babel_engines-source">
+<pre>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 163</span>
+def get_supported_babel_engines(job_data)
+  result = []
+  make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
+    result = conn.neptune_get_supported_babel_engines(job_data, <span class="ruby-ivar">@secret</span>)
+  }
   return result
 end</pre>
           </div>
@@ -423,7 +513,7 @@ The result of the block is returned to the caller.</p>
           <div class="method-source-code"
             id="make_call-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 52</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 54</span>
 def make_call(time, retry_on_except)
   begin
     <span class="ruby-constant">Timeout</span>::timeout(time) {
@@ -433,7 +523,7 @@ def make_call(time, retry_on_except)
     if retry_on_except
       retry
     else
-      abort(<span class="ruby-string">&quot;Connection was refused. Is the AppController running?&quot;</span>)
+      raise <span class="ruby-constant">AppControllerException</span>.new(<span class="ruby-string">&quot;Connection was refused. Is the AppController running?&quot;</span>)
     end
   rescue <span class="ruby-constant">OpenSSL</span>::<span class="ruby-constant">SSL</span>::<span class="ruby-constant">SSLError</span>, <span class="ruby-constant">NotImplementedError</span>, <span class="ruby-constant">Timeout</span>::<span class="ruby-constant">Error</span>
     retry
@@ -441,7 +531,7 @@ def make_call(time, retry_on_except)
     if retry_on_except
       retry
     else
-      abort(&quot;We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.&quot;)
+      raise <span class="ruby-constant">AppControllerException</span>.new(&quot;We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.&quot;)
     end
   end
 end</pre>
@@ -479,13 +569,13 @@ parameter.</p>
           <div class="method-source-code"
             id="put_input-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 96</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 98</span>
 def put_input(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) {
     result = conn.neptune_put_input(job_data, <span class="ruby-ivar">@secret</span>)
   }  
-  abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
   return result
 end</pre>
           </div>
@@ -523,13 +613,13 @@ that of start_neptune_job.</p>
           <div class="method-source-code"
             id="set_acl-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 143</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 145</span>
 def set_acl(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
     result = conn.neptune_set_acl(job_data, <span class="ruby-ivar">@secret</span>)
   }  
-  abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
   return result
 end</pre>
           </div>
@@ -559,23 +649,24 @@ end</pre>
 MapReduce), or a scaling job (e.g., for AppScale itself). This method
 should not be used for retrieving the output of a job or getting / setting
 output ACLs, but just for starting new HPC / scaling jobs. This method
-takes a hash containing the parameters of the job to run, and can abort if
-the AppController it calls returns an error (e.g., if a bad secret is used
-or the machine isn’t running). Otherwise, the return value of this method
-is the result returned from the AppController.</p>
+takes a hash containing the parameters of the job to run, and can raise
+AppControllerException.new if the AppController it calls returns an error
+(e.g., if a bad secret is used or the machine isn’t running). Otherwise,
+the return value of this method is the result returned from the
+AppController.</p>
           
 
           
           <div class="method-source-code"
             id="start_neptune_job-source">
 <pre>
-<span class="ruby-comment"># File lib/app_controller_client.rb, line 82</span>
+<span class="ruby-comment"># File lib/app_controller_client.rb, line 84</span>
 def start_neptune_job(job_data)
   result = <span class="ruby-string">&quot;&quot;</span>
   make_call(<span class="ruby-constant">NO_TIMEOUT</span>, false) { 
     result = conn.neptune_start_job(job_data, <span class="ruby-ivar">@secret</span>)
   }  
-  abort(result) if result =~ <span class="ruby-regexp">/Error:/</span>
+  raise <span class="ruby-constant">AppControllerException</span>.new(result) if result =~ <span class="ruby-regexp">/Error:/</span>
   return result
 end</pre>
           </div>

data/doc/{Kernel.html → AppControllerException.html}

@@ -5,7 +5,7 @@
 <head>
   <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
 
-  <title>Module: Kernel</title>
+  <title>Class: AppControllerException</title>
 
   <link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
 
@@ -19,7 +19,7 @@
     charset="utf-8"></script>
 
 </head>
-<body class="module">
+<body class="class">
 
   <div id="metadata">
     <div id="home-metadata">
@@ -38,8 +38,8 @@
         <div class="section-body">
           <ul>
           
-            <li><a href="./lib/common_functions_rb.html?TB_iframe=true&amp;height=550&amp;width=785"
-              class="thickbox" title="lib/common_functions.rb">lib/common_functions.rb</a></li>
+            <li><a href="./lib/custom_exceptions_rb.html?TB_iframe=true&amp;height=550&amp;width=785"
+              class="thickbox" title="lib/custom_exceptions.rb">lib/custom_exceptions.rb</a></li>
           
           </ul>
         </div>
@@ -52,21 +52,19 @@
 
       <!-- Parent Class -->
       
+      <div id="parent-class-section" class="section">
+        <h3 class="section-header">Parent</h3>
+        
+        <p class="link">Exception</p>
+        
+      </div>
+      
 
       <!-- Namespace Contents -->
       
 
       <!-- Method Quickref -->
       
-      <div id="method-list-section" class="section">
-        <h3 class="section-header">Methods</h3>
-        <ul class="link-list">
-          
-          <li><a href="#method-i-shell">#shell</a></li>
-          
-        </ul>
-      </div>
-      
 
       <!-- Included Modules -->
       
@@ -93,9 +91,17 @@
         
           <li><a href="./AppControllerClient.html">AppControllerClient</a></li>
         
+          <li><a href="./AppControllerException.html">AppControllerException</a></li>
+        
+          <li><a href="./BabelHelper.html">BabelHelper</a></li>
+        
+          <li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
+        
           <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
         
-          <li><a href="./Kernel.html">Kernel</a></li>
+          <li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
+        
+          <li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
         
           <li><a href="./Object.html">Object</a></li>
         
@@ -108,10 +114,12 @@
   </div>
 
   <div id="documentation">
-    <h1 class="module">Kernel</h1>
+    <h1 class="class">AppControllerException</h1>
 
     <div id="description">
       
+<p>Programmer: Chris Bunch</p>
+
     </div>
 
     <!-- Constants -->
@@ -122,46 +130,6 @@
 
     <!-- Methods -->
     
-    <div id="public-instance-method-details" class="method-section section">
-      <h3 class="section-header">Public Instance Methods</h3>
-
-    
-      <div id="shell-method" class="method-detail ">
-        <a name="method-i-shell"></a>
-
-        
-        <div class="method-heading">
-          <span class="method-name">shell</span><span
-            class="method-args">(command)</span>
-          <span class="method-click-advice">click to toggle source</span>
-        </div>
-        
-
-        <div class="method-description">
-          
-          
-          
-
-          
-          <div class="method-source-code"
-            id="shell-source">
-<pre>
-<span class="ruby-comment"># File lib/common_functions.rb, line 13</span>
-def shell(command)
-  return `#{command}`
-end</pre>
-          </div>
-          
-        </div>
-
-        
-
-        
-      </div>
-
-    
-    </div>
-  
 
   </div>
 

data/doc/BabelHelper.html

@@ -0,0 +1,707 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+
+  <title>Module: BabelHelper</title>
+
+  <link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
+
+  <script src="./js/jquery.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="./js/thickbox-compressed.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="./js/quicksearch.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="./js/darkfish.js" type="text/javascript"
+    charset="utf-8"></script>
+
+</head>
+<body class="module">
+
+  <div id="metadata">
+    <div id="home-metadata">
+      <div id="home-section" class="section">
+        <h3 class="section-header">
+          <a href="./index.html">Home</a>
+          <a href="./index.html#classes">Classes</a>
+          <a href="./index.html#methods">Methods</a>
+        </h3>
+      </div>
+    </div>
+
+    <div id="file-metadata">
+      <div id="file-list-section" class="section">
+        <h3 class="section-header">In Files</h3>
+        <div class="section-body">
+          <ul>
+          
+            <li><a href="./lib/babel_rb.html?TB_iframe=true&amp;height=550&amp;width=785"
+              class="thickbox" title="lib/babel.rb">lib/babel.rb</a></li>
+          
+          </ul>
+        </div>
+      </div>
+
+      
+    </div>
+
+    <div id="class-metadata">
+
+      <!-- Parent Class -->
+      
+
+      <!-- Namespace Contents -->
+      
+
+      <!-- Method Quickref -->
+      
+      <div id="method-list-section" class="section">
+        <h3 class="section-header">Methods</h3>
+        <ul class="link-list">
+          
+          <li><a href="#method-c-convert_from_neptune_params">::convert_from_neptune_params</a></li>
+          
+          <li><a href="#method-c-convert_to_neptune_params">::convert_to_neptune_params</a></li>
+          
+          <li><a href="#method-c-ensure_output_does_not_exist">::ensure_output_does_not_exist</a></li>
+          
+          <li><a href="#method-c-generate_output_location">::generate_output_location</a></li>
+          
+          <li><a href="#method-c-get_appcontroller">::get_appcontroller</a></li>
+          
+          <li><a href="#method-c-get_bucket_for_local_data">::get_bucket_for_local_data</a></li>
+          
+          <li><a href="#method-c-put_code">::put_code</a></li>
+          
+          <li><a href="#method-c-put_file">::put_file</a></li>
+          
+          <li><a href="#method-c-put_inputs">::put_inputs</a></li>
+          
+          <li><a href="#method-c-run_job">::run_job</a></li>
+          
+          <li><a href="#method-c-validate_inputs">::validate_inputs</a></li>
+          
+          <li><a href="#method-c-wait_and_get_output">::wait_and_get_output</a></li>
+          
+        </ul>
+      </div>
+      
+
+      <!-- Included Modules -->
+      
+    </div>
+
+    <div id="project-metadata">
+      
+      
+
+      <div id="classindex-section" class="section project-section">
+        <h3 class="section-header">Class/Module Index
+          <span class="search-toggle"><img src="./images/find.png"
+            height="16" width="16" alt="[+]"
+            title="show/hide quicksearch" /></span></h3>
+        <form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
+        <fieldset>
+          <legend>Quicksearch</legend>
+          <input type="text" name="quicksearch" value=""
+            class="quicksearch-field" />
+        </fieldset>
+        </form>
+
+        <ul class="link-list">
+        
+          <li><a href="./AppControllerClient.html">AppControllerClient</a></li>
+        
+          <li><a href="./AppControllerException.html">AppControllerException</a></li>
+        
+          <li><a href="./BabelHelper.html">BabelHelper</a></li>
+        
+          <li><a href="./BadConfigurationException.html">BadConfigurationException</a></li>
+        
+          <li><a href="./CommonFunctions.html">CommonFunctions</a></li>
+        
+          <li><a href="./FileNotFoundException.html">FileNotFoundException</a></li>
+        
+          <li><a href="./NeptuneHelper.html">NeptuneHelper</a></li>
+        
+          <li><a href="./Object.html">Object</a></li>
+        
+        </ul>
+        <div id="no-class-search-results" style="display: none;">No matching classes.</div>
+      </div>
+
+      
+    </div>
+  </div>
+
+  <div id="documentation">
+    <h1 class="module">BabelHelper</h1>
+
+    <div id="description">
+      
+<p>This module provides convenience functions for babel().</p>
+
+    </div>
+
+    <!-- Constants -->
+    
+
+    <!-- Attributes -->
+    
+
+    <!-- Methods -->
+    
+    <div id="public-class-method-details" class="method-section section">
+      <h3 class="section-header">Public Class Methods</h3>
+
+    
+      <div id="convert_from_neptune_params-method" class="method-detail ">
+        <a name="method-c-convert_from_neptune_params"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">convert_from_neptune_params</span><span
+            class="method-args">(params)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>Neptune internally uses job_data with keys of the form @name, but since the
+user has given them to us in the form :name, we convert it here. TODO(cgb):
+It looks like this conversion to/from may be unnecessary since neptune()
+just re-converts it - how can we remove it?</p>
+          
+
+          
+          <div class="method-source-code"
+            id="convert_from_neptune_params-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 198</span>
+def self.convert_from_neptune_params(params)
+  job_data = {}
+  params.each { |k, v|
+    key = &quot;@#{k}&quot;
+    job_data[key] = v
+  }
+  return job_data
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="convert_to_neptune_params-method" class="method-detail ">
+        <a name="method-c-convert_to_neptune_params"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">convert_to_neptune_params</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>Neptune input jobs expect keys of the form :name, but since we’ve already
+converted them to the form @name, this function reverses that conversion.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="convert_to_neptune_params-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 209</span>
+def self.convert_to_neptune_params(job_data)
+  neptune_params = {}
+
+  job_data.each { |k, v|
+    key = k.delete(<span class="ruby-string">&quot;@&quot;</span>).to_sym
+    neptune_params[key] = v
+  }
+
+  return neptune_params
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="ensure_output_does_not_exist-method" class="method-detail ">
+        <a name="method-c-ensure_output_does_not_exist"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">ensure_output_does_not_exist</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>To avoid accidentally overwriting outputs from previous jobs, we first
+check to make sure an output file doesn’t exist before starting a new job
+with the given name.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="ensure_output_does_not_exist-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 134</span>
+def self.ensure_output_does_not_exist(job_data)
+  file = job_data[<span class="ruby-string">&quot;@output&quot;</span>]
+  controller = self.get_appcontroller(job_data)
+  puts job_data.inspect
+  <span class="ruby-constant">NeptuneHelper</span>.require_file_to_not_exist(file, job_data, controller)
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="generate_output_location-method" class="method-detail ">
+        <a name="method-c-generate_output_location"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">generate_output_location</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>If the user fails to give us an output location, this function will
+generate one for them, based on either the location of their code (for
+remotely specified code), or a babel parameter (for locally specified
+code).</p>
+          
+
+          
+          <div class="method-source-code"
+            id="generate_output_location-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 81</span>
+def self.generate_output_location(job_data)
+  if job_data[<span class="ruby-string">&quot;@is_remote&quot;</span>]
+    <span class="ruby-comment"># We already know the bucket name - the same one that the user</span>
+    <span class="ruby-comment"># has told us their code is located in.</span>
+    prefix = job_data[<span class="ruby-string">&quot;@code&quot;</span>].scan(<span class="ruby-regexp">/\/(.*?)\//</span>)[0].to_s
+  else
+    prefix = self.get_bucket_for_local_data(job_data)
+  end
+    
+  return &quot;/#{prefix}/babel/temp-#{CommonFunctions.get_random_alphanumeric()}&quot;
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="get_appcontroller-method" class="method-detail ">
+        <a name="method-c-get_appcontroller"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">get_appcontroller</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>Returns an <a href="AppControllerClient.html">AppControllerClient</a> for
+the given job data.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="get_appcontroller-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 142</span>
+def self.get_appcontroller(job_data)
+  keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>] || <span class="ruby-string">&quot;appscale&quot;</span>
+  shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)
+  secret = <span class="ruby-constant">CommonFunctions</span>.get_secret_key(keyname)
+  return <span class="ruby-constant">AppControllerClient</span>.new(shadow_ip, secret)
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="get_bucket_for_local_data-method" class="method-detail ">
+        <a name="method-c-get_bucket_for_local_data"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">get_bucket_for_local_data</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>Provides a common way for callers to get the name of the bucket that should
+be used for Neptune jobs where the code is stored locally.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="get_bucket_for_local_data-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 95</span>
+def self.get_bucket_for_local_data(job_data)
+  bucket_name = job_data[<span class="ruby-string">&quot;@bucket_name&quot;</span>] || <span class="ruby-constant">ENV</span>[<span class="ruby-string">'BABEL_BUCKET_NAME'</span>]
+
+  if bucket_name.nil?
+    raise <span class="ruby-constant">BadConfigurationException</span>.new(<span class="ruby-constant">NEEDS_BUCKET_INFO</span>)
+  end
+
+  <span class="ruby-comment"># If the bucket name starts with a slash, remove it</span>
+  if bucket_name[0].chr == <span class="ruby-string">&quot;/&quot;</span>
+    bucket_name = bucket_name[1, bucket_name.length]
+  end
+
+  return bucket_name
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="put_code-method" class="method-detail ">
+        <a name="method-c-put_code"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">put_code</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>Stores the user’s code (and the directory it’s in, and directories in
+the same directory as the user’s code, since there could be libraries
+used) in the remote datastore.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="put_code-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 152</span>
+def self.put_code(job_data)
+  code_dir = <span class="ruby-constant">File</span>.dirname(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
+  code = <span class="ruby-constant">File</span>.basename(job_data[<span class="ruby-string">&quot;@code&quot;</span>])
+  remote_code_dir = self.put_file(code_dir, job_data)
+  job_data[<span class="ruby-string">&quot;@code&quot;</span>] = remote_code_dir + <span class="ruby-string">&quot;/&quot;</span> + code
+  return job_data[<span class="ruby-string">&quot;@code&quot;</span>]
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="put_file-method" class="method-detail ">
+        <a name="method-c-put_file"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">put_file</span><span
+            class="method-args">(local_path, job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>If the user gives us local code or local inputs, this function will run a
+Neptune ‘input’ job to store the data remotely.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="put_file-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 181</span>
+def self.put_file(local_path, job_data)
+  input_data = self.convert_to_neptune_params(job_data)
+  input_data[:type] = <span class="ruby-string">&quot;input&quot;</span>
+  input_data[:local] = local_path
+
+  bucket_name = self.get_bucket_for_local_data(job_data)
+  input_data[:remote] = &quot;/#{bucket_name}/babel#{local_path}&quot;
+
+  <span class="ruby-constant">Kernel</span>.neptune(input_data)
+
+  return input_data[:remote]
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="put_inputs-method" class="method-detail ">
+        <a name="method-c-put_inputs"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">put_inputs</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>If any input files are specified, they are copied to the remote datastore
+via Neptune ‘input’ jobs. Inputs are assumed to be files on the local
+filesystem if they begin with a slash, and job_data gets updated with the
+remote location of these files.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="put_inputs-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 164</span>
+def self.put_inputs(job_data)
+  if job_data[<span class="ruby-string">&quot;@argv&quot;</span>].nil? or job_data[<span class="ruby-string">&quot;@argv&quot;</span>].empty?
+    return job_data
+  end
+
+  job_data[<span class="ruby-string">&quot;@argv&quot;</span>].each_index { |i|
+    arg = job_data[<span class="ruby-string">&quot;@argv&quot;</span>][i]
+    if arg[0].chr == <span class="ruby-string">&quot;/&quot;</span>
+      job_data[<span class="ruby-string">&quot;@argv&quot;</span>][i] = self.put_file(arg, job_data)
+    end
+  }
+
+  return job_data
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="run_job-method" class="method-detail ">
+        <a name="method-c-run_job"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">run_job</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>Constructs a Neptune job to run the user’s code as a Babel job (task
+queue) from the given parameters.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="run_job-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 222</span>
+def self.run_job(job_data)
+  run_data = self.convert_to_neptune_params(job_data)
+  run_data[:type] = <span class="ruby-string">&quot;babel&quot;</span>
+
+  <span class="ruby-comment"># TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to</span>
+  <span class="ruby-comment"># exec tasks over it, instead of locally.</span>
+  if job_data[<span class="ruby-string">&quot;@run_local&quot;</span>].nil?
+    run_data[:run_local] = true
+    run_data[:engine] = <span class="ruby-string">&quot;executor-sqs&quot;</span>
+  end
+
+  return <span class="ruby-constant">Kernel</span>.neptune(run_data)
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="validate_inputs-method" class="method-detail ">
+        <a name="method-c-validate_inputs"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">validate_inputs</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>For jobs where the code is stored remotely, this method ensures that  the
+code and any possible inputs actually do exist, before attempting to use
+them for computation.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="validate_inputs-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 113</span>
+def self.validate_inputs(job_data)
+  controller = self.get_appcontroller(job_data)
+
+  <span class="ruby-comment"># First, make sure the code exists</span>
+  <span class="ruby-constant">NeptuneHelper</span>.require_file_to_exist(job_data[<span class="ruby-string">&quot;@code&quot;</span>], job_data, controller)
+
+  if job_data[<span class="ruby-string">&quot;@argv&quot;</span>].nil? or job_data[<span class="ruby-string">&quot;@argv&quot;</span>].empty?
+    return
+  end
+
+  <span class="ruby-comment"># We assume anything that begins with a slash is a remote file</span>
+  job_data[<span class="ruby-string">&quot;@argv&quot;</span>].each { |arg|
+    if arg[0].chr == <span class="ruby-string">&quot;/&quot;</span>
+      <span class="ruby-constant">NeptuneHelper</span>.require_file_to_exist(arg, job_data, controller)
+    end
+  }
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+      <div id="wait_and_get_output-method" class="method-detail ">
+        <a name="method-c-wait_and_get_output"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">wait_and_get_output</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          <p>Constructs a Neptune job to get the output of a Babel job. If the job is
+not yet finished, this function waits until it does, and then returns the
+output of the job.</p>
+          
+
+          
+          <div class="method-source-code"
+            id="wait_and_get_output-source">
+<pre>
+<span class="ruby-comment"># File lib/babel.rb, line 239</span>
+def self.wait_and_get_output(job_data)
+  output_data = self.convert_to_neptune_params(job_data)
+  output_data[:type] = <span class="ruby-string">&quot;output&quot;</span>
+
+  output = <span class="ruby-string">&quot;&quot;</span>
+  time_to_sleep = <span class="ruby-constant">SLEEP_TIME</span>
+  loop {
+    output = <span class="ruby-constant">Kernel</span>.neptune(output_data)[:output]
+    if output == <span class="ruby-constant">DOES_NOT_EXIST</span>
+      <span class="ruby-comment"># Exponentially back off, up to a limit of MAX_SLEEP_TIME</span>
+      <span class="ruby-constant">Kernel</span>.sleep(time_to_sleep)
+      if time_to_sleep &lt; <span class="ruby-constant">MAX_SLEEP_TIME</span>
+        time_to_sleep *= 2
+      end
+    else
+      break
+    end
+  }
+
+  return output
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
+      </div>
+
+    
+    </div>
+  
+
+  </div>
+
+  <div id="validator-badges">
+    <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+    <p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
+      Rdoc Generator</a> 2</small>.</p>
+  </div>
+
+</body>
+</html>
+