neptune 0.2.0 → 0.2.1

data/doc/Object.html

@@ -135,13 +135,8 @@
 
     <div id="description">
       
-<p>A set of methods and constants that we’ve monkey-patched to enable
-Neptune support. In the future, it is likely that the only exposed /
-monkey-patched method should be job, while the others could probably be
-folded into either a Neptune-specific class or into <a
-href="CommonFunctions.html">CommonFunctions</a>. TODO(cbunch): This
-doesn’t look like it does anything - run the integration test and confirm
-one way or the other.</p>
+<p>Since we’re monkeypatching <a href="Object.html">Object</a> to add
+neptune() and babel(), a short blurb is necessary here to make rdoc happy.</p>
 
     </div>
 
@@ -255,13 +250,20 @@ handles this.</p>
           <div class="method-source-code"
             id="babel-source">
 <pre>
-<span class="ruby-comment"># File lib/babel.rb, line 42</span>
+<span class="ruby-comment"># File lib/babel.rb, line 43</span>
 def babel(params)
   <span class="ruby-comment"># Since this whole function should run asynchronously, we run it as a future.</span>
   <span class="ruby-comment"># It automatically starts running in a new thread, and attempting to get the</span>
   <span class="ruby-comment"># value of what this returns causes it to block until the job completes.</span>
   future {
+    if params[:storage]
+      params[:is_remote] = true
+    else
+      params[:is_remote] = false
+    end
+
     job_data = <span class="ruby-constant">BabelHelper</span>.convert_from_neptune_params(params)
+
     <span class="ruby-constant">NeptuneHelper</span>.validate_storage_params(job_data)  <span class="ruby-comment"># adds in S3 storage params</span>
 
     <span class="ruby-comment"># :code is the only required parameter - everything else can use default vals</span>
@@ -280,12 +282,7 @@ def babel(params)
     end
 
     <span class="ruby-constant">BabelHelper</span>.run_job(job_data)
-    <span class="ruby-comment"># So actually retrieving the job's output is done via a promise, so only if</span>
-    <span class="ruby-comment"># the user actually uses the value do we actually go and poll for output.</span>
-    <span class="ruby-comment"># The running of the job is done above, outside of the promise, so</span>
-    <span class="ruby-comment"># the job is always run, regardless of whether or not we get its output.</span>
     <span class="ruby-constant">BabelHelper</span>.wait_and_get_output(job_data)
-    <span class="ruby-comment"># promise { BabelHelper.wait_and_get_output(job_data) }</span>
   }
 end</pre>
           </div>
@@ -323,14 +320,14 @@ vice-versa).</p>
           <div class="method-source-code"
             id="neptune-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 485</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 58</span>
 def neptune(params)
-  <span class="ruby-constant">Kernel</span>.puts <span class="ruby-string">&quot;Received a request to run a job.&quot;</span>
-  <span class="ruby-constant">Kernel</span>.puts params[:type]
+  <span class="ruby-comment"># Kernel.puts &quot;Received a request to run a job.&quot;</span>
+  <span class="ruby-comment"># Kernel.puts params[:type]</span>
 
   job_data = <span class="ruby-constant">NeptuneHelper</span>.get_job_data(params)
   <span class="ruby-constant">NeptuneHelper</span>.validate_storage_params(job_data)
-  <span class="ruby-constant">Kernel</span>.puts &quot;job data = #{job_data.inspect}&quot;
+  <span class="ruby-comment"># Kernel.puts &quot;job data = #{job_data.inspect}&quot;</span>
   keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>]
 
   shadow_ip = <span class="ruby-constant">CommonFunctions</span>.get_from_yaml(keyname, :shadow)

data/doc/bin/neptune.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Fri Feb 10 19:53:13 -0800 2012</dd>
+      <dd class="modified-date">Sun Feb 12 16:12:33 -0800 2012</dd>
 
       
       <dt class="requires">Requires</dt>

data/doc/created.rid

@@ -1,7 +1,7 @@
-Sat, 11 Feb 2012 13:24:19 -0800
-bin/neptune	Fri, 10 Feb 2012 19:53:13 -0800
-lib/babel.rb	Sat, 11 Feb 2012 13:20:56 -0800
-lib/neptune.rb	Sat, 11 Feb 2012 11:16:25 -0800
-lib/custom_exceptions.rb	Sat, 31 Dec 2011 13:13:50 -0800
-lib/app_controller_client.rb	Wed, 25 Jan 2012 12:07:13 -0800
-lib/common_functions.rb	Sat, 11 Feb 2012 13:20:31 -0800
+Sun, 12 Feb 2012 16:40:58 -0800
+bin/neptune	Sun, 12 Feb 2012 16:12:33 -0800
+lib/babel.rb	Sun, 12 Feb 2012 16:35:05 -0800
+lib/neptune.rb	Sun, 12 Feb 2012 16:34:30 -0800
+lib/custom_exceptions.rb	Sun, 12 Feb 2012 16:18:29 -0800
+lib/app_controller_client.rb	Sun, 12 Feb 2012 16:13:32 -0800
+lib/common_functions.rb	Sun, 12 Feb 2012 16:18:14 -0800

data/doc/lib/app_controller_client_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Wed Jan 25 12:07:13 -0800 2012</dd>
+      <dd class="modified-date">Sun Feb 12 16:13:32 -0800 2012</dd>
 
       
       <dt class="requires">Requires</dt>

data/doc/lib/babel_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Sat Feb 11 13:20:56 -0800 2012</dd>
+      <dd class="modified-date">Sun Feb 12 16:35:05 -0800 2012</dd>
 
       
       <dt class="requires">Requires</dt>

data/doc/lib/common_functions_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Sat Feb 11 13:20:31 -0800 2012</dd>
+      <dd class="modified-date">Sun Feb 12 16:18:14 -0800 2012</dd>
 
       
       <dt class="requires">Requires</dt>

data/doc/lib/custom_exceptions_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Sat Dec 31 13:13:50 -0800 2011</dd>
+      <dd class="modified-date">Sun Feb 12 16:18:29 -0800 2012</dd>
 
       
       <dt class="requires">Requires</dt>

data/doc/lib/neptune_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Sat Feb 11 11:16:25 -0800 2012</dd>
+      <dd class="modified-date">Sun Feb 12 16:34:30 -0800 2012</dd>
 
       
       <dt class="requires">Requires</dt>

data/lib/app_controller_client.rb

@@ -12,12 +12,27 @@ require 'timeout'
 # long calls unless necessary.
 NO_TIMEOUT = -1
 
+
 # A client that uses SOAP messages to communicate with the underlying cloud
 # platform (here, AppScale). This client is similar to that used in the AppScale
 # Tools, but with non-Neptune SOAP calls removed.
 class AppControllerClient
-  attr_accessor :conn, :ip, :secret
+
+
+  # The SOAP client that we use to communicate with the AppController.
+  attr_accessor :conn
+  
+
+  # The IP address of the AppController that we will be connecting to.
+  attr_accessor :ip
   
+
+  # The secret string that is used to authenticate this client with
+  # AppControllers. It is initially generated by appscale-run-instances and can
+  # be found on the machine that ran that tool, or on any AppScale machine.
+  attr_accessor :secret
+  
+
   # A constructor that requires both the IP address of the machine to communicate
   # with as well as the secret (string) needed to perform communication.
   # AppControllers will reject SOAP calls if this secret (basically a password)
@@ -38,6 +53,7 @@ class AppControllerClient
     @conn.add_method("neptune_does_file_exist", "file", "job_data", "secret")
   end
 
+
   # A helper method to make SOAP calls for us. This method is mainly here to
   # reduce code duplication: all SOAP calls expect a certain timeout and can
   # tolerate certain exceptions, so we consolidate this code into this method.
@@ -73,6 +89,7 @@ class AppControllerClient
     end
   end
 
+
   # Initiates the start of a Neptune job, whether it be a HPC job (MPI, X10,
   # or MapReduce), or a scaling job (e.g., for AppScale itself). This method
   # should not be used for retrieving the output of a job or getting / setting
@@ -90,6 +107,7 @@ class AppControllerClient
     return result
   end
 
+
   # Stores a file stored on the user's local file system in the underlying
   # database. The user can specify to use either the underlying database
   # that AppScale is using, or alternative storage mechanisms (as of writing,
@@ -104,6 +122,7 @@ class AppControllerClient
     return result
   end
 
+
   # Retrieves the output of a Neptune job, stored in an underlying
   # database. Within AppScale, a special application runs, referred to as the
   # Repository, which provides a key-value interface to Neptune job data.
@@ -123,6 +142,7 @@ class AppControllerClient
     return result
   end
 
+
   # Returns the ACL associated with the named piece of data stored
   # in the underlying cloud platform. Right now, data can only be
   # public or private, but future versions will add individual user
@@ -137,6 +157,7 @@ class AppControllerClient
     return result
   end
 
+
   # Sets the ACL of a specified pieces of data stored in the underlying
   # cloud platform. As is the case with get_acl, ACLs can be either
   # public or private right now, but this will be expanded upon in
@@ -151,6 +172,9 @@ class AppControllerClient
     return result
   end
 
+
+  # Instructs the AppController to fetch the code specified and compile it.
+  # The result should then be placed in a location specified in the job data.
   def compile_code(job_data)
     result = ""
     make_call(NO_TIMEOUT, false) { 
@@ -160,6 +184,10 @@ class AppControllerClient
     return result
   end
 
+
+  # Asks the AppController for a list of all the Babel engines (each of which
+  # is a queue to store jobs and something that executes tasks) that are
+  # supported for the given credentials.
   def get_supported_babel_engines(job_data)
     result = []
     make_call(NO_TIMEOUT, false) {
@@ -168,6 +196,10 @@ class AppControllerClient
     return result
   end
 
+
+  # Asks the AppController to see if the given file exists in the remote
+  # datastore. If extra credentials are needed for this operation, they are
+  # searched for within the job data.
   def does_file_exist?(file, job_data)
     result = false
     make_call(NO_TIMEOUT, false) {

data/lib/babel.rb

@@ -35,6 +35,7 @@ SLEEP_TIME = 5  # seconds
 # job requests.
 MAX_SLEEP_TIME = 60  # seconds
 
+
 # Babel provides a nice wrapper around Neptune jobs. Instead of making users
 # write multiple Neptune jobs to actually run code (e.g., putting input in the
 # datastore, run the job, get the output back), Babel automatically handles
@@ -44,7 +45,14 @@ def babel(params)
   # It automatically starts running in a new thread, and attempting to get the
   # value of what this returns causes it to block until the job completes.
   future {
+    if params[:storage]
+      params[:is_remote] = true
+    else
+      params[:is_remote] = false
+    end
+
     job_data = BabelHelper.convert_from_neptune_params(params)
+
     NeptuneHelper.validate_storage_params(job_data)  # adds in S3 storage params
 
     # :code is the only required parameter - everything else can use default vals
@@ -63,23 +71,20 @@ def babel(params)
     end
 
     BabelHelper.run_job(job_data)
-    # So actually retrieving the job's output is done via a promise, so only if
-    # the user actually uses the value do we actually go and poll for output.
-    # The running of the job is done above, outside of the promise, so
-    # the job is always run, regardless of whether or not we get its output.
     BabelHelper.wait_and_get_output(job_data)
-    # promise { BabelHelper.wait_and_get_output(job_data) }
   }
 end
 
 
 # This module provides convenience functions for babel().
 module BabelHelper
+
+
   # If the user fails to give us an output location, this function will generate
   # one for them, based on either the location of their code (for remotely
   # specified code), or a babel parameter (for locally specified code).
   def self.generate_output_location(job_data)
-    if job_data["@is_remote"]
+    if job_data["@storage"]
       # We already know the bucket name - the same one that the user
       # has told us their code is located in.
       prefix = job_data["@code"].scan(/\/(.*?)\//)[0].to_s
@@ -90,6 +95,7 @@ module BabelHelper
     return "/#{prefix}/babel/temp-#{CommonFunctions.get_random_alphanumeric()}"
   end
 
+
   # Provides a common way for callers to get the name of the bucket that
   # should be used for Neptune jobs where the code is stored locally.
   def self.get_bucket_for_local_data(job_data)
@@ -107,6 +113,7 @@ module BabelHelper
     return bucket_name
   end
 
+
   # For jobs where the code is stored remotely, this method ensures that 
   # the code and any possible inputs actually do exist, before attempting to
   # use them for computation.
@@ -128,16 +135,18 @@ module BabelHelper
     }
   end
 
+
   # To avoid accidentally overwriting outputs from previous jobs, we first
   # check to make sure an output file doesn't exist before starting a new job
   # with the given name.
   def self.ensure_output_does_not_exist(job_data)
     file = job_data["@output"]
     controller = self.get_appcontroller(job_data)
-    puts job_data.inspect
+    # Kernel.puts job_data.inspect
     NeptuneHelper.require_file_to_not_exist(file, job_data, controller)
   end
 
+
   # Returns an AppControllerClient for the given job data.
   def self.get_appcontroller(job_data)
     keyname = job_data["@keyname"] || "appscale"
@@ -146,6 +155,7 @@ module BabelHelper
     return AppControllerClient.new(shadow_ip, secret)
   end
 
+
   # Stores the user's code (and the directory it's in, and directories in the
   # same directory as the user's code, since there could be libraries used)
   # in the remote datastore.
@@ -157,6 +167,7 @@ module BabelHelper
     return job_data["@code"]
   end
 
+
   # If any input files are specified, they are copied to the remote datastore
   # via Neptune 'input' jobs. Inputs are assumed to be files on the local
   # filesystem if they begin with a slash, and job_data gets updated with
@@ -176,6 +187,7 @@ module BabelHelper
     return job_data
   end
 
+
   # If the user gives us local code or local inputs, this function will
   # run a Neptune 'input' job to store the data remotely.
   def self.put_file(local_path, job_data)
@@ -191,6 +203,7 @@ module BabelHelper
     return input_data[:remote]
   end
 
+
   # Neptune internally uses job_data with keys of the form @name, but since the
   # user has given them to us in the form :name, we convert it here.
   # TODO(cgb): It looks like this conversion to/from may be unnecessary since
@@ -204,6 +217,7 @@ module BabelHelper
     return job_data
   end
 
+
   # Neptune input jobs expect keys of the form :name, but since we've already
   # converted them to the form @name, this function reverses that conversion.
   def self.convert_to_neptune_params(job_data)
@@ -216,12 +230,17 @@ module BabelHelper
 
     return neptune_params
   end
-  
+
+
   # Constructs a Neptune job to run the user's code as a Babel job (task queue)
   # from the given parameters.
   def self.run_job(job_data)
     run_data = self.convert_to_neptune_params(job_data)
-    run_data[:type] = "babel"
+
+    # Default to babel as the job type, if the user doesn't specify one.
+    if run_data[:type].nil? or run_data[:type].empty?
+      run_data[:type] = "babel"
+    end
 
     # TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to
     # exec tasks over it, instead of locally.
@@ -233,6 +252,7 @@ module BabelHelper
     return Kernel.neptune(run_data)
   end
 
+
   # Constructs a Neptune job to get the output of a Babel job. If the job is not
   # yet finished, this function waits until it does, and then returns the output
   # of the job.

data/lib/common_functions.rb

@@ -11,12 +11,15 @@ require 'yaml'
 
 require 'custom_exceptions'
 
+
 # A helper module that aggregates functions that are not part of Neptune's
 # core functionality. Specifically, this module contains methods to scp
 # files to other machines and the ability to read YAML files, which are
 # often needed to determine which machine should be used for computation
 # or to copy over code and input files.
 module CommonFunctions
+
+
   # Executes a command and returns the result. Is needed to get around
   # Flexmock's inability to mock out Kernel:` (the standard shell exec
   # method).
@@ -24,6 +27,7 @@ module CommonFunctions
     return `#{cmd}`
   end
 
+
   # Returns a random string composed of alphanumeric characters, as long
   # as the user requests.
   def self.get_random_alphanumeric(length=10)
@@ -32,28 +36,27 @@ module CommonFunctions
     possibleLength = possible.length
     
     length.times { |index|
-      random << possible[rand(possibleLength)]
+      random << possible[Kernel.rand(possibleLength)]
     }
      
     return random
   end
 
+
   # Copies a file to the Shadow node (head node) within AppScale. 
   # The caller specifies
   # the local file location, the destination where the file should be
   # placed, and the name of the key to use. The keyname is typically
   # specified by the Neptune job given, but defaults to ''appscale''
   # if not provided.
-  def self.scp_to_shadow(local_file_loc,
-                         remote_file_loc,
-                         keyname,
-                         is_dir=false)
-
+  def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname, is_dir=false)
     shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
     ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
-    CommonFunctions.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key, is_dir)
+    CommonFunctions.scp_file(local_file_loc, remote_file_loc, shadow_ip,
+      ssh_key, is_dir)
   end 
- 
+
+
   # Performs the actual remote copying of files: given the IP address
   # and other information from scp_to_shadow, attempts to use scp
   # to copy the file over. Aborts if the scp fails, which can occur
@@ -62,9 +65,8 @@ module CommonFunctions
   # actually a directory, we append the -r flag to scp as well.
   def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc,
     is_dir=false)
-    cmd = ""
-    local_file_loc = File.expand_path(local_file_loc)
 
+    local_file_loc = File.expand_path(local_file_loc)
     ssh_args = "-o StrictHostkeyChecking=no 2>&1"
     ssh_args << " -r " if is_dir
 
@@ -85,7 +87,7 @@ module CommonFunctions
 
     loop {
       break if File.exists?(retval_loc)
-      sleep(5)
+      Kernel.sleep(5)
     }
 
     retval = (File.open(retval_loc) { |f| f.read }).chomp
@@ -96,6 +98,7 @@ module CommonFunctions
     return cmd
   end
 
+
   # Given the AppScale keyname, reads the associated YAML file and returns
   # the contents of the given tag. The required flag (default value is true)
   # indicates whether a value must exist for this tag: if set to true, this
@@ -132,6 +135,7 @@ module CommonFunctions
     return value
   end
 
+
   # Returns the secret key needed for communication with AppScale's
   # Shadow node. This method is a nice frontend to the get_from_yaml
   # function, as the secret is stored in a YAML file.