neptune 0.1.4 → 0.2.0

data/lib/babel.rb

@@ -0,0 +1,260 @@
+#!/usr/bin/ruby
+# Programmer: Chris Bunch (cgb@cs.ucsb.edu)
+
+require 'app_controller_client'
+require 'common_functions'
+require 'custom_exceptions'
+require 'neptune'
+
+
+# The promise gem gives us futures / promises out-of-the-box, which we need
+# to hide the fact that babel jobs are asynchronous.
+require 'rubygems'
+require 'promise'
+require 'future'
+
+
+# If the user doesn't give us enough info to infer what bucket we should place
+# their code in, this message is displayed and execution aborts.
+NEEDS_BUCKET_INFO = "When running Babel jobs with local inputs / code, the " +
+  "bucket to store them in must be specified by either the :bucket_name " +
+  "parameter or the BABEL_BUCKET_NAME environment variable."
+
+
+# The constant string that a Neptune output job returns if the output does not
+# yet exist.
+DOES_NOT_EXIST = "error: output does not exist"
+
+
+# The initial amount of time, in seconds, to sleep between output job requests.
+# An exponential backoff is used with this value as the starting sleep time.
+SLEEP_TIME = 5  # seconds
+
+
+# The maximum amount of time that we should sleep to, when waiting for output
+# job requests.
+MAX_SLEEP_TIME = 60  # seconds
+
+# Babel provides a nice wrapper around Neptune jobs. Instead of making users
+# write multiple Neptune jobs to actually run code (e.g., putting input in the
+# datastore, run the job, get the output back), Babel automatically handles
+# this.
+def babel(params)
+  # Since this whole function should run asynchronously, we run it as a future.
+  # It automatically starts running in a new thread, and attempting to get the
+  # value of what this returns causes it to block until the job completes.
+  future {
+    job_data = BabelHelper.convert_from_neptune_params(params)
+    NeptuneHelper.validate_storage_params(job_data)  # adds in S3 storage params
+
+    # :code is the only required parameter - everything else can use default vals
+    NeptuneHelper.require_param("@code", job_data)
+
+    if job_data["@output"].nil? or job_data["@output"].empty?
+      job_data["@output"] = BabelHelper.generate_output_location(job_data)
+    end
+    BabelHelper.ensure_output_does_not_exist(job_data)
+
+    if job_data["@is_remote"]
+      BabelHelper.validate_inputs(job_data)
+    else
+      BabelHelper.put_code(job_data)
+      BabelHelper.put_inputs(job_data)
+    end
+
+    BabelHelper.run_job(job_data)
+    # So actually retrieving the job's output is done via a promise, so only if
+    # the user actually uses the value do we actually go and poll for output.
+    # The running of the job is done above, outside of the promise, so
+    # the job is always run, regardless of whether or not we get its output.
+    BabelHelper.wait_and_get_output(job_data)
+    # promise { BabelHelper.wait_and_get_output(job_data) }
+  }
+end
+
+
+# This module provides convenience functions for babel().
+module BabelHelper
+  # If the user fails to give us an output location, this function will generate
+  # one for them, based on either the location of their code (for remotely
+  # specified code), or a babel parameter (for locally specified code).
+  def self.generate_output_location(job_data)
+    if job_data["@is_remote"]
+      # We already know the bucket name - the same one that the user
+      # has told us their code is located in.
+      prefix = job_data["@code"].scan(/\/(.*?)\//)[0].to_s
+    else
+      prefix = self.get_bucket_for_local_data(job_data)
+    end
+      
+    return "/#{prefix}/babel/temp-#{CommonFunctions.get_random_alphanumeric()}"
+  end
+
+  # Provides a common way for callers to get the name of the bucket that
+  # should be used for Neptune jobs where the code is stored locally.
+  def self.get_bucket_for_local_data(job_data)
+    bucket_name = job_data["@bucket_name"] || ENV['BABEL_BUCKET_NAME']
+
+    if bucket_name.nil?
+      raise BadConfigurationException.new(NEEDS_BUCKET_INFO)
+    end
+
+    # If the bucket name starts with a slash, remove it
+    if bucket_name[0].chr == "/"
+      bucket_name = bucket_name[1, bucket_name.length]
+    end
+
+    return bucket_name
+  end
+
+  # For jobs where the code is stored remotely, this method ensures that 
+  # the code and any possible inputs actually do exist, before attempting to
+  # use them for computation.
+  def self.validate_inputs(job_data)
+    controller = self.get_appcontroller(job_data)
+
+    # First, make sure the code exists
+    NeptuneHelper.require_file_to_exist(job_data["@code"], job_data, controller)
+
+    if job_data["@argv"].nil? or job_data["@argv"].empty?
+      return
+    end
+
+    # We assume anything that begins with a slash is a remote file
+    job_data["@argv"].each { |arg|
+      if arg[0].chr == "/"
+        NeptuneHelper.require_file_to_exist(arg, job_data, controller)
+      end
+    }
+  end
+
+  # To avoid accidentally overwriting outputs from previous jobs, we first
+  # check to make sure an output file doesn't exist before starting a new job
+  # with the given name.
+  def self.ensure_output_does_not_exist(job_data)
+    file = job_data["@output"]
+    controller = self.get_appcontroller(job_data)
+    puts job_data.inspect
+    NeptuneHelper.require_file_to_not_exist(file, job_data, controller)
+  end
+
+  # Returns an AppControllerClient for the given job data.
+  def self.get_appcontroller(job_data)
+    keyname = job_data["@keyname"] || "appscale"
+    shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+    secret = CommonFunctions.get_secret_key(keyname)
+    return AppControllerClient.new(shadow_ip, secret)
+  end
+
+  # Stores the user's code (and the directory it's in, and directories in the
+  # same directory as the user's code, since there could be libraries used)
+  # in the remote datastore.
+  def self.put_code(job_data)
+    code_dir = File.dirname(job_data["@code"])
+    code = File.basename(job_data["@code"])
+    remote_code_dir = self.put_file(code_dir, job_data)
+    job_data["@code"] = remote_code_dir + "/" + code
+    return job_data["@code"]
+  end
+
+  # If any input files are specified, they are copied to the remote datastore
+  # via Neptune 'input' jobs. Inputs are assumed to be files on the local
+  # filesystem if they begin with a slash, and job_data gets updated with
+  # the remote location of these files.
+  def self.put_inputs(job_data)
+    if job_data["@argv"].nil? or job_data["@argv"].empty?
+      return job_data
+    end
+
+    job_data["@argv"].each_index { |i|
+      arg = job_data["@argv"][i]
+      if arg[0].chr == "/"
+        job_data["@argv"][i] = self.put_file(arg, job_data)
+      end
+    }
+
+    return job_data
+  end
+
+  # If the user gives us local code or local inputs, this function will
+  # run a Neptune 'input' job to store the data remotely.
+  def self.put_file(local_path, job_data)
+    input_data = self.convert_to_neptune_params(job_data)
+    input_data[:type] = "input"
+    input_data[:local] = local_path
+
+    bucket_name = self.get_bucket_for_local_data(job_data)
+    input_data[:remote] = "/#{bucket_name}/babel#{local_path}"
+
+    Kernel.neptune(input_data)
+
+    return input_data[:remote]
+  end
+
+  # Neptune internally uses job_data with keys of the form @name, but since the
+  # user has given them to us in the form :name, we convert it here.
+  # TODO(cgb): It looks like this conversion to/from may be unnecessary since
+  # neptune() just re-converts it - how can we remove it?
+  def self.convert_from_neptune_params(params)
+    job_data = {}
+    params.each { |k, v|
+      key = "@#{k}"
+      job_data[key] = v
+    }
+    return job_data
+  end
+
+  # Neptune input jobs expect keys of the form :name, but since we've already
+  # converted them to the form @name, this function reverses that conversion.
+  def self.convert_to_neptune_params(job_data)
+    neptune_params = {}
+
+    job_data.each { |k, v|
+      key = k.delete("@").to_sym
+      neptune_params[key] = v
+    }
+
+    return neptune_params
+  end
+  
+  # Constructs a Neptune job to run the user's code as a Babel job (task queue)
+  # from the given parameters.
+  def self.run_job(job_data)
+    run_data = self.convert_to_neptune_params(job_data)
+    run_data[:type] = "babel"
+
+    # TODO(cgb): Once AppScale+Babel gets support for RabbitMQ, change this to
+    # exec tasks over it, instead of locally.
+    if job_data["@run_local"].nil?
+      run_data[:run_local] = true
+      run_data[:engine] = "executor-sqs"
+    end
+
+    return Kernel.neptune(run_data)
+  end
+
+  # Constructs a Neptune job to get the output of a Babel job. If the job is not
+  # yet finished, this function waits until it does, and then returns the output
+  # of the job.
+  def self.wait_and_get_output(job_data)
+    output_data = self.convert_to_neptune_params(job_data)
+    output_data[:type] = "output"
+
+    output = ""
+    time_to_sleep = SLEEP_TIME
+    loop {
+      output = Kernel.neptune(output_data)[:output]
+      if output == DOES_NOT_EXIST
+        # Exponentially back off, up to a limit of MAX_SLEEP_TIME
+        Kernel.sleep(time_to_sleep)
+        if time_to_sleep < MAX_SLEEP_TIME
+          time_to_sleep *= 2
+        end
+      else
+        break
+      end
+    }
+  
+    return output
+  end
+end

data/lib/common_functions.rb

@@ -9,11 +9,7 @@ require 'socket'
 require 'timeout'
 require 'yaml'
 
-module Kernel
-  def shell(command)
-    return `#{command}`
-  end
-end
+require 'custom_exceptions'
 
 # A helper module that aggregates functions that are not part of Neptune's
 # core functionality. Specifically, this module contains methods to scp
@@ -21,6 +17,27 @@ end
 # often needed to determine which machine should be used for computation
 # or to copy over code and input files.
 module CommonFunctions
+  # Executes a command and returns the result. Is needed to get around
+  # Flexmock's inability to mock out Kernel:` (the standard shell exec
+  # method).
+  def self.shell(cmd)
+    return `#{cmd}`
+  end
+
+  # Returns a random string composed of alphanumeric characters, as long
+  # as the user requests.
+  def self.get_random_alphanumeric(length=10)
+    random = ""
+    possible = "0123456789abcdefghijklmnopqrstuvxwyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    possibleLength = possible.length
+    
+    length.times { |index|
+      random << possible[rand(possibleLength)]
+    }
+     
+    return random
+  end
+
   # Copies a file to the Shadow node (head node) within AppScale. 
   # The caller specifies
   # the local file location, the destination where the file should be
@@ -30,14 +47,11 @@ module CommonFunctions
   def self.scp_to_shadow(local_file_loc,
                          remote_file_loc,
                          keyname,
-                         is_dir=false,
-                         file=File,
-                         get_from_yaml=CommonFunctions.method(:get_from_yaml),
-                         scp_file=CommonFunctions.method(:scp_file))
+                         is_dir=false)
 
-    shadow_ip = get_from_yaml.call(keyname, :shadow, file)
-    ssh_key = file.expand_path("~/.appscale/#{keyname}.key")
-    scp_file.call(local_file_loc, remote_file_loc, shadow_ip, ssh_key, is_dir)
+    shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+    ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
+    CommonFunctions.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key, is_dir)
   end 
  
   # Performs the actual remote copying of files: given the IP address
@@ -47,22 +61,22 @@ module CommonFunctions
   # wrong IP is given. If the user specifies that the file to copy is
   # actually a directory, we append the -r flag to scp as well.
   def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc,
-    is_dir=false, file=File, fileutils=FileUtils, kernel=Kernel)
+    is_dir=false)
     cmd = ""
-    local_file_loc = file.expand_path(local_file_loc)
+    local_file_loc = File.expand_path(local_file_loc)
 
     ssh_args = "-o StrictHostkeyChecking=no 2>&1"
     ssh_args << " -r " if is_dir
 
-    public_key_loc = file.expand_path(public_key_loc)
+    public_key_loc = File.expand_path(public_key_loc)
     cmd = "scp -i #{public_key_loc} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
     cmd << "; echo $? >> ~/.appscale/retval"
 
-    retval_loc = file.expand_path("~/.appscale/retval")
-    fileutils.rm_f(retval_loc)
+    retval_loc = File.expand_path("~/.appscale/retval")
+    FileUtils.rm_f(retval_loc)
 
     begin
-      Timeout::timeout(-1) { kernel.shell("#{cmd}") }
+      Timeout::timeout(-1) { CommonFunctions.shell("#{cmd}") }
     rescue Timeout::Error
       abort("Remotely copying over files failed. Is the destination machine" +
         " on and reachable from this computer? We tried the following" +
@@ -70,11 +84,11 @@ module CommonFunctions
     end
 
     loop {
-      break if file.exists?(retval_loc)
+      break if File.exists?(retval_loc)
       sleep(5)
     }
 
-    retval = (file.open(retval_loc) { |f| f.read }).chomp
+    retval = (File.open(retval_loc) { |f| f.read }).chomp
     if retval != "0"
       abort("\n\n[#{cmd}] returned #{retval} instead of 0 as expected. Is " +
         "your environment set up properly?")
@@ -88,16 +102,16 @@ module CommonFunctions
   # method aborts if the value doesn't exist or the YAML file is malformed.
   # If the required flag is set to false, it returns nil in either scenario
   # instead.
-  def self.get_from_yaml(keyname, tag, required=true, file=File, yaml=YAML)
-    location_file = file.expand_path("~/.appscale/locations-#{keyname}.yaml")
+  def self.get_from_yaml(keyname, tag, required=true)
+    location_file = File.expand_path("~/.appscale/locations-#{keyname}.yaml")
   
-    if !file.exists?(location_file)
-      abort("An AppScale instance is not currently running with the provided" +
-        " keyname, \"#{keyname}\".")
+    if !File.exists?(location_file)
+      raise BadConfigurationException.new("An AppScale instance is not " +
+        "currently running with the provided keyname, \"#{keyname}\".")
     end
     
     begin
-      tree = yaml.load_file(location_file)
+      tree = YAML.load_file(location_file)
     rescue ArgumentError
       if required
         abort("The yaml file you provided was malformed. Please correct any" +
@@ -121,7 +135,7 @@ module CommonFunctions
   # Returns the secret key needed for communication with AppScale's
   # Shadow node. This method is a nice frontend to the get_from_yaml
   # function, as the secret is stored in a YAML file.
-  def self.get_secret_key(keyname, required=true, file=File, yaml=YAML)
-    return CommonFunctions.get_from_yaml(keyname, :secret, required, file, yaml)
+  def self.get_secret_key(keyname, required=true)
+    return CommonFunctions.get_from_yaml(keyname, :secret, required)
   end
 end

data/lib/custom_exceptions.rb

@@ -0,0 +1,10 @@
+# Programmer: Chris Bunch
+
+class AppControllerException < Exception
+end
+
+class BadConfigurationException < Exception
+end
+
+class FileNotFoundException < Exception
+end

data/lib/neptune.rb

@@ -3,6 +3,7 @@
 
 require 'app_controller_client'
 require 'common_functions'
+require 'custom_exceptions'
 
 # Setting verbose to nil here suppresses the otherwise
 # excessive SSL cert warning messages that will pollute
@@ -20,6 +21,12 @@ $VERBOSE = nil
 #MR_RUN_JOB_REQUIRED = %w{ }
 #MR_REQUIRED = %w{ output }
 
+# A list of all the Neptune job types that we support
+ALLOWED_JOB_TYPES = %w{acl cicero compile erlang mpi input output ssa babel upc x10}
+
+# The string to display for disallowed job types.
+JOB_TYPE_NOT_ALLOWED = "The job type you specified is not supported."
+
 # A list of Neptune jobs that do not require nodes to be spawned
 # up for computation
 NO_NODES_NEEDED = ["acl", "input", "output", "compile"]
@@ -34,7 +41,7 @@ ALLOWED_STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"]
 
 # A list of jobs that require some kind of work to be done before
 # the actual computation can be performed.
-NEED_PREPROCESSING = ["compile", "erlang", "mpi", "ssa"]
+NEED_PREPROCESSING = ["babel", "compile", "erlang", "mpi", "ssa"]
 
 # A set of methods and constants that we've monkey-patched to enable Neptune
 # support. In the future, it is likely that the only exposed / monkey-patched
@@ -45,385 +52,443 @@ NEED_PREPROCESSING = ["compile", "erlang", "mpi", "ssa"]
 class Object
 end
 
-# Certain types of jobs need steps to be taken before they
-# can be started (e.g., copying input data or code over).
-# This method dispatches the right method to use based
-# on the type of the job that the user has asked to run.
-def do_preprocessing(job_data)
-  job_type = job_data["@type"]
-  if !NEED_PREPROCESSING.include?(job_type)
-    return
-  end
-
-  preprocess = "preprocess_#{job_type}".to_sym
-  send(preprocess, job_data)
-end
+module NeptuneHelper
+  # Certain types of jobs need steps to be taken before they
+  # can be started (e.g., copying input data or code over).
+  # This method dispatches the right method to use based
+  # on the type of the job that the user has asked to run.
+  def self.do_preprocessing(job_data, controller)
+    job_type = job_data["@type"]
+    if !NEED_PREPROCESSING.include?(job_type)
+      return
+    end
 
-# This preprocessing method copies over the user's code to the
-# Shadow node so that it can be compiled there. A future version
-# of this method may also copy over libraries as well.
-def preprocess_compile(job_data, shell=Kernel.method(:`))
-  code = File.expand_path(job_data["@code"])
-  if !File.exists?(code)
-    abort("The source file #{code} does not exist.")
+    # Don't worry about adding on the self. prefix - send will resolve
+    # it the right way
+    preprocess = "preprocess_#{job_type}".to_sym
+    send(preprocess, job_data, controller)
   end
 
-  suffix = code.split('/')[-1]
-  dest = "/tmp/#{suffix}"
-  keyname = job_data["@keyname"]
-  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
-
-  ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}"
-  remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}' 2>&1"
-  puts remove_dir
-  shell.call(remove_dir)
+  # This preprocessing method copies over the user's code to the
+  # Shadow node so that it can be compiled there. A future version
+  # of this method may also copy over libraries as well.
+  def self.preprocess_compile(job_data, controller)
+    code = File.expand_path(job_data["@code"])
+    if !File.exists?(code)
+      raise BadConfigurationException.new("The source file #{code} does not exist.")
+    end
 
-  CommonFunctions.scp_to_shadow(code, dest, keyname, is_dir=true)
+    suffix = code.split('/')[-1]
+    dest = "/tmp/#{suffix}"
+    keyname = job_data["@keyname"]
+    shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
 
-  job_data["@code"] = dest
-end
+    ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}"
+    remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}' 2>&1"
+    Kernel.puts remove_dir
+    CommonFunctions.shell(remove_dir)
+    CommonFunctions.scp_to_shadow(code, dest, keyname, is_dir=true)
 
-def preprocess_erlang(job_data, file=File, common_functions=CommonFunctions)
-  if !job_data["@code"]
-    abort("When running Erlang jobs, :code must be specified.")
+    job_data["@code"] = dest
   end
 
-  source_code = file.expand_path(job_data["@code"])
-  if !file.exists?(source_code)
-    abort("The specified code, #{job_data['@code']}," +
-      " didn't exist. Please specify one that exists and try again")
-  end
-  dest_code = "/tmp/"
+  def self.preprocess_erlang(job_data, controller)
+    self.require_param("@code", job_data)
 
-  keyname = job_data["@keyname"]
-  common_functions.scp_to_shadow(source_code, dest_code, keyname)
-end
+    source_code = File.expand_path(job_data["@code"])
+    if !File.exists?(source_code)
+      raise BadConfigurationException.new("The specified code, #{job_data['@code']}," +
+        " didn't exist. Please specify one that exists and try again")
+    end
+    dest_code = "/tmp/"
 
-# This preprocessing method verifies that the user specified the number of nodes
-# to use. If they also specified the number of processes to use, we also verify
-# that this value is at least as many as the number of nodes (that is, nodes
-# can't be underprovisioned in MPI).
-def preprocess_mpi(job_data)
-  if !job_data["@nodes_to_use"]
-    abort("When running MPI jobs, :nodes_to_use must be specified.")
+    keyname = job_data["@keyname"]
+    CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
   end
 
-  if !job_data["@procs_to_use"]
-    abort("When running MPI jobs, :procs_to_use must be specified.")
-  end
+  # This preprocessing method verifies that the user specified the number of nodes
+  # to use. If they also specified the number of processes to use, we also verify
+  # that this value is at least as many as the number of nodes (that is, nodes
+  # can't be underprovisioned in MPI).
+  def self.preprocess_mpi(job_data, controller)
+    self.require_param("@nodes_to_use", job_data)
+    self.require_param("@procs_to_use", job_data)
+
+    if job_data["@procs_to_use"]
+      p = job_data["@procs_to_use"]
+      n = job_data["@nodes_to_use"]
+      if p < n
+        raise BadConfigurationException.new(":procs_to_use must be at least as " +
+          "large as :nodes_to_use.") 
+      end
+    end
 
-  if job_data["@procs_to_use"]
-    p = job_data["@procs_to_use"]
-    n = job_data["@nodes_to_use"]
-    if p < n
-      abort("When specifying both :procs_to_use and :nodes_to_use" +
-        ", :procs_to_use must be at least as large as :nodes_to_use. Please " +
-        "change this and try again. You specified :procs_to_use = #{p} and" +
-        ":nodes_to_use = #{n}.")
+    if job_data["@argv"]
+      argv = job_data["@argv"]
+
+      if argv.class == String
+        job_data["@argv"] = argv
+      elsif argv.class == Array
+        job_data["@argv"] = argv.join(' ')
+      else
+        raise BadConfigurationException.new(":argv must be either a String or Array") 
+      end
     end
+
+    return job_data
   end
 
-  if job_data["@argv"]
-    argv = job_data["@argv"]
-    if argv.class != String and argv.class != Array
-      abort("The value specified for :argv must be either a String or Array") 
+  # This preprocessing method verifies that the user specified the number of
+  # trajectories to run, via either :trajectories or :simulations. Both should
+  # not be specified - only one or the other, and regardless of which they
+  # specify, convert it to be :trajectories.
+  def self.preprocess_ssa(job_data, controller)
+    if job_data["@simulations"] and job_data["@trajectories"]
+      raise BadConfigurationException.new(":simulations and :trajectories " +
+        "not both be specified.")
     end
 
-    if argv.class == Array
-      job_data["@argv"] = argv.join(' ')
+    if job_data["@simulations"]
+      job_data["@trajectories"] = job_data["@simulations"]
+      job_data.delete("@simulations")
     end
-  end
 
-  return job_data
-end
+    self.require_param("@trajectories", job_data)
+    return job_data
+  end
 
-# This preprocessing method verifies that the user specified the number of
-# trajectories to run, via either :trajectories or :simulations. Both should
-# not be specified - only one or the other, and regardless of which they
-# specify, convert it to be :trajectories.
-def preprocess_ssa(job_data)
-  if job_data["@simulations"] and job_data["@trajectories"]
-    abort("Both :simulations and :trajectories cannot be specified - use one" +
-      " or the other.")
+  def self.require_param(param, job_data)
+    if !job_data[param]
+      raise BadConfigurationException.new("#{param} must be specified")
+    end
   end
 
-  if job_data["@simulations"]
-    job_data["@trajectories"] = job_data["@simulations"]
-    job_data.delete("@simulations")
+  def self.require_file_to_exist(file, job_data, controller)
+    if controller.does_file_exist?(file, job_data)
+      return
+    else
+      raise FileNotFoundException
+    end
   end
 
-  if !job_data["@trajectories"]
-    abort(":trajectories needs to be specified when running ssa jobs")
+  def self.require_file_to_not_exist(file, job_data, controller)
+    begin
+      self.require_file_to_exist(file, job_data, controller)
+      # no exception thrown previously means that the output file exists
+      raise BadConfigurationException.new('Output specified already exists')
+    rescue FileNotFoundException
+      return
+    end
   end
 
-  return job_data
-end
+  # This preprocessing method verifies that the user specified code that
+  # should be run, where the output should be placed, and an engine to run over.
+  # It also verifies that all files to be used are actually reachable.
+  # Supported engines can be found by contacting an AppScale node.
+  def self.preprocess_babel(job_data, controller)
+    self.require_param("@code", job_data)
+    self.require_param("@engine", job_data)
+    self.require_param("@output", job_data)
+
+    # For most code types, the file's name given is the thing to exec.
+    # For Java, the actual file to search for is whatever the user gives
+    # us, with a .class extension.
+    code_file_name = job_data["@code"]
+    if !job_data["@executable"].nil? and job_data["@executable"] == "java"
+      code_file_name += ".class"
+    end
+
+    self.require_file_to_exist(code_file_name, job_data, controller)
+    self.require_file_to_not_exist(job_data["@output"], job_data, controller)
+
+    if job_data["@argv"]
+      argv = job_data["@argv"]
+      if argv.class != Array
+        raise BadConfigurationException.new("argv must be an array")
+      end
 
-def get_job_data(params)
-  job_data = {}
-  params.each { |k, v|
-    key = "@#{k}"
-    job_data[key] = v
-  }
+      argv.each { |arg|
+        if arg =~ /\/.*\/.*/
+          self.require_file_to_exist(arg, job_data, controller)
+        end
+      }
+    end
 
-  job_data.delete("@job")
-  job_data["@keyname"] = params[:keyname] || "appscale"
+    if job_data["@appcfg_cookies"]
+      self.require_file_to_exist(job_data["@appcfg_cookies"], job_data, controller)
+    end
 
-  job_data["@type"] = job_data["@type"].to_s
-  type = job_data["@type"]
+    user_specified_engine = job_data["@engine"]
 
-  if type == "upc" or type == "x10"
-    job_data["@type"] = "mpi"
-    type = "mpi"
+    # validate the engine here
+    engines = controller.get_supported_babel_engines(job_data)
+    if !engines.include?(user_specified_engine)
+      raise BadConfigurationException.new("The engine you specified, " +
+        "#{user_specified_engine}, is not a supported engine. Supported engines" +
+        " are: #{engines.join(', ')}")
+    end
   end
 
-  # kdt jobs also run as mpi jobs, but need to pass along an executable
-  # parameter to let mpiexec know to use python to exec it
-  if type == "kdt"
-    job_data["@type"] = "mpi"
-    type = "mpi"
+  def self.get_job_data(params)
+    job_data = {}
+    params.each { |k, v|
+      key = "@#{k}"
+      job_data[key] = v
+    }
 
-    job_data["@executable"] = "python"
-  end
+    job_data.delete("@job")
+    job_data["@keyname"] = params[:keyname] || "appscale"
 
-  if job_data["@nodes_to_use"].class == Hash
-    job_data["@nodes_to_use"] = job_data["@nodes_to_use"].to_a.flatten
-  end
+    job_data["@type"] = job_data["@type"].to_s
+    type = job_data["@type"]
 
-  if !NO_OUTPUT_NEEDED.include?(type)
-    if (job_data["@output"].nil? or job_data["@output"] == "")
-      abort("Job output must be specified")
+    if !ALLOWED_JOB_TYPES.include?(type)
+      raise BadConfigurationException.new(JOB_TYPE_NOT_ALLOWED)
     end
 
-    if job_data["@output"][0].chr != "/"
-      abort("Job output must begin with a slash ('/')")
+    if type == "upc" or type == "x10"
+      job_data["@type"] = "mpi"
+      type = "mpi"
     end
-  end
 
-  return job_data
-end
+    # kdt jobs also run as mpi jobs, but need to pass along an executable
+    # parameter to let mpiexec know to use python to exec it
+    if type == "kdt"
+      job_data["@type"] = "mpi"
+      type = "mpi"
 
-def validate_storage_params(job_data)
-  if !job_data["@storage"]
-    job_data["@storage"] = "appdb"
-  end
+      job_data["@executable"] = "python"
+    end
 
-  storage = job_data["@storage"]
-  if !ALLOWED_STORAGE_TYPES.include?(storage)
-    abort("Supported storage types are #{ALLOWED_STORAGE_TYPES.join(', ')}" +
-      " - we do not support #{storage}.")
-  end
+    if job_data["@nodes_to_use"].class == Hash
+      job_data["@nodes_to_use"] = job_data["@nodes_to_use"].to_a.flatten
+    end
 
-  # Our implementation for storing / retrieving via Google Storage
-  # and Walrus uses
-  # the same library as we do for S3 - so just tell it that it's S3
-  if storage == "gstorage" or storage == "walrus"
-    storage = "s3"
-    job_data["@storage"] = "s3"
-  end
+    if !NO_OUTPUT_NEEDED.include?(type)
+      if (job_data["@output"].nil? or job_data["@output"].empty?)
+        raise BadConfigurationException.new("Job output must be specified")
+      end
 
-  if storage == "s3"
-    ["EC2_ACCESS_KEY", "EC2_SECRET_KEY", "S3_URL"].each { |item|
-      if job_data["@#{item}"]
-        puts "Using specified #{item}"
-      else
-        if ENV[item]
-          puts "Using #{item} from environment"
-          job_data["@#{item}"] = ENV[item]
-        else
-          abort("When storing data to S3, #{item} must be specified or be in " + 
-            "your environment. Please do so and try again.")
-        end
+      if job_data["@output"][0].chr != "/"
+        raise BadConfigurationException.new("Job output must begin with a slash ('/')")
       end
-    }
+    end
+
+    return job_data
   end
 
-  return job_data
-end
+  def self.validate_storage_params(job_data)
+    job_data["@storage"] ||= "appdb"
 
-# This method takes a file on the local user's computer and stores it remotely
-# via AppScale. It returns a hash map indicating whether or not the job
-# succeeded and if it failed, the reason for it.
-def get_input(job_data, ssh_args, shadow_ip, controller, file=File,
-  shell=Kernel.method(:`))
-  result = {:result => :success}
+    storage = job_data["@storage"]
+    if !ALLOWED_STORAGE_TYPES.include?(storage)
+      raise BadConfigurationException.new("Supported storage types are " +
+        "#{ALLOWED_STORAGE_TYPES.join(', ')} - #{storage} is not supported.")
+    end
 
-  if !job_data["@local"]
-    abort("You failed to specify a file to copy over via the :local flag.")
-  end
+    # Our implementation for storing / retrieving via Google Storage
+    # and Walrus uses
+    # the same library as we do for S3 - so just tell it that it's S3
+    if storage == "gstorage" or storage == "walrus"
+      storage = "s3"
+      job_data["@storage"] = "s3"
+    end
 
-  local_file = file.expand_path(job_data["@local"])
-  if !file.exists?(local_file)
-    reason = "the file you specified to copy, #{local_file}, doesn't exist." + 
-        " Please specify a file that exists and try again."
-    return {:result => :failure, :reason => reason}  
-  end
+    if storage == "s3"
+      ["EC2_ACCESS_KEY", "EC2_SECRET_KEY", "S3_URL"].each { |item|
+        if job_data["@#{item}"]
+          Kernel.puts "Using specified #{item}"
+        else
+          if ENV[item]
+            Kernel.puts "Using #{item} from environment"
+            job_data["@#{item}"] = ENV[item]
+          else
+            raise BadConfigurationException.new("When storing data to S3, #{item} must be specified or be in " + 
+              "your environment. Please do so and try again.")
+          end
+        end
+      }
+    end
 
-  remote = "/tmp/neptune-input-#{rand(100000)}"
-  scp_cmd = "scp -r #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
-  puts scp_cmd
-  shell.call(scp_cmd)
-
-  job_data["@local"] = remote
-  puts "job data = #{job_data.inspect}"
-  response = controller.put_input(job_data)
-  if response
-    return {:result => :success}
-  else
-    # TODO - expand this to include the reason why it failed
-    return {:result => :failure}
+    return job_data
   end
-end
 
-# This method waits for AppScale to finish compiling the user's code, indicated
-# by AppScale copying the finished code to a pre-determined location.
-def wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location,
-  shell=Kernel.method(:`))
-  loop {
-    ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2>&1"
-    puts ssh_command
-    ssh_result = shell.call(ssh_command)
-    puts "result was [#{ssh_result}]"
-    if ssh_result =~ /No such file or directory/
-      puts "Still waiting for code to be compiled..."
-    else
-      puts "compilation complete! Copying compiled code to #{copy_to}"
-      return
-    end
-    sleep(5)
-  }
-end
+  # This method takes a file on the local user's computer and stores it remotely
+  # via AppScale. It returns a hash map indicating whether or not the job
+  # succeeded and if it failed, the reason for it.
+  def self.get_input(job_data, ssh_args, shadow_ip, controller)
+    result = {:result => :success}
 
-# This method sends out a request to compile code, waits for it to finish, and
-# gets the standard out and error returned from the compilation. This method
-# returns a hash containing the standard out, error, and a result that indicates
-# whether or not the compilation was successful.
-def compile_code(job_data, ssh_args, shadow_ip, shell=Kernel.method(:`))
-  compiled_location = controller.compile_code(job_data)
+    self.require_param("@local", job_data)
 
-  copy_to = job_data["@copy_to"]
+    local_file = File.expand_path(job_data["@local"])
+    if !File.exists?(local_file)
+      reason = "the file you specified to copy, #{local_file}, doesn't exist." + 
+          " Please specify a file that exists and try again."
+      return {:result => :failure, :reason => reason}  
+    end
 
-  wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
+    remote = "/tmp/neptune-input-#{rand(100000)}"
+    scp_cmd = "scp -r #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
+    Kernel.puts scp_cmd
+    CommonFunctions.shell(scp_cmd)
 
-  FileUtils.rm_rf(copy_to)
+    job_data["@local"] = remote
+    Kernel.puts "job data = #{job_data.inspect}"
+    response = controller.put_input(job_data)
+    if response
+      return {:result => :success}
+    else
+      # TODO - expand this to include the reason why it failed
+      return {:result => :failure}
+    end
+  end
 
-  scp_command = "scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2>&1"
-  puts scp_command
-  shell.call(scp_command)
+  # This method waits for AppScale to finish compiling the user's code, indicated
+  # by AppScale copying the finished code to a pre-determined location.
+  def self.wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
+    loop {
+      ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2>&1"
+      Kernel.puts ssh_command
+      ssh_result = CommonFunctions.shell(ssh_command)
+      Kernel.puts "result was [#{ssh_result}]"
+      if ssh_result =~ /No such file or directory/
+        Kernel.puts "Still waiting for code to be compiled..."
+      else
+        Kernel.puts "compilation complete! Copying compiled code to #{copy_to}"
+        return
+      end
+      sleep(5)
+    }
+  end
 
-  code = job_data["@code"]
-  dirs = code.split(/\//)
-  remote_dir = "/tmp/" + dirs[-1] 
+  # This method sends out a request to compile code, waits for it to finish, and
+  # gets the standard out and error returned from the compilation. This method
+  # returns a hash containing the standard out, error, and a result that indicates
+  # whether or not the compilation was successful.
+  def self.compile_code(job_data, ssh_args, shadow_ip)
+    compiled_location = controller.compile_code(job_data)
+    copy_to = job_data["@copy_to"]
+    self.wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
+
+    FileUtils.rm_rf(copy_to)
+
+    scp_command = "scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2>&1"
+    Kernel.puts scp_command
+    CommonFunctions.shell(scp_command)
+
+    code = job_data["@code"]
+    dirs = code.split(/\//)
+    remote_dir = "/tmp/" + dirs[-1] 
+
+    [remote_dir, compiled_location].each { |remote_files|
+      ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_files}' 2>&1"
+      Kernel.puts ssh_command
+      CommonFunctions.shell(ssh_command)
+    }
 
-  [remote_dir, compiled_location].each { |remote_files|
-    ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_files}' 2>&1"
-    puts ssh_command
-    shell.call(ssh_command)
-  }
+    return get_std_out_and_err(copy_to)
+  end
 
-  return get_std_out_and_err(copy_to)
-end
+  # This method returns a hash containing the standard out and standard error
+  # from a completed job, as well as a result field that indicates whether or
+  # not the job completed successfully (success = no errors).
+  def self.get_std_out_and_err(location)
+    result = {}
 
-# This method returns a hash containing the standard out and standard error
-# from a completed job, as well as a result field that indicates whether or
-# not the job completed successfully (success = no errors).
-def get_std_out_and_err(location)
-  result = {}
+    out = File.open("#{location}/compile_out") { |f| f.read.chomp! }
+    result[:out] = out
 
-  out = File.open("#{location}/compile_out") { |f| f.read.chomp! }
-  result[:out] = out
+    err = File.open("#{location}/compile_err") { |f| f.read.chomp! }
+    result[:err] = err
 
-  err = File.open("#{location}/compile_err") { |f| f.read.chomp! }
-  result[:err] = err
+    if result[:err]
+      result[:result] = :failure
+    else
+      result[:result] = :success
+    end    
 
-  if result[:err]
-    result[:result] = :failure
-  else
-    result[:result] = :success
-  end    
+    return result
+  end
 
-  return result
-end
+  def self.upload_app_for_cicero(job_data)
+    if !job_data["@app"]
+      Kernel.puts "No app specified, not uploading..." 
+      return
+    end
 
-def upload_app_for_cicero(job_data)
-  if !job_data["@app"]
-    puts "No app specified, not uploading..." 
-    return
-  end
+    app_location = File.expand_path(job_data["@app"])
+    if !File.exists?(app_location)
+      raise BadConfigurationException.new("The app you specified, #{app_location}, does not exist." + 
+        "Please specify one that does and try again.")
+    end
 
-  app_location = File.expand_path(job_data["@app"])
-  if !File.exists?(app_location)
-    abort("The app you specified, #{app_location}, does not exist." + 
-      "Please specify one that does and try again.")
-  end
+    keyname = job_data["@keyname"] || "appscale"
+    if job_data["@appscale_tools"]
+      upload_app = File.expand_path(job_data["@appscale_tools"]) +
+        File::SEPARATOR + "bin" + File::SEPARATOR + "appscale-upload-app"
+    else
+      upload_app = "appscale-upload-app"
+    end
 
-  keyname = job_data["@keyname"] || "appscale"
-  if job_data["@appscale_tools"]
-    upload_app = File.expand_path(job_data["@appscale_tools"]) +
-      File::SEPARATOR + "bin" + File::SEPARATOR + "appscale-upload-app"
-  else
-    upload_app = "appscale-upload-app"
+    Kernel.puts "Uploading AppEngine app at #{app_location}"
+    upload_command = "#{upload_app} --file #{app_location} --test --keyname #{keyname}"
+    Kernel.puts upload_command
+    Kernel.puts `#{upload_command}`
   end
 
-  puts "Uploading AppEngine app at #{app_location}"
-  upload_command = "#{upload_app} --file #{app_location} --test --keyname #{keyname}"
-  puts upload_command
-  puts `#{upload_command}`
-end
+  # This method actually runs the Neptune job, given information about the job
+  # as well as information about the node to send the request to.
+  def self.run_job(job_data, ssh_args, shadow_ip, secret)
+    controller = AppControllerClient.new(shadow_ip, secret)
+
+    # TODO - right now the job is assumed to succeed in many cases
+    # need to investigate the various failure scenarios
+    result = { :result => :success }
+
+    case job_data["@type"]
+    when "input"
+      result = self.get_input(job_data, ssh_args, shadow_ip, controller)
+    when "output"
+      result[:output] = controller.get_output(job_data)
+    when "get-acl"
+      job_data["@type"] = "acl"
+      result[:acl] = controller.get_acl(job_data)
+    when "set-acl"
+      job_data["@type"] = "acl"
+      result[:acl] = controller.set_acl(job_data)
+    when "compile"
+      result = self.compile_code(job_data, ssh_args, shadow_ip)
+    when "cicero"
+      self.upload_app_for_cicero(job_data)
+      msg = controller.start_neptune_job(job_data)
+      result[:msg] = msg
+      result[:result] = :failure if result[:msg] !~ /job is now running\Z/
+    else
+      msg = controller.start_neptune_job(job_data)
+      result[:msg] = msg
+      result[:result] = :failure if result[:msg] !~ /job is now running\Z/
+    end
 
-# This method actually runs the Neptune job, given information about the job
-# as well as information about the node to send the request to.
-def run_job(job_data, ssh_args, shadow_ip, secret,
-  controller=AppControllerClient, file=File)
-  controller = controller.new(shadow_ip, secret)
-
-  # TODO - right now the job is assumed to succeed in many cases
-  # need to investigate the various failure scenarios
-  result = { :result => :success }
-
-  case job_data["@type"]
-  when "input"
-    result = get_input(job_data, ssh_args, shadow_ip, controller, file)
-  when "output"
-    result[:output] = controller.get_output(job_data)
-  when "get-acl"
-    job_data["@type"] = "acl"
-    result[:acl] = controller.get_acl(job_data)
-  when "set-acl"
-    job_data["@type"] = "acl"
-    result[:acl] = controller.set_acl(job_data)
-  when "compile"
-    result = compile_code(job_data, ssh_args, shadow_ip)
-  when "cicero"
-    upload_app_for_cicero(job_data)
-    msg = controller.start_neptune_job(job_data)
-    result[:msg] = msg
-    result[:result] = :failure if result[:msg] !~ /job is now running\Z/
-  else
-    msg = controller.start_neptune_job(job_data)
-    result[:msg] = msg
-    result[:result] = :failure if result[:msg] !~ /job is now running\Z/
+    return result
   end
-
-  return result
 end
 
-# This method is the heart of Neptune - here, we take
-# blocks of code that the user has written and convert them
-# into HPC job requests. At a high level, the user can
-# request to run a job, retrieve a job's output, or
-# modify the access policy (ACL) for the output of a
-# job. By default, job data is private, but a Neptune
-# job can be used to set it to public later (and
-# vice-versa).
+# Make neptune() public so that babel() can call it
+public
+
+# This method is the heart of Neptune - here, we take blocks of code that the
+# user has written and convert them into HPC job requests. At a high level, 
+# the user can request to run a job, retrieve a job's output, or modify the 
+# access policy (ACL) for the output of a job. By default, job data is private,
+# but a Neptune job can be used to set it to public later (and vice-versa).
 def neptune(params)
-  puts "Received a request to run a job."
-  puts params[:type]
+  Kernel.puts "Received a request to run a job."
+  Kernel.puts params[:type]
 
-  job_data = get_job_data(params)
-  validate_storage_params(job_data)
-  puts "job data = #{job_data.inspect}"
-  do_preprocessing(job_data) 
+  job_data = NeptuneHelper.get_job_data(params)
+  NeptuneHelper.validate_storage_params(job_data)
+  Kernel.puts "job data = #{job_data.inspect}"
   keyname = job_data["@keyname"]
 
   shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
@@ -431,5 +496,7 @@ def neptune(params)
   ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
   ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "
 
-  return run_job(job_data, ssh_args, shadow_ip, secret)
+  controller = AppControllerClient.new(shadow_ip, secret)
+  NeptuneHelper.do_preprocessing(job_data, controller) 
+  return NeptuneHelper.run_job(job_data, ssh_args, shadow_ip, secret)
 end