neptune 0.2.1 → 0.2.2

data/lib/custom_exceptions.rb

@@ -1,8 +1,8 @@
 # Programmer: Chris Bunch
 
-# A special class of exceptions that are thrown whenever the AppController
+# A special class of exceptions that are thrown whenever the NeptuneManager
 # experiences an unexpected result.
-class AppControllerException < Exception
+class NeptuneManagerException < Exception
 end
 
 

data/lib/exodus.rb

@@ -0,0 +1,311 @@
+#!/usr/bin/env ruby
+# Programmer: Chris Bunch
+
+
+require 'babel'
+require 'custom_exceptions'
+require 'exodus_task_info'
+
+
+# Exodus provides further improvements to Babel. Instead of making users tell
+# us what compute, storage, and queue services they want to use (required for
+# babel calls), Exodus will automatically handle this for us. Callers need
+# to specify what clouds their job can run over, and Exodus will automatically
+# select the best cloud for their job and run it there.
+def exodus(jobs)
+  if jobs.class == Hash
+    job_given_as_hash = true
+    jobs = [jobs]
+  elsif jobs.class == Array
+    job_given_as_hash = false
+    ExodusHelper.ensure_all_jobs_are_hashes(jobs)
+  else
+    raise BadConfigurationException.new("jobs was a #{jobs.class}, which " +
+      "is not an acceptable class type")
+  end
+
+  tasks = []
+
+  jobs.each { |job|
+    ExodusHelper.ensure_all_params_are_present(job)
+    profiling_info = ExodusHelper.get_profiling_info(job)
+    clouds_to_run_task_on = ExodusHelper.get_clouds_to_run_task_on(job, 
+      profiling_info)
+    babel_tasks_to_run = ExodusHelper.generate_babel_tasks(job, 
+      clouds_to_run_task_on)
+    dispatched_tasks = ExodusHelper.run_job(babel_tasks_to_run)
+    tasks << ExodusTaskInfo.new(dispatched_tasks)
+  }
+
+  if job_given_as_hash
+    return tasks[0]
+  else
+    return tasks
+  end
+end
+
+
+# This module provides convenience functions for exodus(), to avoid cluttering
+# up Object or Kernel's namespace.
+module ExodusHelper
+
+
+  # A list of clouds that users can run tasks on via Exodus.
+  SUPPORTED_CLOUDS = [:AmazonEC2, :Eucalyptus, :GoogleAppEngine, 
+    :MicrosoftAzure]
+
+
+  CLOUD_CREDENTIALS = {
+    :AmazonEC2 => [:EC2_ACCESS_KEY, :EC2_SECRET_KEY, :EC2_URL, :S3_URL,
+      :S3_bucket_name],
+    :Eucalyptus => [:EUCA_ACCESS_KEY, :EUCA_SECRET_KEY, :EUCA_URL, 
+      :WALRUS_URL, :Walrus_bucket_name],
+    :GoogleAppEngine => [:appid, :appcfg_cookies, :function, 
+      :GStorage_bucket_name],
+    :MicrosoftAzure => [:WAZ_Account_Name, :WAZ_Access_Key, 
+      :WAZ_Container_Name]
+  }
+
+  
+  CLOUD_BABEL_PARAMS = {
+    :AmazonEC2 => {
+      :storage => "s3",
+      :engine => "executor-sqs"
+    },
+    :Eucalyptus => {
+      :storage => "walrus",
+      :engine => "executor-rabbitmq"
+    },
+    :GoogleAppEngine => {
+      :storage => "gstorage",
+      :engine => "appengine-push-q"
+    },
+    :MicrosoftAzure => {
+      :storage => "waz-storage",
+      :engine => "waz-push-q"
+    }
+  }
+
+
+  OPTIMIZE_FOR_CHOICES = [:performance, :cost, :auto]
+
+
+  # Given an Array of jobs to run, ensures that they are all Hashes, the
+  # standard format for Neptune jobs.
+  def self.ensure_all_jobs_are_hashes(jobs)
+    jobs.each { |job|
+      if job.class != Hash
+        raise BadConfigurationException.new("A job passed to exodus() was " +
+          "not a Hash, but was a #{job.class}")
+      end
+    }
+  end
+
+
+  # Given an Exodus job, validates its parameters, raising a 
+  # BadConfigurationException for any missing params.
+  def self.ensure_all_params_are_present(job)
+    if job[:clouds_to_use].nil?
+      raise BadConfigurationException.new(":clouds_to_use was not specified")
+    else
+      self.convert_clouds_to_use_to_array(job)
+      self.validate_clouds_to_use(job)
+      self.validate_optimize_for_param(job)
+      self.validate_files_argv_executable(job)
+    end
+  end
+
+
+  # Given a single Exodus job, checks to make sure it has either an Array
+  # of Strings or a single String listing the clouds that a given task can
+  # run on. Raises a BadConfigurationException if :clouds_to_use is not in
+  # the right format.
+  def self.convert_clouds_to_use_to_array(job)
+    clouds_class = job[:clouds_to_use].class
+    if clouds_class == Symbol
+      job[:clouds_to_use] = [job[:clouds_to_use]]
+    elsif clouds_class == Array
+      job[:clouds_to_use].each { |cloud|
+        if cloud.class != Symbol
+          raise BadConfigurationException.new("#{cloud} was not a String, " +
+            "but was a #{cloud.class}")
+        end
+      }
+    else
+      raise BadConfigurationException.new("#{job[:clouds_to_use]} was not " +
+        "a String or Array, but was a #{clouds_class}")
+    end
+  end
+
+
+  # Given a single Exodus job, checks to make sure that we can actually run
+  # it in this version of Neptune, and that the user has given us all the 
+  # credentials needed to use that cloud.
+  def self.validate_clouds_to_use(job)
+    self.ensure_credentials_are_in_correct_format(job)
+    self.propogate_credentials_from_environment(job)
+
+    job[:clouds_to_use].each { |cloud|
+      if SUPPORTED_CLOUDS.include?(cloud)
+        CLOUD_CREDENTIALS[cloud].each { |required_credential|
+          val_for_credential = job[:credentials][required_credential]
+          if val_for_credential.nil? or val_for_credential.empty?
+            raise BadConfigurationException.new("To use #{cloud}, " +
+              "#{required_credential} must be specified.")
+          end
+        }
+      else
+        raise BadConfigurationException.new("#{cloud} was specified as in " +
+          ":clouds_to_use, which is not a supported cloud.")
+      end
+    }
+  end
+
+
+  def self.ensure_credentials_are_in_correct_format(job)
+    if job[:credentials].nil?
+      raise BadConfigurationException.new("No credentials were specified.")
+    end
+
+    if job[:credentials].class != Hash
+      raise BadConfigurationException.new("Credentials given were not a " +
+        "Hash, but were a #{job[:credentials].class}")
+    end
+  end
+
+
+  # Searches the caller's environment variables, and adds any that could
+  # be used in this Exodus job. Only takes in credentials from the
+  # environment if the job does not specify it.
+  def self.propogate_credentials_from_environment(job)
+    CLOUD_CREDENTIALS.each { |cloud_name, credential_list|
+      credential_list.each { |cred|
+        if job[:credentials][cred].nil? and !ENV[cred.to_s].nil?
+          job[:credentials][cred] = ENV[cred.to_s]
+        end
+      }
+    }
+  end
+
+
+  def self.validate_optimize_for_param(job)
+    if job[:optimize_for].nil?
+      raise BadConfigurationException.new(":optimize_for needs to be " +
+        "specified when running Exodus jobs")
+    end
+
+    if !OPTIMIZE_FOR_CHOICES.include?(job[:optimize_for])
+      raise BadConfigurationException.new("The value given for " +
+        ":optimize_for was not an acceptable value. Acceptable values are: " +
+        "#{OPTIMIZE_FOR_CHOICES.join(', ')}")
+    end
+  end
+
+
+  def self.validate_files_argv_executable(job)
+    [:code, :argv, :executable].each { |param|
+      if job[param].nil?
+        raise BadConfigurationException.new("#{param} was not specified")
+      end
+    }
+  end
+
+
+  def self.get_profiling_info(job)
+    key = self.get_key_from_job_data(job)
+    neptune_manager = BabelHelper.get_neptune_manager_client(job)
+    return neptune_manager.get_profiling_info(key)
+  end
+
+
+  # TODO(cgb): what is a job's key?
+  def self.get_key_from_job_data(job)
+    return job[:code]
+  end
+
+  
+  def self.get_clouds_to_run_task_on(job, profiling_info)
+    optimize_for = job[:optimize_for]
+    if optimize_for == :performance or optimize_for == :cost
+      return self.get_minimum_val_in_data(job, profiling_info)
+    else
+      return self.find_optimal_cloud_for_task(job, profiling_info)
+    end
+  end
+
+
+  def self.get_minimum_val_in_data(job, profiling_info)
+    min_cloud = nil
+    min_val = 1_000_000  # infinity
+    optimize_for = job[:optimize_for].to_s
+
+    clouds_to_run_on = []
+    job[:clouds_to_use].each { |cloud|
+      # If we have no information on this cloud, then add it to the list
+      # of clouds we should run the task on, since it could potentially be
+      # lower than the minimum in the data we've seen so far.
+      if profiling_info[cloud.to_s].nil?
+        clouds_to_run_on << cloud
+        next
+      end
+
+      val = self.average(profiling_info[cloud.to_s][optimize_for])
+      if val < min_val
+        min_cloud = cloud
+        min_val = val
+      end
+    }
+
+    if !min_cloud.nil?
+      clouds_to_run_on << min_cloud
+    end
+
+    return clouds_to_run_on
+  end
+
+
+  # Given an Array of values, calculates and returns their average.
+  def self.average(vals)
+    sum = vals.reduce(0.0) { |running_total, val|
+      running_total + val
+    }
+
+    return sum / vals.length
+  end
+
+
+  def self.find_optimal_cloud_for_task(job, profiling_info)
+    raise NotImplementedError
+  end
+
+
+  def self.generate_babel_tasks(job, clouds_to_run_task_on)
+    tasks = []
+
+    clouds_to_run_task_on.each { |cloud|
+      task = { :type => "babel",
+        :code => job[:code],
+        :argv => job[:argv],
+        :executable => job[:executable],
+        :is_remote => false,
+        :run_local => false
+      }
+
+      CLOUD_CREDENTIALS[cloud].each { |credential|
+        task[credential] = job[:credentials][credential]
+      }
+
+      task.merge!(CLOUD_BABEL_PARAMS[cloud])
+      tasks << task
+    }
+
+    return tasks
+  end
+
+
+  def self.run_job(tasks_to_run)
+    return babel(tasks_to_run)
+  end
+
+
+end

data/lib/exodus_task_info.rb

@@ -0,0 +1,36 @@
+# Programmer: Chris Bunch
+
+
+require 'task_info'
+
+
+class ExodusTaskInfo
+
+  
+  def initialize(dispatched_babel_tasks)
+    @babel_tasks = dispatched_babel_tasks
+  end
+
+  
+  def to_s
+    method_missing(:to_s)
+  end
+
+
+  def method_missing(id, *args, &block)
+    loop {
+      @babel_tasks.each_with_index { |task, i|
+      begin
+        Timeout::timeout(2) {
+          result = task.send(id, *args, &block)
+          return result
+        }
+      rescue Timeout::Error
+        next
+      end
+      }
+    }
+  end
+
+
+end

data/lib/neptune.rb

@@ -1,9 +1,9 @@
 #!/usr/bin/ruby
 # Programmer: Chris Bunch (cgb@cs.ucsb.edu)
 
-require 'app_controller_client'
 require 'common_functions'
 require 'custom_exceptions'
+require 'neptune_manager_client'
 
 
 # Setting verbose to nil here suppresses the otherwise
@@ -13,7 +13,7 @@ $VERBOSE = nil
 
 
 # A list of all the Neptune job types that we support
-ALLOWED_JOB_TYPES = %w{acl cicero compile erlang mpi input output ssa babel upc x10}
+ALLOWED_JOB_TYPES = %w{acl cicero compile erlang mpi input output ssa babel upc x10 mapreduce}
 
 
 # The string to display for disallowed job types.
@@ -32,7 +32,7 @@ NO_OUTPUT_NEEDED = ["input"]
 
 # A list of storage mechanisms that we can use to store and retrieve
 # data to for Neptune jobs.
-ALLOWED_STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"]
+ALLOWED_STORAGE_TYPES = %w{appdb gstorage s3 walrus waz-storage}
 
 
 # A list of jobs that require some kind of work to be done before
@@ -55,23 +55,50 @@ public
 # the user can request to run a job, retrieve a job's output, or modify the 
 # access policy (ACL) for the output of a job. By default, job data is private,
 # but a Neptune job can be used to set it to public later (and vice-versa).
-def neptune(params)
+def neptune(jobs)
   # Kernel.puts "Received a request to run a job."
   # Kernel.puts params[:type]
+  if jobs.class == Hash
+    jobs = [jobs]
+  end
 
-  job_data = NeptuneHelper.get_job_data(params)
-  NeptuneHelper.validate_storage_params(job_data)
-  # Kernel.puts "job data = #{job_data.inspect}"
-  keyname = job_data["@keyname"]
+  job_data_list = []
+  shadow_ip = nil
+  ssh_args = ""
+  secret = ""
+  controller = nil
 
-  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
-  secret = CommonFunctions.get_secret_key(keyname)
-  ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
-  ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "
+  jobs.each { |params|
+    job_data = NeptuneHelper.get_job_data(params)
+    NeptuneHelper.validate_storage_params(job_data)
+    # Kernel.puts "job data = #{job_data.inspect}"
+    keyname = job_data["@keyname"]
 
-  controller = AppControllerClient.new(shadow_ip, secret)
-  NeptuneHelper.do_preprocessing(job_data, controller) 
-  return NeptuneHelper.run_job(job_data, ssh_args, shadow_ip, secret)
+    shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+    secret = CommonFunctions.get_secret_key(keyname)
+    ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
+    ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "
+
+    controller = NeptuneManagerClient.new(shadow_ip, secret)
+    NeptuneHelper.do_preprocessing(job_data, controller)
+    job_data_list << job_data
+  }
+
+  if job_data_list.length == 1
+    return NeptuneHelper.run_job(job_data_list[0], ssh_args, shadow_ip, 
+      secret)
+  else  # right now we only support batch run_job operations
+    msg = controller.start_neptune_job(job_data_list)
+    result = {}
+    result[:msg] = msg
+    if result[:msg] =~ /job is now running\Z/
+      result[:result] = :success
+    else
+      result[:result] = :failure
+    end
+
+    return result
+  end
 end
 
 
@@ -145,6 +172,9 @@ module NeptuneHelper
   def self.preprocess_mpi(job_data, controller)
     self.require_param("@nodes_to_use", job_data)
     self.require_param("@procs_to_use", job_data)
+    self.require_param("@output", job_data)
+    self.require_param("@error", job_data)
+    self.require_param("@metadata", job_data)
 
     if job_data["@procs_to_use"]
       p = job_data["@procs_to_use"]
@@ -200,7 +230,7 @@ module NeptuneHelper
   end
 
 
-  # This helper method asks the AppController if the named file exists,
+  # This helper method asks the NeptuneManager if the named file exists,
   # and if it does not, throws an exception.
   def self.require_file_to_exist(file, job_data, controller)
     if controller.does_file_exist?(file, job_data)
@@ -232,6 +262,8 @@ module NeptuneHelper
     self.require_param("@code", job_data)
     self.require_param("@engine", job_data)
     self.require_param("@output", job_data)
+    self.require_param("@error", job_data)
+    self.require_param("@metadata", job_data)
 
     # For most code types, the file's name given is the thing to exec.
     # For Java, the actual file to search for is whatever the user gives
@@ -243,6 +275,8 @@ module NeptuneHelper
 
     self.require_file_to_exist(code_file_name, job_data, controller)
     self.require_file_to_not_exist(job_data["@output"], job_data, controller)
+    self.require_file_to_not_exist(job_data["@error"], job_data, controller)
+    self.require_file_to_not_exist(job_data["@metadata"], job_data, controller)
 
     if job_data["@argv"]
       argv = job_data["@argv"]
@@ -275,7 +309,7 @@ module NeptuneHelper
 
   # This method takes in a hash in the format that users write neptune/babel
   # jobs in {:a => "b"} and converts it to the legacy format that Neptune
-  # used to use {"@a" => "b"}, and is understood by the AppController.
+  # used to use {"@a" => "b"}, and is understood by the NeptuneManager.
   def self.get_job_data(params)
     job_data = {}
     params.each { |k, v|
@@ -500,7 +534,7 @@ module NeptuneHelper
   # This method actually runs the Neptune job, given information about the job
   # as well as information about the node to send the request to.
   def self.run_job(job_data, ssh_args, shadow_ip, secret)
-    controller = AppControllerClient.new(shadow_ip, secret)
+    controller = NeptuneManagerClient.new(shadow_ip, secret)
 
     # TODO - right now the job is assumed to succeed in many cases
     # need to investigate the various failure scenarios