neptune 0.2.1 → 0.2.2

data/lib/{app_controller_client.rb → neptune_manager_client.rb}

@@ -10,47 +10,52 @@ require 'timeout'
 # endlessly timeout and retry, so as a hack, just don't let them timeout.
 # The next version should replace this and properly timeout and not use
 # long calls unless necessary.
-NO_TIMEOUT = -1
+NO_TIMEOUT = 100000
 
 
 # A client that uses SOAP messages to communicate with the underlying cloud
 # platform (here, AppScale). This client is similar to that used in the AppScale
 # Tools, but with non-Neptune SOAP calls removed.
-class AppControllerClient
+class NeptuneManagerClient
+
+  
+  # The port that the Neptune Manager runs on, by default.
+  SERVER_PORT = 17445
 
 
-  # The SOAP client that we use to communicate with the AppController.
+  # The SOAP client that we use to communicate with the NeptuneManager.
   attr_accessor :conn
   
 
-  # The IP address of the AppController that we will be connecting to.
+  # The IP address of the NeptuneManager that we will be connecting to.
   attr_accessor :ip
   
 
   # The secret string that is used to authenticate this client with
-  # AppControllers. It is initially generated by appscale-run-instances and can
+  # NeptuneManagers. It is initially generated by appscale-run-instances and can
   # be found on the machine that ran that tool, or on any AppScale machine.
   attr_accessor :secret
   
 
   # A constructor that requires both the IP address of the machine to communicate
   # with as well as the secret (string) needed to perform communication.
-  # AppControllers will reject SOAP calls if this secret (basically a password)
+  # NeptuneManagers will reject SOAP calls if this secret (basically a password)
   # is not present - it can be found in the user's .appscale directory, and a
   # helper method is usually present to fetch this for us.
   def initialize(ip, secret)
     @ip = ip
     @secret = secret
     
-    @conn = SOAP::RPC::Driver.new("https://#{@ip}:17443")
-    @conn.add_method("neptune_start_job", "job_data", "secret")
-    @conn.add_method("neptune_put_input", "job_data", "secret")
-    @conn.add_method("neptune_get_output", "job_data", "secret")
-    @conn.add_method("neptune_get_acl", "job_data", "secret")
-    @conn.add_method("neptune_set_acl", "job_data", "secret")
-    @conn.add_method("neptune_compile_code", "job_data", "secret")
-    @conn.add_method("neptune_get_supported_babel_engines", "job_data", "secret")
-    @conn.add_method("neptune_does_file_exist", "file", "job_data", "secret")
+    @conn = SOAP::RPC::Driver.new("https://#{@ip}:#{SERVER_PORT}")
+    @conn.add_method("start_job", "jobs", "secret")
+    @conn.add_method("put_input", "job_data", "secret")
+    @conn.add_method("get_output", "job_data", "secret")
+    @conn.add_method("get_acl", "job_data", "secret")
+    @conn.add_method("set_acl", "job_data", "secret")
+    @conn.add_method("compile_code", "job_data", "secret")
+    @conn.add_method("get_supported_babel_engines", "job_data", "secret")
+    @conn.add_method("does_file_exist", "file", "job_data", "secret")
+    @conn.add_method("get_profiling_info", "key", "secret")
   end
 
 
@@ -76,7 +81,7 @@ class AppControllerClient
       if retry_on_except
         retry
       else
-        raise AppControllerException.new("Connection was refused. Is the AppController running?")
+        raise NeptuneManagerException.new("Connection was refused. Is the NeptuneManager running?")
       end
     rescue OpenSSL::SSL::SSLError, NotImplementedError, Timeout::Error
       retry
@@ -84,7 +89,7 @@ class AppControllerClient
       if retry_on_except
         retry
       else
-        raise AppControllerException.new("We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.")
+        raise NeptuneManagerException.new("We saw an unexpected error of the type #{except.class} talking to #{@ip} with the following message:\n#{except}.")
       end
     end
   end
@@ -94,16 +99,16 @@ class AppControllerClient
   # or MapReduce), or a scaling job (e.g., for AppScale itself). This method
   # should not be used for retrieving the output of a job or getting / setting
   # output ACLs, but just for starting new HPC / scaling jobs. This method
-  # takes a hash containing the parameters of the job to run, and can raise AppControllerException.new if
-  # the AppController it calls returns an error (e.g., if a bad secret is used
+  # takes a hash containing the parameters of the job to run, and can raise NeptuneManagerException.new if
+  # the NeptuneManager it calls returns an error (e.g., if a bad secret is used
   # or the machine isn't running). Otherwise, the return value of this method
-  # is the result returned from the AppController.
+  # is the result returned from the NeptuneManager.
   def start_neptune_job(job_data)
     result = ""
     make_call(NO_TIMEOUT, false) { 
-      result = conn.neptune_start_job(job_data, @secret)
+      result = conn.start_job(job_data, @secret)
     }  
-    raise AppControllerException.new(result) if result =~ /Error:/
+    raise NeptuneManagerException.new(result) if result =~ /Error:/
     return result
   end
 
@@ -116,9 +121,9 @@ class AppControllerClient
   def put_input(job_data)
     result = ""
     make_call(NO_TIMEOUT, false) {
-      result = conn.neptune_put_input(job_data, @secret)
+      result = conn.put_input(job_data, @secret)
     }  
-    raise AppControllerException.new(result) if result =~ /Error:/
+    raise NeptuneManagerException.new(result) if result =~ /Error:/
     return result
   end
 
@@ -132,13 +137,13 @@ class AppControllerClient
   # for non-trivial output jobs, the next version of Neptune will add an
   # additional call to directly copy the output to a file on the local
   # filesystem. See start_neptune_job for conditions by which this method
-  # can raise AppControllerException.new as well as the input format used for job_data.
+  # can raise NeptuneManagerException.new as well as the input format used for job_data.
   def get_output(job_data)
     result = ""
     make_call(NO_TIMEOUT, false) { 
-      result = conn.neptune_get_output(job_data, @secret)
+      result = conn.get_output(job_data, @secret)
     }  
-    raise AppControllerException.new(result) if result =~ /Error:/
+    raise NeptuneManagerException.new(result) if result =~ /Error:/
     return result
   end
 
@@ -151,9 +156,9 @@ class AppControllerClient
   def get_acl(job_data)
     result = ""
     make_call(NO_TIMEOUT, false) { 
-      result = conn.neptune_get_acl(job_data, @secret)
+      result = conn.get_acl(job_data, @secret)
     }  
-    raise AppControllerException.new(result) if result =~ /Error:/
+    raise NeptuneManagerException.new(result) if result =~ /Error:/
     return result
   end
 
@@ -166,45 +171,56 @@ class AppControllerClient
   def set_acl(job_data)
     result = ""
     make_call(NO_TIMEOUT, false) { 
-      result = conn.neptune_set_acl(job_data, @secret)
+      result = conn.set_acl(job_data, @secret)
     }  
-    raise AppControllerException.new(result) if result =~ /Error:/
+    raise NeptuneManagerException.new(result) if result =~ /Error:/
     return result
   end
 
 
-  # Instructs the AppController to fetch the code specified and compile it.
+  # Instructs the NeptuneManager to fetch the code specified and compile it.
   # The result should then be placed in a location specified in the job data.
   def compile_code(job_data)
     result = ""
     make_call(NO_TIMEOUT, false) { 
-      result = conn.neptune_compile_code(job_data, @secret)
+      result = conn.compile_code(job_data, @secret)
     }  
-    raise AppControllerException.new(result) if result =~ /Error:/
+    raise NeptuneManagerException.new(result) if result =~ /Error:/
     return result
   end
 
 
-  # Asks the AppController for a list of all the Babel engines (each of which
+  # Asks the NeptuneManager for a list of all the Babel engines (each of which
   # is a queue to store jobs and something that executes tasks) that are
   # supported for the given credentials.
   def get_supported_babel_engines(job_data)
     result = []
     make_call(NO_TIMEOUT, false) {
-      result = conn.neptune_get_supported_babel_engines(job_data, @secret)
+      result = conn.get_supported_babel_engines(job_data, @secret)
     }
     return result
   end
 
 
-  # Asks the AppController to see if the given file exists in the remote
+  # Asks the NeptuneManager to see if the given file exists in the remote
   # datastore. If extra credentials are needed for this operation, they are
   # searched for within the job data.
   def does_file_exist?(file, job_data)
     result = false
     make_call(NO_TIMEOUT, false) {
-      result = conn.neptune_does_file_exist(file, job_data, @secret)
+      result = conn.does_file_exist(file, job_data, @secret)
+    }
+    return result
+  end
+
+
+  def get_profiling_info(key)
+    result = {}
+    make_call(NO_TIMEOUT, false) {
+      result = conn.get_profiling_info(key, @secret)
     }
     return result
   end
+
+
 end

data/lib/task_info.rb

@@ -0,0 +1,155 @@
+# Programmer: Chris Bunch
+
+
+# Imports from Ruby's stdlib
+require 'thread'  # needed for Mutex
+
+
+# Imports for RubyGems
+require 'rubygems'
+require 'json'
+
+
+# Imports for other Neptune libraries
+require 'babel'
+require 'custom_exceptions'
+
+
+# TaskInfo represents the result of a babel call, an object with all the
+# information that the user would be interested in relating to their task.
+# At the simplest level, this is just the output of their job, but it also
+# can includes profiling information (e.g., performance and cost), as well
+# as information that may help with debugging (e.g. info about the environment
+# we executed the task in).
+class TaskInfo
+
+
+  # A Hash consisting of the parameters that the user passed to babel().
+  attr_accessor :job_data
+
+
+  # Creates a new TaskInfo object, storing the parameters the user gave us to
+  # invoke the job for later use. The user can give us a Hash containing the
+  # parameters that the job was started with, or a String that is the
+  # JSON-dumped version of that data (also obtainable from TaskInfo objects
+  # via to_json).
+  def initialize(job_data)
+    if job_data.class == String
+      begin
+        job_data = JSON.load(job_data)
+      rescue JSON::ParserError
+        raise BadConfigurationException.new("job data not JSONable")
+      end
+    end
+
+    if job_data.class != Hash
+      raise BadConfigurationException.new("job data not a Hash")
+    end
+    @job_data = job_data
+
+    # To prevent us from repeatedly grabbing (potentially) large files over the
+    # network repeatedly, we keep a local, cached copy of the task's standard
+    # output, error, and metadata - initially empty, but pulled in the first
+    # time that the user asks for it. Since we expose this functionality through
+    # the accessor methods below, we should not use attr_accessor or attr_reader
+    # to directly expose this variables.
+    @output = nil
+    @error = nil
+    @metadata = nil
+
+    # To prevent concurrent threads from pulling in output multiple times, we
+    # guard access to remotely grabbing output/error/metadata with this
+    # lock.
+    @lock = Mutex.new
+  end
+
+
+  # Returns a string with the standard output produced by this Babel task. If
+  # the task has not yet completed, this call blocks until it completes.
+  def stdout
+    if @output.nil?
+      @lock.synchronize {
+        @output = BabelHelper.wait_and_get_output(@job_data,
+          @job_data['@output'])
+      }
+    end
+
+    return @output
+  end
+
+
+  # Returns a string with the standard error produced by this Babel task. While
+  # all jobs should produce standard output, they may not produce standard
+  # error, so it is reasonable that this could return an empty string to the
+  # user.
+  def stderr
+    if @error.nil?
+      @lock.synchronize {
+        @error = BabelHelper.wait_and_get_output(@job_data, @job_data['@error'])
+      }
+    end
+
+    return @error
+  end
+
+
+  # An alias for stdout.
+  def to_s
+    return stdout
+  end
+
+
+  # A common operation that users may perform is asking if the task executed
+  # successfully, indicated by a return value of zero. This method provides
+  # a quick alias for that functionality.
+  def success?
+    return return_value.zero?
+  end
+
+
+  # Converts this object to JSON, so that it can be written to disk or
+  # passed over the network. Since our stdout/stderr/metadata objects
+  # are all locally cached, we don't need to write them (and thus can
+  # potentially save a lot of space).
+  def to_json
+    return JSON.dump(@job_data)
+  end
+
+
+  private
+
+
+  # We store all the task information that isn't standard out or standard err
+  # as a JSON-encoded Hash in a metadata file. This function provides easy
+  # access to that hash, retrieving it remotely if needed. It's private since
+  # we intend for other methods in this class to call it, and not the user
+  # directly.
+  def metadata
+    if @metadata.nil?
+      @lock.synchronize {
+        info = BabelHelper.wait_and_get_output(@job_data, 
+          @job_data['@metadata'])
+        @metadata = JSON.load(info)
+      }
+    end
+
+    return @metadata
+  end
+
+
+  # We would like to be able to directly call .name on anything in the metadata
+  # hash for a task. One way to avoid having to add all of these method calls
+  # ourselves and keep it in sync with whatever Neptune over AppScale offers
+  # is just to use method_missing and automatically respond to anything that
+  # is a key in the metadata hash.
+  def method_missing(id, *args, &block)
+    methods_available = metadata()
+    if methods_available[id.to_s].nil?
+      super
+    else
+      return methods_available[id.to_s]
+    end
+  end
+
+
+end

data/test/{unit/test_babel.rb → test_babel.rb}

@@ -1,9 +1,12 @@
 # Programmer: Chris Bunch (cgb@cs.ucsb.edu)
 
-$:.unshift File.join(File.dirname(__FILE__), "..", "..", "lib")
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 require 'babel'
 
-require 'test/unit'
+
+require 'rubygems'
+require 'flexmock/test_unit'
 
 
 class TestBabel < Test::Unit::TestCase
@@ -19,44 +22,102 @@ class TestBabel < Test::Unit::TestCase
       :EC2_ACCESS_KEY => "boo",
       :EC2_SECRET_KEY => "baz",
       :S3_URL => "http://baz.com",
-      :keyname => keyname
+      :is_remote => true,
+      :keyname => keyname,
+      :metadata_info => {'time_to_store_inputs' => 0.0}
     }
 
     job_data = {}
     params.each { |k, v|
       job_data["@#{k}"] = v
     }
-    job_data["@is_remote"] = true
 
     output = "/bucket/babel/temp-0123456789"
     job_data["@output"] = output
+    job_data_no_err = job_data.dup
+
+    error = "/bucket/babel/temp-1111111111"
+    job_data["@error"] = error
+    job_data_no_metadata = job_data.dup
+
+    metadata = "/bucket/babel/temp-2222222222"
+    job_data["@metadata"] = metadata
 
     run_job_data = job_data.dup
     run_job_data["@engine"] = "executor-sqs"
     run_job_data["@run_local"] = true
+    run_job_data["@failed_attempts"] = 0
+
+    run_job_data_second_try = run_job_data.dup
+    run_job_data["@failed_attempts"] = 1
 
     output_job_data = job_data.dup
     output_job_data["@type"] = "output"
 
+    error_job_data = output_job_data.dup
+    error_job_data['@output'] = error
+
+    metadata_job_data = output_job_data.dup
+    metadata_job_data['@output'] = metadata
+    json_metadata = JSON.dump({'command' => 'ls /home/baz', 'return_value' => 0})
+
     kernel = flexmock(Kernel)
     kernel.should_receive(:puts).and_return()
-    kernel.should_receive(:rand).and_return(0,1,2,3,4,5,6,7,8,9)
+    kernel.should_receive(:rand).and_return(0,1,2,3,4,5,6,7,8,9,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2)
     kernel.should_receive(:sleep).and_return()
 
-    flexmock(AppControllerClient).new_instances { |instance|
-      instance.should_receive(:does_file_exist?).with(file, job_data).and_return(true)
-      instance.should_receive(:does_file_exist?).with(output, job_data).and_return(false)
-
-      instance.should_receive(:start_neptune_job).with(run_job_data).and_return("MPI job is now running")
-
-      instance.should_receive(:get_output).with(output_job_data).and_return("output")
+    flexmock(NeptuneManagerClient).new_instances { |instance|
+      instance.should_receive(:does_file_exist?).with(file, job_data).
+        and_return(true)
+      instance.should_receive(:does_file_exist?).with(output, job_data_no_err).
+        and_return(false)
+      instance.should_receive(:does_file_exist?).
+        with(error, job_data_no_metadata).and_return(false)
+      instance.should_receive(:does_file_exist?).with(metadata, job_data).
+        and_return(false)
+
+      # So the first time we start the job, let's say that it failed, so that
+      # we can make sure that the caller properly catches this and tries again.
+      instance.should_receive(:start_neptune_job).with(run_job_data).
+        and_return("error")
+      instance.should_receive(:start_neptune_job).with(run_job_data_second_try).
+        and_return("MPI job is now running")
+
+      instance.should_receive(:get_output).with(output_job_data).
+        and_return("output")
+      instance.should_receive(:get_output).with(error_job_data).
+        and_return("error")
+      instance.should_receive(:get_output).with(metadata_job_data).
+        and_return(json_metadata)
     }
 
     commonfunctions = flexmock(CommonFunctions)
-    commonfunctions.should_receive(:get_from_yaml).with(keyname, :shadow).and_return("127.0.0.1")
-    commonfunctions.should_receive(:get_secret_key).with(keyname).and_return("secret")
-
-    assert_equal("output", babel(params))
+    commonfunctions.should_receive(:get_from_yaml).with(keyname, :shadow).
+      and_return("127.0.0.1")
+    commonfunctions.should_receive(:get_secret_key).with(keyname).
+      and_return("secret")
+
+    # Calling either to_s or stdout will return the standard out that the
+    # program produced.
+    expected = "output"
+    actual = babel(params)
+    assert_equal(expected, actual.to_s)
+    assert_equal(expected, actual.stdout)
+
+    # Calling stderr returns the standard error that we are expecting.
+    assert_equal("error", actual.stderr)
+
+    # Calling command returns the command that was remotely exec'ed, hidden in
+    # the metadata.
+    assert_equal("ls /home/baz", actual.command)
+
+    # Calling success? returns true if the command's return value is zero,
+    # also hidden in the metadata
+    assert_equal(true, actual.success?)
+    
+    # We're using method_missing under the hood, so make sure that a method call
+    # that doesn't exist fails accordingly
+    assert_raise(NoMethodError) { actual.baz }
   end
 
   def test_bad_babel_params
@@ -104,7 +165,8 @@ class TestBabel < Test::Unit::TestCase
 
     # Finally, if we run a job and specify remote code, that should be used
     # as the bucket.
-    job_data_remote_code = {"@code" => "/baz/boo/code.baz", "@storage" => "s3"}
+    job_data_remote_code = {"@code" => "/baz/boo/code.baz", "@storage" => "s3",
+      "@is_remote" => true}
     expected_remote = "/baz/babel/temp-10"
 
     actual_remote = BabelHelper.generate_output_location(job_data_remote_code)
@@ -141,8 +203,8 @@ class TestBabel < Test::Unit::TestCase
     actual_2 = BabelHelper.put_inputs(job_data)
     assert_equal(job_data, actual_2)
 
-    # If we specify inputs on the file system, they should be uploaded and replaced
-    # with remote file locations
+    # If we specify inputs on the file system, they should be uploaded and 
+    # replaced with remote file locations
     neptune_params = {
       :type => "input",
       :local => "/baz",
@@ -155,26 +217,31 @@ class TestBabel < Test::Unit::TestCase
     kernel = flexmock(Kernel)
     kernel.should_receive(:neptune).with(neptune_params)
 
+    time = flexmock(Time)
+    time.should_receive(:now).and_return(0.0)
+
     job_data["@argv"] = ["boo", "/baz", "gbaz"]
     expected = job_data.dup
     expected["@argv"] = ["boo", "/remote/babel/baz", "gbaz"]
+    expected["@metadata_info"] = {"time_to_store_inputs" => 0.0}
     actual_3 = BabelHelper.put_inputs(job_data.dup)
     assert_equal(expected, actual_3)
   end
 
   def test_run_babel_job
     # Running a job with no @type specified means it should be a Babel job
-    job_data = {
+    job_data = [{
       "@code" => "/baz/boo/code.baz",
       "@argv" => ["boo", "/remote/babel/baz", "gbaz"]
-    }
+    }]
 
     neptune_params = {
       :type => "babel",
       :code => "/baz/boo/code.baz",
       :argv => ["boo", "/remote/babel/baz", "gbaz"],
       :run_local => true,
-      :engine => "executor-sqs"
+      :engine => "executor-sqs",
+      :failed_attempts => 0
     }
 
     result = { :result => :success }
@@ -188,18 +255,19 @@ class TestBabel < Test::Unit::TestCase
 
   def test_run_mpi_job
     # Running a job with @type specified should preserve the job type
-    job_data = {
+    job_data = [{
       "@type" => "mpi",
       "@code" => "/baz/boo/code.baz",
       "@argv" => ["boo", "/remote/babel/baz", "gbaz"]
-    }
+    }]
 
     neptune_params = {
       :type => "mpi",
       :code => "/baz/boo/code.baz",
       :argv => ["boo", "/remote/babel/baz", "gbaz"],
       :run_local => true,
-      :engine => "executor-sqs"
+      :engine => "executor-sqs",
+      :failed_attempts => 0
     }
 
     result = { :result => :success }
@@ -228,7 +296,74 @@ class TestBabel < Test::Unit::TestCase
     kernel.should_receive(:sleep).and_return()
 
     expected = "output goes here"
-    actual = BabelHelper.wait_and_get_output(job_data)
+    actual = BabelHelper.wait_and_get_output(job_data, job_data['@output'])
     assert_equal(expected, actual)
   end
+
+  def test_batch_tasks_operation
+    # if we give babel an array of hashes, it should should give us back
+    # task information for each of the jobs we asked it to run
+    # essentially this saves us the overhead of the repeated SOAP calls
+    # to AppScale
+
+    neptune_params = {
+      :type => "babel",
+      :code => "/baz/boo/code.baz",
+      :argv => ["boo", "/remote/babel/baz", "gbaz"],
+      :output => "/baz/output",
+      :error => "/baz/error",
+      :metadata => "/baz/metadata",
+      :run_local => true,
+      :engine => "executor-sqs",
+      :failed_attempts => 0,
+      :metadata_info => {'time_to_store_inputs' => 0.0},
+      :storage => "appdb",
+      :keyname => "appscale"
+    }
+    tasks = [neptune_params, neptune_params]
+
+    job_data = {}
+    neptune_params.each { |k, v|
+      job_data["@#{k}"] = v
+    }
+
+    # mocks - mock out most of the babel stuff, since we just want to verify
+    # the semantics of passing in an array of hashes instead of just a hash
+    babelhelper = flexmock(BabelHelper)
+    babelhelper.should_receive(:check_output_files).and_return()
+    babelhelper.should_receive(:validate_inputs).and_return()
+    babelhelper.should_receive(:put_code).and_return()
+    babelhelper.should_receive(:put_inputs).and_return()
+
+    # mocks for neptune
+    neptunehelper = flexmock(NeptuneHelper)
+    neptunehelper.should_receive(:require_file_to_exist).and_return()
+    neptunehelper.should_receive(:require_file_to_not_exist).and_return()
+
+    # finally, mock out the libraries that neptune uses
+    commonfunctions = flexmock(CommonFunctions)
+    commonfunctions.should_receive(:get_from_yaml).with("appscale", :shadow).
+      and_return("public_ip")
+    commonfunctions.should_receive(:get_from_yaml).with("appscale", :secret, 
+      true).and_return("secret")
+
+    appcontroller = flexmock('appcontroller')
+    appcontroller.should_receive(:get_supported_babel_engines).with(job_data).
+      and_return(["executor-sqs"])
+    appcontroller.should_receive(:start_neptune_job).
+      and_return("babel job is now running")
+    flexmock(NeptuneManagerClient).should_receive(:new).and_return(appcontroller)
+
+    flexmock(TaskInfo).new_instances { |instance|
+      instance.should_receive(:stdout).and_return("output")
+    }
+
+    expected = ["output", "output"]
+    actual = []
+    babel(tasks).each { |task|
+      actual << task.stdout
+    }
+    assert_equal(expected, actual)
+  end
+
 end