neptune 0.2.0 → 0.2.1

data/lib/custom_exceptions.rb

@@ -1,10 +1,18 @@
 # Programmer: Chris Bunch
 
+# A special class of exceptions that are thrown whenever the AppController
+# experiences an unexpected result.
 class AppControllerException < Exception
 end
 
+
+# A class of exceptions that are thrown when the user tries to run a Neptune
+# job but fails to give us the correct parameters to do so.
 class BadConfigurationException < Exception
 end
 
+
+# An exception that is thrown whenever the user specifies a file to use
+# that does not exist.
 class FileNotFoundException < Exception
 end

data/lib/neptune.rb

@@ -5,54 +5,81 @@ require 'app_controller_client'
 require 'common_functions'
 require 'custom_exceptions'
 
+
 # Setting verbose to nil here suppresses the otherwise
 # excessive SSL cert warning messages that will pollute
 # stderr and worry users unnecessarily.
 $VERBOSE = nil
 
-#MPI_RUN_JOB_REQUIRED = %w{ input output code filesystem }
-#MPI_REQUIRED = %w{ output }
-#X10_RUN_JOB_REQUIRED = %w{ input output code filesystem }
-#X10_REQUIRED = %w{ output }
-#DFSP_RUN_JOB_REQUIRED = %w{ output simulations }
-#DFSP_REQUIRED = %w{ output }
-#CEWSSA_RUN_JOB_REQUIRED = %w{ output simulations }
-#CEWSSA_REQUIRED = %w{ output }
-#MR_RUN_JOB_REQUIRED = %w{ }
-#MR_REQUIRED = %w{ output }
 
 # A list of all the Neptune job types that we support
 ALLOWED_JOB_TYPES = %w{acl cicero compile erlang mpi input output ssa babel upc x10}
 
+
 # The string to display for disallowed job types.
 JOB_TYPE_NOT_ALLOWED = "The job type you specified is not supported."
 
+
 # A list of Neptune jobs that do not require nodes to be spawned
 # up for computation
 NO_NODES_NEEDED = ["acl", "input", "output", "compile"]
 
+
 # A list of Neptune jobs that do not require the output to be
 # specified beforehand
 NO_OUTPUT_NEEDED = ["input"]
 
+
 # A list of storage mechanisms that we can use to store and retrieve
 # data to for Neptune jobs.
 ALLOWED_STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"]
 
+
 # A list of jobs that require some kind of work to be done before
 # the actual computation can be performed.
 NEED_PREPROCESSING = ["babel", "compile", "erlang", "mpi", "ssa"]
 
-# A set of methods and constants that we've monkey-patched to enable Neptune
-# support. In the future, it is likely that the only exposed / monkey-patched
-# method should be job, while the others could probably be folded into either
-# a Neptune-specific class or into CommonFunctions.
-# TODO(cbunch): This doesn't look like it does anything - run the integration
-# test and confirm one way or the other.
+
+# Since we're monkeypatching Object to add neptune() and babel(), a short
+# blurb is necessary here to make rdoc happy.
 class Object
 end
 
+
+# Make neptune() public so that babel() can call it
+public
+
+
+# This method is the heart of Neptune - here, we take blocks of code that the
+# user has written and convert them into HPC job requests. At a high level, 
+# the user can request to run a job, retrieve a job's output, or modify the 
+# access policy (ACL) for the output of a job. By default, job data is private,
+# but a Neptune job can be used to set it to public later (and vice-versa).
+def neptune(params)
+  # Kernel.puts "Received a request to run a job."
+  # Kernel.puts params[:type]
+
+  job_data = NeptuneHelper.get_job_data(params)
+  NeptuneHelper.validate_storage_params(job_data)
+  # Kernel.puts "job data = #{job_data.inspect}"
+  keyname = job_data["@keyname"]
+
+  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+  secret = CommonFunctions.get_secret_key(keyname)
+  ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
+  ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "
+
+  controller = AppControllerClient.new(shadow_ip, secret)
+  NeptuneHelper.do_preprocessing(job_data, controller) 
+  return NeptuneHelper.run_job(job_data, ssh_args, shadow_ip, secret)
+end
+
+
+# NeptuneHelper provides methods that are used by neptune() and babel() to 
+# validate parameters and run the user's job.
 module NeptuneHelper
+
+
   # Certain types of jobs need steps to be taken before they
   # can be started (e.g., copying input data or code over).
   # This method dispatches the right method to use based
@@ -69,6 +96,7 @@ module NeptuneHelper
     send(preprocess, job_data, controller)
   end
 
+
   # This preprocessing method copies over the user's code to the
   # Shadow node so that it can be compiled there. A future version
   # of this method may also copy over libraries as well.
@@ -85,13 +113,16 @@ module NeptuneHelper
 
     ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}"
     remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}' 2>&1"
-    Kernel.puts remove_dir
+    # Kernel.puts remove_dir
     CommonFunctions.shell(remove_dir)
     CommonFunctions.scp_to_shadow(code, dest, keyname, is_dir=true)
 
     job_data["@code"] = dest
   end
 
+
+  # This preprocessing method makes sure that the user's Erlang code exists
+  # and copies it over to the AppScale Shadow node.
   def self.preprocess_erlang(job_data, controller)
     self.require_param("@code", job_data)
 
@@ -106,6 +137,7 @@ module NeptuneHelper
     CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
   end
 
+
   # This preprocessing method verifies that the user specified the number of nodes
   # to use. If they also specified the number of processes to use, we also verify
   # that this value is at least as many as the number of nodes (that is, nodes
@@ -138,6 +170,7 @@ module NeptuneHelper
     return job_data
   end
 
+
   # This preprocessing method verifies that the user specified the number of
   # trajectories to run, via either :trajectories or :simulations. Both should
   # not be specified - only one or the other, and regardless of which they
@@ -157,12 +190,18 @@ module NeptuneHelper
     return job_data
   end
 
+
+  # This helper method aborts if the given parameter is not present in the
+  # job data provided.
   def self.require_param(param, job_data)
     if !job_data[param]
       raise BadConfigurationException.new("#{param} must be specified")
     end
   end
 
+
+  # This helper method asks the AppController if the named file exists,
+  # and if it does not, throws an exception.
   def self.require_file_to_exist(file, job_data, controller)
     if controller.does_file_exist?(file, job_data)
       return
@@ -171,6 +210,9 @@ module NeptuneHelper
     end
   end
 
+
+  # This helper method performs the opposite function of require_file_to_exist,
+  # raising an exception if the named file does exist.
   def self.require_file_to_not_exist(file, job_data, controller)
     begin
       self.require_file_to_exist(file, job_data, controller)
@@ -181,6 +223,7 @@ module NeptuneHelper
     end
   end
 
+
   # This preprocessing method verifies that the user specified code that
   # should be run, where the output should be placed, and an engine to run over.
   # It also verifies that all files to be used are actually reachable.
@@ -229,6 +272,10 @@ module NeptuneHelper
     end
   end
 
+
+  # This method takes in a hash in the format that users write neptune/babel
+  # jobs in {:a => "b"} and converts it to the legacy format that Neptune
+  # used to use {"@a" => "b"}, and is understood by the AppController.
   def self.get_job_data(params)
     job_data = {}
     params.each { |k, v|
@@ -277,6 +324,11 @@ module NeptuneHelper
     return job_data
   end
 
+
+  # This method looks through the given job data and makes sure that the correct
+  # parameters are present for the storage mechanism specified. It throws an
+  # exception if there are errors in the job data or if a needed parameter is
+  # missing.
   def self.validate_storage_params(job_data)
     job_data["@storage"] ||= "appdb"
 
@@ -297,10 +349,10 @@ module NeptuneHelper
     if storage == "s3"
       ["EC2_ACCESS_KEY", "EC2_SECRET_KEY", "S3_URL"].each { |item|
         if job_data["@#{item}"]
-          Kernel.puts "Using specified #{item}"
+          # Kernel.puts "Using specified #{item}"
         else
           if ENV[item]
-            Kernel.puts "Using #{item} from environment"
+            # Kernel.puts "Using #{item} from environment"
             job_data["@#{item}"] = ENV[item]
           else
             raise BadConfigurationException.new("When storing data to S3, #{item} must be specified or be in " + 
@@ -313,6 +365,7 @@ module NeptuneHelper
     return job_data
   end
 
+
   # This method takes a file on the local user's computer and stores it remotely
   # via AppScale. It returns a hash map indicating whether or not the job
   # succeeded and if it failed, the reason for it.
@@ -330,11 +383,11 @@ module NeptuneHelper
 
     remote = "/tmp/neptune-input-#{rand(100000)}"
     scp_cmd = "scp -r #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
-    Kernel.puts scp_cmd
+    # Kernel.puts scp_cmd
     CommonFunctions.shell(scp_cmd)
 
     job_data["@local"] = remote
-    Kernel.puts "job data = #{job_data.inspect}"
+    # Kernel.puts "job data = #{job_data.inspect}"
     response = controller.put_input(job_data)
     if response
       return {:result => :success}
@@ -344,24 +397,26 @@ module NeptuneHelper
     end
   end
 
+
   # This method waits for AppScale to finish compiling the user's code, indicated
   # by AppScale copying the finished code to a pre-determined location.
   def self.wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location)
     loop {
       ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2>&1"
-      Kernel.puts ssh_command
+      # Kernel.puts ssh_command
       ssh_result = CommonFunctions.shell(ssh_command)
-      Kernel.puts "result was [#{ssh_result}]"
+      # Kernel.puts "result was [#{ssh_result}]"
       if ssh_result =~ /No such file or directory/
-        Kernel.puts "Still waiting for code to be compiled..."
+        # Kernel.puts "Still waiting for code to be compiled..."
       else
-        Kernel.puts "compilation complete! Copying compiled code to #{copy_to}"
+        # Kernel.puts "compilation complete! Copying compiled code to #{copy_to}"
         return
       end
       sleep(5)
     }
   end
 
+
   # This method sends out a request to compile code, waits for it to finish, and
   # gets the standard out and error returned from the compilation. This method
   # returns a hash containing the standard out, error, and a result that indicates
@@ -374,7 +429,7 @@ module NeptuneHelper
     FileUtils.rm_rf(copy_to)
 
     scp_command = "scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2>&1"
-    Kernel.puts scp_command
+    # Kernel.puts scp_command
     CommonFunctions.shell(scp_command)
 
     code = job_data["@code"]
@@ -383,13 +438,14 @@ module NeptuneHelper
 
     [remote_dir, compiled_location].each { |remote_files|
       ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_files}' 2>&1"
-      Kernel.puts ssh_command
+      # Kernel.puts ssh_command
       CommonFunctions.shell(ssh_command)
     }
 
     return get_std_out_and_err(copy_to)
   end
 
+
   # This method returns a hash containing the standard out and standard error
   # from a completed job, as well as a result field that indicates whether or
   # not the job completed successfully (success = no errors).
@@ -411,9 +467,12 @@ module NeptuneHelper
     return result
   end
 
+
+  # This method uploads a Google App Engine application into AppScale, for use
+  # with Cicero jobs. It requires the AppScale tools to be installed.
   def self.upload_app_for_cicero(job_data)
     if !job_data["@app"]
-      Kernel.puts "No app specified, not uploading..." 
+      # Kernel.puts "No app specified, not uploading..." 
       return
     end
 
@@ -431,12 +490,13 @@ module NeptuneHelper
       upload_app = "appscale-upload-app"
     end
 
-    Kernel.puts "Uploading AppEngine app at #{app_location}"
+    # Kernel.puts "Uploading AppEngine app at #{app_location}"
     upload_command = "#{upload_app} --file #{app_location} --test --keyname #{keyname}"
-    Kernel.puts upload_command
-    Kernel.puts `#{upload_command}`
+    # Kernel.puts upload_command
+    # Kernel.puts `#{upload_command}`
   end
 
+
   # This method actually runs the Neptune job, given information about the job
   # as well as information about the node to send the request to.
   def self.run_job(job_data, ssh_args, shadow_ip, secret)
@@ -473,30 +533,3 @@ module NeptuneHelper
     return result
   end
 end
-
-# Make neptune() public so that babel() can call it
-public
-
-# This method is the heart of Neptune - here, we take blocks of code that the
-# user has written and convert them into HPC job requests. At a high level, 
-# the user can request to run a job, retrieve a job's output, or modify the 
-# access policy (ACL) for the output of a job. By default, job data is private,
-# but a Neptune job can be used to set it to public later (and vice-versa).
-def neptune(params)
-  Kernel.puts "Received a request to run a job."
-  Kernel.puts params[:type]
-
-  job_data = NeptuneHelper.get_job_data(params)
-  NeptuneHelper.validate_storage_params(job_data)
-  Kernel.puts "job data = #{job_data.inspect}"
-  keyname = job_data["@keyname"]
-
-  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
-  secret = CommonFunctions.get_secret_key(keyname)
-  ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
-  ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no "
-
-  controller = AppControllerClient.new(shadow_ip, secret)
-  NeptuneHelper.do_preprocessing(job_data, controller) 
-  return NeptuneHelper.run_job(job_data, ssh_args, shadow_ip, secret)
-end

data/test/unit/test_babel.rb

@@ -7,6 +7,58 @@ require 'test/unit'
 
 
 class TestBabel < Test::Unit::TestCase
+  def test_babel_mpi_job
+    keyname = "appscale"
+    file = "/bucket/file.py"
+    params = { :type => "mpi",
+      :code => file,
+      :executable => 'python',
+      :procs_to_use => 1,
+      :nodes_to_use => 1,
+      :storage => "s3",
+      :EC2_ACCESS_KEY => "boo",
+      :EC2_SECRET_KEY => "baz",
+      :S3_URL => "http://baz.com",
+      :keyname => keyname
+    }
+
+    job_data = {}
+    params.each { |k, v|
+      job_data["@#{k}"] = v
+    }
+    job_data["@is_remote"] = true
+
+    output = "/bucket/babel/temp-0123456789"
+    job_data["@output"] = output
+
+    run_job_data = job_data.dup
+    run_job_data["@engine"] = "executor-sqs"
+    run_job_data["@run_local"] = true
+
+    output_job_data = job_data.dup
+    output_job_data["@type"] = "output"
+
+    kernel = flexmock(Kernel)
+    kernel.should_receive(:puts).and_return()
+    kernel.should_receive(:rand).and_return(0,1,2,3,4,5,6,7,8,9)
+    kernel.should_receive(:sleep).and_return()
+
+    flexmock(AppControllerClient).new_instances { |instance|
+      instance.should_receive(:does_file_exist?).with(file, job_data).and_return(true)
+      instance.should_receive(:does_file_exist?).with(output, job_data).and_return(false)
+
+      instance.should_receive(:start_neptune_job).with(run_job_data).and_return("MPI job is now running")
+
+      instance.should_receive(:get_output).with(output_job_data).and_return("output")
+    }
+
+    commonfunctions = flexmock(CommonFunctions)
+    commonfunctions.should_receive(:get_from_yaml).with(keyname, :shadow).and_return("127.0.0.1")
+    commonfunctions.should_receive(:get_secret_key).with(keyname).and_return("secret")
+
+    assert_equal("output", babel(params))
+  end
+
   def test_bad_babel_params
     job_data_no_code_param = {}
     assert_raise(BadConfigurationException) {
@@ -42,15 +94,17 @@ class TestBabel < Test::Unit::TestCase
     ENV['BABEL_BUCKET_NAME'] = "/baz"
     actual_local_2 = BabelHelper.generate_output_location(job_data_local_code)
     assert_equal(expected_local, actual_local_2)
+    ENV['BABEL_BUCKET_NAME'] = nil
 
     # Not putting the initial slash on the bucket name should be fine too.
     ENV['BABEL_BUCKET_NAME'] = "baz"
     actual_local_3 = BabelHelper.generate_output_location(job_data_local_code)
     assert_equal(expected_local, actual_local_3)
+    ENV['BABEL_BUCKET_NAME'] = nil
 
     # Finally, if we run a job and specify remote code, that should be used
     # as the bucket.
-    job_data_remote_code = {"@code" => "/baz/boo/code.baz", "@is_remote" => true}
+    job_data_remote_code = {"@code" => "/baz/boo/code.baz", "@storage" => "s3"}
     expected_remote = "/baz/babel/temp-10"
 
     actual_remote = BabelHelper.generate_output_location(job_data_remote_code)
@@ -108,7 +162,8 @@ class TestBabel < Test::Unit::TestCase
     assert_equal(expected, actual_3)
   end
 
-  def test_run_job
+  def test_run_babel_job
+    # Running a job with no @type specified means it should be a Babel job
     job_data = {
       "@code" => "/baz/boo/code.baz",
       "@argv" => ["boo", "/remote/babel/baz", "gbaz"]
@@ -131,6 +186,31 @@ class TestBabel < Test::Unit::TestCase
     assert_equal(expected, actual)
   end
 
+  def test_run_mpi_job
+    # Running a job with @type specified should preserve the job type
+    job_data = {
+      "@type" => "mpi",
+      "@code" => "/baz/boo/code.baz",
+      "@argv" => ["boo", "/remote/babel/baz", "gbaz"]
+    }
+
+    neptune_params = {
+      :type => "mpi",
+      :code => "/baz/boo/code.baz",
+      :argv => ["boo", "/remote/babel/baz", "gbaz"],
+      :run_local => true,
+      :engine => "executor-sqs"
+    }
+
+    result = { :result => :success }
+    kernel = flexmock(Kernel)
+    kernel.should_receive(:neptune).with(neptune_params).and_return(result)
+
+    expected = :success
+    actual = BabelHelper.run_job(job_data)[:result]
+    assert_equal(expected, actual)
+  end
+
   def test_get_output
     job_data = {
       "@output" => "/baz/boo/code.baz"