neptune 0.0.7 → 0.0.8

data/README

@@ -38,7 +38,10 @@ be found in the test folder, with the standard naming convention
 cases for each type of job that Neptune offers. Before running
 ts_neptune, you should export the environment variable APPSCALE_HEAD_NODE,
 which should be set to the IP address of the AppScale machine that runs
-the Shadow daemon (a.k.a. the Master AppController).
+the Shadow daemon (a.k.a. the Master AppController). Running
+generate_coverage.sh in the top-level directory will run rcov
+and generate the coverage reports automatically - currently this process
+takes about an hour.
 
 Developed by Chris Bunch as part of the AppScale project.
 See LICENSE for the specifics of the New BSD License
@@ -63,10 +66,14 @@ in for a link to that as it becomes available.
 
 Version History:
 
-April 2, 2001 - 0.0.7 released, adding automatic test suite
+April 8, 2011 - 0.0.8 released, fixing MapReduce support
+for both regular Hadoop and Hadoop Streaming. Also increased
+code coverage to cover a number of failure scenarios.
+
+April 2, 2011 - 0.0.7 released, adding automatic test suite
 and many bug fixes for all scenarios. rcov can also be used
 to generate test coverage information: current coverage stats
-can be found in coverage directory. mapreduce broken at the
+can be found in coverage directory. MapReduce broken at the
 moment - will fix in next release
 
 March 28, 2011 - 0.0.6 released, adding support for input jobs,

data/bin/neptune

@@ -5,10 +5,19 @@ $VERBOSE = nil # to surpress excessive SSL cert warnings
 $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 require 'neptune'
 
-file = ARGV[0]
-unless File.exists?(file)
-  abort("File #{file} does not exist.")
-end
+if ARGV.length == 1
+  file = ARGV[0]
+  unless File.exists?(file)
+    abort("File #{file} does not exist.")
+  end
+
+  load file
+else
+  unless ARGV.length % 2 == 0
+    abort("When creating a Neptune job on the fly, an even number of parameters must be specified.")
+  end
 
-load file
+  params = Hash[*ARGV]
+  puts neptune(params)
+end
 

data/lib/app_controller_client.rb

@@ -95,7 +95,7 @@ class AppControllerClient
   # storage parameter.
   def put_input(job_data)
     result = ""
-    make_call(NO_TIMEOUT, false) { 
+    make_call(NO_TIMEOUT, false) {
       result = conn.neptune_put_input(job_data, @secret)
     }  
     abort(result) if result =~ /Error:/

data/lib/common_functions.rb

@@ -41,17 +41,8 @@ module CommonFunctions
     ssh_args = "-o StrictHostkeyChecking=no 2>&1"
     ssh_args << " -r " if is_dir
 
-    if public_key_loc.class == Array
-      public_key_loc.each { |key|
-        key = File.expand_path(key)
-      }
-      
-      cmd = "scp -i #{public_key_loc.join(' -i ')} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
-    else
-      public_key_loc = File.expand_path(public_key_loc)
-      cmd = "scp -i #{public_key_loc} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
-    end
-
+    public_key_loc = File.expand_path(public_key_loc)
+    cmd = "scp -i #{public_key_loc} #{ssh_args} #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
     cmd << "; echo $? >> ~/.appscale/retval"
 
     retval_loc = File.expand_path("~/.appscale/retval")

data/lib/neptune.rb

@@ -29,11 +29,11 @@ NO_OUTPUT_NEEDED = ["input"]
 
 # A list of storage mechanisms that we can use to store and retrieve
 # data to for Neptune jobs.
-ALLOWED_STORAGE_TYPES = ["appdb", "gstorage", "s3"]
+ALLOWED_STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"]
 
 # A list of jobs that require some kind of work to be done before
 # the actual computation can be performed.
-NEED_PREPROCESSING = ["compile", "erlang", "mapreduce", "mpi"]
+NEED_PREPROCESSING = ["compile", "erlang", "mpi"]
 
 # A set of methods and constants that we've monkey-patched to enable Neptune
 # support. In the future, it is likely that the only exposed / monkey-patched
@@ -91,34 +91,6 @@ def preprocess_erlang(job_data)
   CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
 end
 
-# This preprocessing method handles copying data for regular
-# Hadoop MapReduce and Hadoop MapReduce Streaming. For the former
-# case, we copy over just the JAR the user has given us, and
-# in the latter case, we copy over the Map and Reduce files
-# that have been specified. In either case, if the user has
-# specified to us to copy over an input file, we do that as well:
-# AppScale will copy it into HDFS for us.
-def preprocess_mapreduce(job_data)
-  return
-  #items_to_copy = ["@map", "@reduce"] if job_data["@map"] and job_data["@reduce"]
-  items_to_copy = ["@mapreducejar"] if job_data["@mapreducejar"]
-  #items_to_copy << "@input" if job_data["@copy_input"]
-  items_to_copy.each { |item|
-    source = File.expand_path(job_data[item])
-    unless File.exists?(source)
-      abort("The #{item} file #{source} does not exist.")
-    end
-
-    suffix = source.split('/')[-1]
-    dest = "/tmp/#{suffix}"
-
-    keyname = job_data["@keyname"]
-    CommonFunctions.scp_to_shadow(source, dest, keyname)
-
-    job_data[item] = dest
-  }
-end
-
 # This preprocessing method copies over the user's MPI
 # code to the master node in AppScale - this node will
 # then copy it to whoever will run the MPI job.
@@ -220,9 +192,10 @@ def neptune(params)
       abort(msg)
     end
 
-    # Our implementation for storing / retrieving via Google Storage uses
+    # Our implementation for storing / retrieving via Google Storage
+    # and Walrus uses
     # the same library as we do for S3 - so just tell it that it's S3
-    if storage == "gstorage"
+    if storage == "gstorage" or storage == "walrus"
       storage = "s3"
       job_data["@storage"] = "s3"
     end

data/test/tc_erlang.rb

@@ -6,6 +6,32 @@ class TestErlang < Test::Unit::TestCase
     }
   end
 
+  def test_nonexistent_source_code
+    location = "baz" + TestHelper.get_random_alphanumeric
+    main_file = "boo"
+    compiled = "bazagain"
+
+    msg = "Running a compile job with a non-existent source code location" +
+      " should have thrown an exception, when in fact it did not."
+
+    assert_raise(SystemExit, msg) {
+      TestHelper.compile_code(location, main_file, compiled)
+    }
+  end
+
+  def test_nonexistent_compiled_code
+    location = "baz" + TestHelper.get_random_alphanumeric
+    output = "/bazboo2"
+    storage = "appdb"
+
+    msg = "Running an Erlang compute job with a non-existent code location" +
+      " should have thrown an exception, when in fact it did not."
+
+    assert_raise(SystemExit, msg) {
+      TestHelper.start_job("erlang", location, output, storage)
+    }
+  end
+
   def run_ring_code(storage)
     expected_output = "total time for"
     ring_code = <<BAZ

data/test/tc_mapreduce.rb

@@ -1,13 +1,87 @@
 
 class TestMapReduce < Test::Unit::TestCase
+  def test_java_mr_code
+    STORAGE_TYPES.each { |storage|
+      # TODO: once files api is good in appscale, test this use case
+      next if storage == "appdb"
+      run_java_mr_code(storage)
+    }
+  end
+
   def test_mr_streaming_code
     STORAGE_TYPES.each { |storage|
       run_streaming_code(storage)
     }
   end
 
+  def run_java_mr_code(storage)
+    local_input = File.expand_path("~/neptune/samples/mapreduce/the-end-of-time.txt")
+    unless File.exists?(local_input)
+      abort("missing input corpus - please download it and try again.")
+    end
+    input = TestHelper.read_file(local_input)
+
+    local_code = File.expand_path("~/neptune/samples/mapreduce/hadoop-0.20.0-examples.jar")
+    unless File.exists?(local_code)
+      abort("missing hadoop examples jar - please download it and try again.")
+    end
+    main = "wordcount"
+
+    local_output = File.expand_path("~/neptune/samples/mapreduce/expected-output.txt")
+    unless File.exists?(local_output)
+      abort("missing expected output - please download it and try again.")
+    end
+    expected_output = TestHelper.read_file(local_output)
+
+    folder = "wordcount-#{TestHelper.get_random_alphanumeric}"
+    tmp_folder = "/tmp/#{folder}"
+    FileUtils.mkdir_p(tmp_folder)
+
+    input_name = "input"
+    local_input = "#{tmp_folder}/#{input_name}"
+    TestHelper.write_file(local_input, input)
+
+    remote_input = TestHelper.get_output_location("#{folder}-input", storage)
+    remote_code = TestHelper.get_output_location("#{folder}-code.jar", storage, notxt=true)
+    remote_output = TestHelper.get_output_location("#{folder}-output", storage)
+
+    put_file_in_storage(local_input, remote_input, storage)
+    put_file_in_storage(local_code, remote_code, storage)
+
+    start_mr_code(remote_input, remote_output, remote_code, main, storage)
+    get_mr_output(remote_output, expected_output, storage)
+  end
+
+
+  def start_mr_code(input, output, code, main, storage)
+    params = {
+      :type => "mapreduce",
+      :input  => input,
+      :output => output,
+      :mapreducejar => code,
+      :main => main,
+      :nodes_to_use => 1
+    }.merge(TestHelper.get_storage_params(storage))
+
+    status = nil
+
+    loop {
+      status = neptune(params)
+      if status[:msg] =~ /not enough free nodes/
+        puts status[:msg]
+      else
+        break
+      end
+      sleep(5)
+    }
+
+    msg = "Your job was not started successfully. The failure message " +
+      "reported was #{status[:msg]}"
+    assert_equal(status[:result], :success, msg)
+  end
+
   def run_streaming_code(storage)
-    expected_output = "0: We have 1 processors"
+    expected_output = "sum x ="
     input = <<BAZ
 1	32
 33	64
@@ -16,7 +90,7 @@ class TestMapReduce < Test::Unit::TestCase
 BAZ
 
     map_code = <<BAZ
-#!/usr/local/bin/ruby -w
+#!/usr/bin/ruby -w
 # Programmer: Chris Bunch
 # mapper-ruby.rb: Solves part of the EP parallel benchmark via the 
 # MapReduce framework as follows:
@@ -47,7 +121,7 @@ def ep(k)
     
     max = [xk.abs, yk.abs].max
     l = max.floor
-    puts l + " " + xk + " " + yk
+    puts l.to_s + " " + xk.to_s + " " + yk.to_s
   end
 end
 
@@ -68,7 +142,7 @@ loop {
 BAZ
 
     red_code = <<BAZ
-#!/usr/local/bin/ruby -w
+#!/usr/bin/ruby -w
 # Programmer: Chris Bunch
 # reducer-ruby.rb: Solves part of the EP parallel benchmark via the 
 # MapReduce framework as follows:
@@ -94,7 +168,7 @@ loop {
   current_l = l if current_l.nil?
   
   if l != current_l
-    puts "bucket = " + current_l + ", |x| = " + x_count + ", |y| = " + y_count
+    puts "bucket = " + current_l.to_s + ", |x| = " + x_count.to_s + ", |y| = " + y_count.to_s
     current_l = l
     x_count = 0
     y_count = 0
@@ -113,8 +187,8 @@ loop {
   end
 }
 
-puts "bucket = " + current_l + ", |x| = " + x_count + ", |y| = " + y_count
-puts "sum x = " + sum_x + ", sum y = " + sum_y
+puts "bucket = " + current_l.to_s + ", |x| = " + x_count.to_s + ", |y| = " + y_count.to_s
+puts "sum x = " + sum_x.to_s + ", sum y = " + sum_y.to_s
 
 BAZ
 
@@ -127,8 +201,6 @@ BAZ
 
     tmp_folder = "/tmp/#{folder}"
     FileUtils.mkdir_p(tmp_folder)
-    compiled = "#{tmp_folder}-compiled"
-    compiled_code = "#{compiled}/HelloWorld"
 
     local_input = "#{tmp_folder}/#{input_name}"
     local_map = "#{tmp_folder}/#{map_source}"
@@ -199,9 +271,12 @@ BAZ
   def get_mr_output(output, expected, storage)
     result = TestHelper.get_job_output(output, storage)
 
+    TestHelper.write_file("/tmp/result", result)
+
     msg = "The MapReduce job you ran did not return the expected result. " +
       "We expected to see [#{expected}] but instead saw [#{result}]"
-    assert_equal(result, expected, msg)
+    success = result.include?(expected)
+    assert(success, msg)
   end
 end 
 

data/test/tc_mpi.rb

@@ -1,15 +1,51 @@
 
 class TestMPI < Test::Unit::TestCase
   def test_hello_world_code
-    num_procs = [1]
+    num_procs = 1
+  
+    STORAGE_TYPES.each { |storage|
+      run_hello_world_code(storage, num_procs)
+    }
+  end
 
+  def test_not_enough_procs
+    num_procs = 0
+  
     STORAGE_TYPES.each { |storage|
-      num_procs.each { |p|
-        run_hello_world_code(storage, p)
+      msg = "Running an MPI compute job with p < n should have thrown " +
+        "an exception, when in fact it did not. Here we used #{storage} " +
+        "as the storage backend."
+
+      assert_raise(SystemExit, msg) {
+        run_hello_world_code(storage, num_procs)
       }
     }
   end
 
+  def test_bad_source_code
+    location = "/tmp/baz" + TestHelper.get_random_alphanumeric
+    output = "/bazboo2"
+    storage = "appdb"
+
+    msg = "Running an MPI compute job with a non-existent code location" +
+      " should have thrown an exception, when in fact it did not."
+
+    assert_raise(SystemExit, msg) {
+      TestHelper.start_job("mpi", location, output, storage)
+    }
+
+    FileUtils.mkdir_p(location)
+
+    bad_file_msg = "Running an MPI compute job with a code location that" +
+      " is not a file should have thrown an exception, when in fact it did not."
+
+    assert_raise(SystemExit, bad_file_msg) {
+      TestHelper.start_job("mpi", location, output, storage)
+    }
+
+    FileUtils.rmdir(location)
+  end
+
   def run_hello_world_code(storage, num_procs)
     expected_output = "0: We have 1 processors"
     ring_code = <<BAZ

data/test/tc_storage.rb

@@ -1,13 +1,177 @@
 
 class TestStorage < Test::Unit::TestCase
+  def test_acl
+    STORAGE_TYPES.each { |storage|
+      run_in_acl(storage)
+    }
+  end
+
   def test_in_out
     STORAGE_TYPES.each { |storage|
       run_in_out(storage)
     }
   end
 
+  def test_run_in_out_w_env
+    STORAGE_TYPES.each { |storage|
+      run_in_out_w_env(storage)
+    }
+  end
+
+  def test_no_creds
+    creds = %w{
+GSTORAGE_ACCESS_KEY GSTORAGE_SECRET_KEY GSTORAGE_URL 
+S3_ACCESS_KEY S3_SECRET_KEY S3_URL 
+WALRUS_ACCESS_KEY WALRUS_SECRET_KEY WALRUS_URL
+}
+
+    old_creds = {}
+    creds.each { |c|
+      old_creds[c] = ENV[c]
+      ENV[c] = nil
+    }
+
+    # try an input job with creds in env but not in job
+    # should succeed
+
+    STORAGE_TYPES.each { |storage|
+      params = { :storage => storage }
+      testhelper = flexmock(TestHelper)
+      testhelper.should_receive(:get_storage_params).with(storage).and_return(params)
+
+      no_msg = "Trying to start a storage job and failing to specify " +
+        "necessary credentials should not have failed, but it did." +
+        " The storage type used was #{storage}."
+
+      msg = "Trying to start a storage job and failing to specify " +
+        "necessary credentials should have failed, but it didn't." +
+        " The storage type used was #{storage}."
+
+      if storage == "appdb"
+        assert_nothing_raised(SystemExit, no_msg) {
+          run_in_out(storage)
+        }
+      else
+        assert_raise(SystemExit, msg) {
+          run_in_out(storage)
+        }
+      end
+    }
+
+    creds.each { |c|
+      ENV[c] = old_creds[c]
+    }
+  end
+
+  def test_bad_storage
+    msg = "Specifying an incorrect storage backend should have thrown an " + 
+      "exception, when in fact it did not."
+    assert_raise(SystemExit, msg) { run_in_out("blarg_storage") }
+  end
+
+  def test_bad_output_location
+    output = "baz-boo-for-me-too"
+
+    STORAGE_TYPES.each { |storage|
+      end_of_msg = " should have thrown an exception, when in fact it did not." +
+        "Here we tested with #{storage} as the storage backend."
+
+      no_slash_msg = "Specifying an output location without a leading slash"
+
+      assert_raise(SystemExit, no_slash_msg + end_of_msg) { 
+        TestHelper.get_job_output(output, storage) 
+      }
+
+      no_output_msg = "Specifying an output job with a blank output parameter"
+      assert_raise(SystemExit, no_output_msg + end_of_msg) {
+        TestHelper.get_job_output("", storage)
+      }
+
+      nil_output_msg = "Specifying an output job with a nil output" 
+      assert_raise(SystemExit, nil_output_msg + end_of_msg) {
+        TestHelper.get_job_output(nil, storage)
+      }
+    }
+  end
+
+  def run_in_acl(storage)
+    contents = TestHelper.get_random_alphanumeric(1024) + "+&baz"
+    suffix = "neptune-testfile-#{TestHelper.get_random_alphanumeric}"
+    local = "/tmp/#{suffix}"
+    TestHelper.write_file(local, contents)
+    remote = TestHelper.get_output_location(suffix, storage)
+
+    in_params = {
+      :type => "input",
+      :local => local,
+      :remote => remote
+    }.merge(TestHelper.get_storage_params(storage))
+
+    input_result = neptune(in_params)
+
+    msg = "We were unable to store a file in the datastore. We " +
+      " got back this: #{msg}"
+    assert(input_result, msg)
+
+    get_params = {
+      :type => "get-acl",
+      :output => remote
+    }.merge(TestHelper.get_storage_params(storage))
+
+    acl = neptune(get_params)
+
+    get_acl_msg1 = "The default ACL should be private, but was [#{acl}] instead."
+    assert_equal("private", acl, get_acl_msg1)
+
+    # TODO: set acl is currently broken - once we fix it, we should
+    # do the following:
+
+    # set acl to something else
+    # verify that it was set correctly
+
+    FileUtils.rm_rf(local)
+  end
+ 
+  def run_in_out_w_env(storage)
+    creds = %w{
+GSTORAGE_ACCESS_KEY GSTORAGE_SECRET_KEY GSTORAGE_URL 
+S3_ACCESS_KEY S3_SECRET_KEY S3_URL 
+WALRUS_ACCESS_KEY WALRUS_SECRET_KEY WALRUS_URL
+}
+
+    old_creds = {}
+    creds.each { |c|
+      old_creds[c] = ENV[c]
+    }
+
+    s3_creds = %w{ EC2_ACCESS_KEY EC2_SECRET_KEY S3_URL }
+
+    needed_creds = TestHelper.get_storage_params(storage)
+    puts needed_creds.inspect
+
+    params = { :storage => storage }
+    testhelper = flexmock(TestHelper)
+    testhelper.should_receive(:get_storage_params).with(storage).and_return(params)
+
+    s3_creds.each { |c|
+      ENV[c] = needed_creds[c.to_sym]
+    }
+
+    run_in_out(storage)
+
+    s3_creds.each { |c|
+      ENV[c] = nil
+    }
+    
+    creds.each { |c|
+      ENV[c] = old_creds[c]
+    }
+
+    testhelper.flexmock_teardown
+  end
+
   def run_in_out(storage)
-    contents = TestHelper.get_random_alphanumeric(1024)
+    contents = TestHelper.get_random_alphanumeric(1024) + "+&baz"
     suffix = "neptune-testfile-#{TestHelper.get_random_alphanumeric}"
     local = "/tmp/#{suffix}"
     TestHelper.write_file(local, contents)

data/test/test_helper.rb

@@ -98,11 +98,9 @@ module TestHelper
         :EC2_SECRET_KEY => ENV['WALRUS_SECRET_KEY'],
         :S3_URL => ENV['WALRUS_URL']
       }
-    elsif storage == "appdb"
-      return {}
-      # nothing special to do
     else
-      abort "Storage specified was not an acceptable value: #{storage}"
+      return { :storage => storage }
+      # nothing special to do
     end
   end
 

data/test/ts_neptune.rb

@@ -1,5 +1,4 @@
-#STORAGE_TYPES = ["gstorage"]
-STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"]
+STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"] - ["appdb"]
 
 $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 require 'neptune'
@@ -13,6 +12,7 @@ S3_ACCESS_KEY S3_SECRET_KEY S3_URL
 WALRUS_ACCESS_KEY WALRUS_SECRET_KEY WALRUS_URL }
 
 require 'test/unit'
+require 'flexmock/test_unit'
 
 REQUIRED_CREDS.each { |cred|
   msg = "The environment variable #{cred} was not set. Please " +
@@ -25,17 +25,13 @@ msg = "AppScale is not currently running at " +
   "#{APPSCALE_HEAD_NODE_IP}. Please start AppScale and try again."
 abort(msg) unless TestHelper.is_appscale_running?(APPSCALE_HEAD_NODE_IP)
 
-TEST_ALL_WORKING = true
-
-if TEST_ALL_WORKING
-  require 'tc_c'
-  require 'tc_dfsp'
-  require 'tc_dwssa'
-  require 'tc_erlang'
-  require 'tc_mpi'
-  require 'tc_storage'
-  require 'tc_upc'
-  require 'tc_x10'
-else
-  require 'tc_mapreduce'
-end
+require 'tc_c'
+require 'tc_dfsp'
+require 'tc_dwssa'
+require 'tc_erlang'
+require 'tc_mapreduce'
+require 'tc_mpi'
+require 'tc_storage'
+require 'tc_upc'
+require 'tc_x10'
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: neptune
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 15
   prerelease: 
   segments: 
   - 0
   - 0
-  - 7
-  version: 0.0.7
+  - 8
+  version: 0.0.8
 platform: ruby
 authors: 
 - Chris Bunch
@@ -15,7 +15,7 @@ autorequire: neptune
 bindir: bin
 cert_chain: []
 
-date: 2011-04-02 00:00:00 -07:00
+date: 2011-04-08 00:00:00 -07:00
 default_executable: neptune
 dependencies: []