neptune 0.2.1 → 0.2.2

data/test/integration/tc_mapreduce.rb

@@ -1,282 +0,0 @@
-
-class TestMapReduce < Test::Unit::TestCase
-  def test_java_mr_code
-    STORAGE_TYPES.each { |storage|
-      # TODO: once files api is good in appscale, test this use case
-      next if storage == "appdb"
-      run_java_mr_code(storage)
-    }
-  end
-
-  def test_mr_streaming_code
-    STORAGE_TYPES.each { |storage|
-      run_streaming_code(storage)
-    }
-  end
-
-  def run_java_mr_code(storage)
-    local_input = File.expand_path("~/neptune/samples/mapreduce/the-end-of-time.txt")
-    unless File.exists?(local_input)
-      abort("missing input corpus - please download it and try again.")
-    end
-    input = TestHelper.read_file(local_input)
-
-    local_code = File.expand_path("~/neptune/samples/mapreduce/hadoop-0.20.0-examples.jar")
-    unless File.exists?(local_code)
-      abort("missing hadoop examples jar - please download it and try again.")
-    end
-    main = "wordcount"
-
-    local_output = File.expand_path("~/neptune/samples/mapreduce/expected-output.txt")
-    unless File.exists?(local_output)
-      abort("missing expected output - please download it and try again.")
-    end
-    expected_output = TestHelper.read_file(local_output)
-
-    folder = "wordcount-#{TestHelper.get_random_alphanumeric}"
-    tmp_folder = "/tmp/#{folder}"
-    FileUtils.mkdir_p(tmp_folder)
-
-    input_name = "input"
-    local_input = "#{tmp_folder}/#{input_name}"
-    TestHelper.write_file(local_input, input)
-
-    remote_input = TestHelper.get_output_location("#{folder}-input", storage)
-    remote_code = TestHelper.get_output_location("#{folder}-code.jar", storage, notxt=true)
-    remote_output = TestHelper.get_output_location("#{folder}-output", storage)
-
-    put_file_in_storage(local_input, remote_input, storage)
-    put_file_in_storage(local_code, remote_code, storage)
-
-    start_mr_code(remote_input, remote_output, remote_code, main, storage)
-    get_mr_output(remote_output, expected_output, storage)
-  end
-
-
-  def start_mr_code(input, output, code, main, storage)
-    params = {
-      :type => "mapreduce",
-      :input  => input,
-      :output => output,
-      :mapreducejar => code,
-      :main => main,
-      :nodes_to_use => 1
-    }.merge(TestHelper.get_storage_params(storage))
-
-    status = nil
-
-    loop {
-      status = neptune(params)
-      if status[:msg] =~ /not enough free nodes/
-        puts status[:msg]
-      else
-        break
-      end
-      sleep(5)
-    }
-
-    msg = "Your job was not started successfully. The failure message " +
-      "reported was #{status[:msg]}"
-    assert_equal(status[:result], :success, msg)
-  end
-
-  def run_streaming_code(storage)
-    expected_output = "sum x ="
-    input = <<BAZ
-1	32
-33	64
-65	96
-97	128
-BAZ
-
-    map_code = <<BAZ
-#!/usr/bin/ruby -w
-# Programmer: Chris Bunch
-# mapper-ruby.rb: Solves part of the EP parallel benchmark via the 
-# MapReduce framework as follows:
-# Input: Takes in ranges of k values to compute over STDIN.
-# Output: list [l, X_k, Y_k]
-
-A = 5 ** 13
-S = 271828183
-MIN_VAL = 2 ** -46
-MAX_VAL = 2 ** 46
-
-def generate_random(k)
-  xk = (A ** k) * S % MAX_VAL
-  MIN_VAL * xk
-end
-
-def ep(k)
-  k = Integer(k)
-  
-  xj = generate_random(k)
-  yj = generate_random(k+1)
-  
-  t = xj * xj + yj * yj
-  
-  if t <= 1
-    xk = xj * Math.sqrt(-2 * Math.log(t) / t)
-    yk = yj * Math.sqrt(-2 * Math.log(t) / t)
-    
-    max = [xk.abs, yk.abs].max
-    l = max.floor
-    puts l.to_s + " " + xk.to_s + " " + yk.to_s
-  end
-end
-
-loop {
-  input = STDIN.gets
-  break if input.nil?
-  start, fin = input.chomp.split
-  start = Integer(start)
-  fin = Integer(fin)
-  current = start
-  loop {
-    ep(current)
-    current = current + 2
-    break if current > fin
-  }
-}
-
-BAZ
-
-    red_code = <<BAZ
-#!/usr/bin/ruby -w
-# Programmer: Chris Bunch
-# reducer-ruby.rb: Solves part of the EP parallel benchmark via the 
-# MapReduce framework as follows:
-# Input: list [l, X_k, Y_k]
-# Output: [l, sum(X_k), sum(Y_k)]
-
-current_l = nil
-
-x_count = 0
-y_count = 0
-
-sum_x = 0.0
-sum_y = 0.0
-
-loop {
-  input = STDIN.gets
-  break if input.nil?
-  l, x, y = input.chomp.split
-  l = Integer(l)
-  x = Float(x)
-  y = Float(y)
-  
-  current_l = l if current_l.nil?
-  
-  if l != current_l
-    puts "bucket = " + current_l.to_s + ", |x| = " + x_count.to_s + ", |y| = " + y_count.to_s
-    current_l = l
-    x_count = 0
-    y_count = 0
-  end
-  
-  sum_x = sum_x + x
-  sum_y = sum_y + y
-
-  abs_x = x.abs
-  abs_y = y.abs
-
-  if abs_x > abs_y
-    x_count = x_count + 1
-  else
-    y_count = y_count + 1 
-  end
-}
-
-puts "bucket = " + current_l.to_s + ", |x| = " + x_count.to_s + ", |y| = " + y_count.to_s
-puts "sum x = " + sum_x.to_s + ", sum y = " + sum_y.to_s
-
-BAZ
-
-    contents = TestHelper.get_random_alphanumeric(1024)
-    folder = "ep-#{TestHelper.get_random_alphanumeric}"
-
-    input_name = "input"
-    map_source = "map.rb"
-    red_source = "reduce.rb"
-
-    tmp_folder = "/tmp/#{folder}"
-    FileUtils.mkdir_p(tmp_folder)
-
-    local_input = "#{tmp_folder}/#{input_name}"
-    local_map = "#{tmp_folder}/#{map_source}"
-    local_red = "#{tmp_folder}/#{red_source}"
-
-    TestHelper.write_file(local_input, input)
-    TestHelper.write_file(local_map, map_code)
-    TestHelper.write_file(local_red, red_code)
-
-    remote_input = TestHelper.get_output_location("#{folder}-input", storage)
-    remote_map = TestHelper.get_output_location("#{folder}-map.rb", storage, notxt=true)
-    remote_red = TestHelper.get_output_location("#{folder}-reduce.rb", storage, notxt=true)
-    remote_output = TestHelper.get_output_location("#{folder}-output", storage)
-
-    put_file_in_storage(local_input, remote_input, storage)
-    put_file_in_storage(local_map, remote_map, storage)
-    put_file_in_storage(local_red, remote_red, storage)
-
-    start_mr_streaming_code(remote_input, remote_output, remote_map, remote_red, storage)
-    get_mr_output(remote_output, expected_output, storage)
-
-    FileUtils.rm_rf(local_input)
-    FileUtils.rm_rf(local_map)
-    FileUtils.rm_rf(local_red)
-  end
-
-  def put_file_in_storage(local, remote, storage)
-    params = {
-      :type => "input",
-      :local => local,
-      :remote => remote
-    }.merge(TestHelper.get_storage_params(storage))
-
-    input_result = neptune(params)
-
-    msg = "We were unable to store a file in the database. We " +
-      " got back this: #{msg}"
-    assert(input_result, msg)
-  end
-
-  def start_mr_streaming_code(input, output, map, reduce, storage)
-    params = {
-      :type => "mapreduce",
-      :input  => input,
-      :output => output,
-      :map => map,
-      :reduce => reduce,
-      :nodes_to_use => 1
-    }.merge(TestHelper.get_storage_params(storage))
-
-    status = nil
-
-    loop {
-      status = neptune(params)
-      if status[:msg] =~ /not enough free nodes/
-        puts status[:msg]
-      else
-        break
-      end
-      sleep(5)
-    }
-
-    msg = "Your job was not started successfully. The failure message " +
-      "reported was #{status[:msg]}"
-    assert_equal(status[:result], :success, msg)
-  end
-
-  def get_mr_output(output, expected, storage)
-    result = TestHelper.get_job_output(output, storage)
-
-    TestHelper.write_file("/tmp/result", result)
-
-    msg = "The MapReduce job you ran did not return the expected result. " +
-      "We expected to see [#{expected}] but instead saw [#{result}]"
-    success = result.include?(expected)
-    assert(success, msg)
-  end
-end 
-

data/test/integration/tc_mpi.rb

@@ -1,160 +0,0 @@
-
-class TestMPI < Test::Unit::TestCase
-  def test_hello_world_code
-    num_procs = 1
-  
-    STORAGE_TYPES.each { |storage|
-      run_hello_world_code(storage, num_procs)
-    }
-  end
-
-  def test_not_enough_procs
-    num_procs = 0
-  
-    STORAGE_TYPES.each { |storage|
-      msg = "Running an MPI compute job with p < n should have thrown " +
-        "an exception, when in fact it did not. Here we used #{storage} " +
-        "as the storage backend."
-
-      assert_raise(SystemExit, msg) {
-        run_hello_world_code(storage, num_procs)
-      }
-    }
-  end
-
-  def test_bad_source_code
-    location = "/tmp/baz" + TestHelper.get_random_alphanumeric
-    output = "/bazboo2"
-    storage = "appdb"
-
-    msg = "Running an MPI compute job with a non-existent code location" +
-      " should have thrown an exception, when in fact it did not."
-
-    assert_raise(SystemExit, msg) {
-      TestHelper.start_job("mpi", location, output, storage)
-    }
-
-    FileUtils.mkdir_p(location)
-
-    bad_file_msg = "Running an MPI compute job with a code location that" +
-      " is not a file should have thrown an exception, when in fact it did not."
-
-    assert_raise(SystemExit, bad_file_msg) {
-      TestHelper.start_job("mpi", location, output, storage)
-    }
-
-    FileUtils.rmdir(location)
-  end
-
-  def run_hello_world_code(storage, num_procs)
-    expected_output = "0: We have 1 processors"
-    ring_code = <<BAZ
- /*
-  "Hello World" MPI Test Program
- */
- #include <mpi.h>
- #include <stdio.h>
- #include <string.h>
- 
- #define BUFSIZE 128
- #define TAG 0
- 
- int main(int argc, char *argv[])
- {
-   char idstr[32];
-   char buff[BUFSIZE];
-   int numprocs;
-   int myid;
-   int i;
-   MPI_Status stat;
- 
-   MPI_Init(&argc,&argv); /* all MPI programs start with MPI_Init; all 'N' processes exist thereafter */
-   MPI_Comm_size(MPI_COMM_WORLD,&numprocs); /* find out how big the SPMD world is */
-   MPI_Comm_rank(MPI_COMM_WORLD,&myid); /* and this processes' rank is */
- 
-   /* At this point, all programs are running equivalently, the rank distinguishes
-      the roles of the programs in the SPMD model, with rank 0 often used specially... */
-   if(myid == 0)
-   {
-     printf("%d: We have %d processors", myid, numprocs);
-     for(i=1;i<numprocs;i++)
-     {
-       sprintf(buff, "Hello %d! ", i);
-       MPI_Send(buff, BUFSIZE, MPI_CHAR, i, TAG, MPI_COMM_WORLD);
-     }
-     for(i=1;i<numprocs;i++)
-     {
-       MPI_Recv(buff, BUFSIZE, MPI_CHAR, i, TAG, MPI_COMM_WORLD, &stat);
-       printf("%d: %s", myid, buff);
-     }
-   }
-   else
-   {
-     /* receive from rank 0: */
-     MPI_Recv(buff, BUFSIZE, MPI_CHAR, 0, TAG, MPI_COMM_WORLD, &stat);
-     sprintf(idstr, "Processor %d ", myid);
-     strncat(buff, idstr, BUFSIZE-1);
-     strncat(buff, "reporting for duty", BUFSIZE-1);
-     /* send to rank 0: */
-     MPI_Send(buff, BUFSIZE, MPI_CHAR, 0, TAG, MPI_COMM_WORLD);
-   }
- 
-   MPI_Finalize(); /* MPI Programs end with MPI Finalize; this is a weak synchronization point */
-   return 0;
- }
-
-BAZ
-
-    contents = TestHelper.get_random_alphanumeric(1024)
-    folder = "hello-world-#{TestHelper.get_random_alphanumeric}"
-    source = "HelloWorld.c"
-
-    tmp_folder = "/tmp/#{folder}"
-    FileUtils.mkdir_p(tmp_folder)
-    compiled = "#{tmp_folder}-compiled"
-    compiled_code = "#{compiled}/HelloWorld"
-
-    local = "#{tmp_folder}/#{source}"
-    TestHelper.write_file(local, ring_code)
-
-    output = TestHelper.get_output_location(folder, storage)
-
-    compile_mpi_code(tmp_folder, source, compiled)
-    start_mpi_code(compiled_code, num_procs, output, storage)
-    get_mpi_output(output, expected_output, storage)
-
-    FileUtils.rm_rf(tmp_folder)
-    FileUtils.rm_rf(compiled)
-  end
-
-  def compile_mpi_code(location, main_file, compiled)
-    std_out, std_err = TestHelper.compile_code(location, main_file, compiled)
-
-    make = "mpicc HelloWorld.c -o HelloWorld -Wall"
-    msg = "The MPI code did not compile as expected. It should have " +
-      "compiled with the command [#{make}] instead of [#{std_out}]."
-    assert_equal(std_out, make, msg)
-
-    msg = "The MPI code did not compile successfully. It reported " +
-      "the following error: #{std_err}"
-    assert_nil(std_err, msg)
-  end
-
-  def start_mpi_code(code_location, num_procs, output, storage)
-    params = { :procs_to_use => num_procs }
-    status = TestHelper.start_job("mpi", code_location, output, storage, params)
-
-    msg = "Your job was not started successfully. The failure message " +
-      "reported was #{status[:msg]}"
-    assert_equal(status[:result], :success, msg)
-  end
-
-  def get_mpi_output(output, expected, storage)
-    result = TestHelper.get_job_output(output, storage)
-
-    msg = "The MPI job you ran did not return the expected result. " +
-      "We expected to see [#{expected}] but instead saw [#{result}]"
-    assert_equal(result, expected, msg)
-  end
-end 
-

data/test/integration/tc_storage.rb

@@ -1,209 +0,0 @@
-
-class TestStorage < Test::Unit::TestCase
-  def test_acl
-    STORAGE_TYPES.each { |storage|
-      run_in_acl(storage)
-    }
-  end
-
-  def test_in_out
-    STORAGE_TYPES.each { |storage|
-      run_in_out(storage)
-    }
-  end
-
-  def test_run_in_out_w_env
-    STORAGE_TYPES.each { |storage|
-      run_in_out_w_env(storage)
-    }
-  end
-
-  def test_no_creds
-    creds = %w{
-GSTORAGE_ACCESS_KEY GSTORAGE_SECRET_KEY GSTORAGE_URL 
-S3_ACCESS_KEY S3_SECRET_KEY S3_URL 
-WALRUS_ACCESS_KEY WALRUS_SECRET_KEY WALRUS_URL
-}
-
-    old_creds = {}
-    creds.each { |c|
-      old_creds[c] = ENV[c]
-      ENV[c] = nil
-    }
-
-    # try an input job with creds in env but not in job
-    # should succeed
-
-    STORAGE_TYPES.each { |storage|
-      params = { :storage => storage }
-      testhelper = flexmock(TestHelper)
-      testhelper.should_receive(:get_storage_params).with(storage).and_return(params)
-
-      no_msg = "Trying to start a storage job and failing to specify " +
-        "necessary credentials should not have failed, but it did." +
-        " The storage type used was #{storage}."
-
-      msg = "Trying to start a storage job and failing to specify " +
-        "necessary credentials should have failed, but it didn't." +
-        " The storage type used was #{storage}."
-
-      if storage == "appdb"
-        assert_nothing_raised(SystemExit, no_msg) {
-          run_in_out(storage)
-        }
-      else
-        assert_raise(SystemExit, msg) {
-          run_in_out(storage)
-        }
-      end
-    }
-
-    creds.each { |c|
-      ENV[c] = old_creds[c]
-    }
-  end
-
-  def test_bad_storage
-    msg = "Specifying an incorrect storage backend should have thrown an " + 
-      "exception, when in fact it did not."
-    assert_raise(SystemExit, msg) { run_in_out("blarg_storage") }
-  end
-
-  def test_bad_output_location
-    output = "baz-boo-for-me-too"
-
-    STORAGE_TYPES.each { |storage|
-      end_of_msg = " should have thrown an exception, when in fact it did not." +
-        "Here we tested with #{storage} as the storage backend."
-
-      no_slash_msg = "Specifying an output location without a leading slash"
-
-      assert_raise(SystemExit, no_slash_msg + end_of_msg) { 
-        TestHelper.get_job_output(output, storage) 
-      }
-
-      no_output_msg = "Specifying an output job with a blank output parameter"
-      assert_raise(SystemExit, no_output_msg + end_of_msg) {
-        TestHelper.get_job_output("", storage)
-      }
-
-      nil_output_msg = "Specifying an output job with a nil output" 
-      assert_raise(SystemExit, nil_output_msg + end_of_msg) {
-        TestHelper.get_job_output(nil, storage)
-      }
-    }
-  end
-
-  def run_in_acl(storage)
-    contents = TestHelper.get_random_alphanumeric(1024) + "+&baz"
-    suffix = "neptune-testfile-#{TestHelper.get_random_alphanumeric}"
-    local = "/tmp/#{suffix}"
-    TestHelper.write_file(local, contents)
-    remote = TestHelper.get_output_location(suffix, storage)
-
-    in_params = {
-      :type => "input",
-      :local => local,
-      :remote => remote
-    }.merge(TestHelper.get_storage_params(storage))
-
-    input_result = neptune(in_params)
-
-    msg = "We were unable to store a file in the datastore. We " +
-      " got back this: #{msg}"
-    assert(input_result, msg)
-
-    get_params = {
-      :type => "get-acl",
-      :output => remote
-    }.merge(TestHelper.get_storage_params(storage))
-
-    acl = neptune(get_params)
-
-    get_acl_msg1 = "The default ACL should be private, but was [#{acl}] instead."
-    assert_equal("private", acl, get_acl_msg1)
-
-    # TODO: set acl is currently broken - once we fix it, we should
-    # do the following:
-
-    # set acl to something else
-    # verify that it was set correctly
-
-    FileUtils.rm_rf(local)
-  end
- 
-  def run_in_out_w_env(storage)
-    creds = %w{
-GSTORAGE_ACCESS_KEY GSTORAGE_SECRET_KEY GSTORAGE_URL 
-S3_ACCESS_KEY S3_SECRET_KEY S3_URL 
-WALRUS_ACCESS_KEY WALRUS_SECRET_KEY WALRUS_URL
-}
-
-    old_creds = {}
-    creds.each { |c|
-      old_creds[c] = ENV[c]
-    }
-
-    s3_creds = %w{ EC2_ACCESS_KEY EC2_SECRET_KEY S3_URL }
-
-    needed_creds = TestHelper.get_storage_params(storage)
-    puts needed_creds.inspect
-
-    params = { :storage => storage }
-    testhelper = flexmock(TestHelper)
-    testhelper.should_receive(:get_storage_params).with(storage).and_return(params)
-
-    s3_creds.each { |c|
-      ENV[c] = needed_creds[c.to_sym]
-    }
-
-    run_in_out(storage)
-
-    s3_creds.each { |c|
-      ENV[c] = nil
-    }
-    
-    creds.each { |c|
-      ENV[c] = old_creds[c]
-    }
-
-    testhelper.flexmock_teardown
-  end
-
-  def run_in_out(storage)
-    contents = TestHelper.get_random_alphanumeric(1024) + "+&baz"
-    suffix = "neptune-testfile-#{TestHelper.get_random_alphanumeric}"
-    local = "/tmp/#{suffix}"
-    TestHelper.write_file(local, contents)
-    
-    run_input(local, suffix, storage)
-    run_output(contents, suffix, storage)
-
-    FileUtils.rm_rf(local)
-  end
-
-  def run_input(local, suffix, storage)
-    params = {
-      :type => "input",
-      :local => local,
-      :remote => TestHelper.get_output_location(suffix, storage)
-    }.merge(TestHelper.get_storage_params(storage))
-
-    input_result = neptune(params)
-
-    msg = "We were unable to store a file in the database. We " +
-      " got back this: #{msg}"
-    assert(input_result, msg)
-  end
-
-  def run_output(local, suffix, storage)
-    output = TestHelper.get_output_location(suffix, storage)
-    remote = TestHelper.get_job_output(output, storage)
-
-    msg = "We were unable to verify that the remote file matches the " +
-      "local version. The local copy's contents are: " +
-      "[#{local}], while the remote copy's contents are [#{remote}]."
-    assert_equal(local, remote, msg)
-  end
-end 
-