neptune 0.0.6 → 0.0.7

data/test/tc_erlang.rb

@@ -0,0 +1,157 @@
+
+class TestErlang < Test::Unit::TestCase
+  def test_ring_code
+    STORAGE_TYPES.each { |storage|
+      run_ring_code(storage)
+    }
+  end
+
+  def run_ring_code(storage)
+    expected_output = "total time for"
+    ring_code = <<BAZ
+-module(ring).
+-compile(export_all).
+
+% spawn N processes
+% M times, send message to proc 1 
+% when proc i recv's a message, send it to i+1
+
+% distributed version:
+% global var for master node
+% var with list of nodes
+% if master:
+%   no high-level changes
+% else:
+%   wait for master to give me work
+%   loop until i receive a kill message
+%   which breaks this loop and kills this node
+
+% smaller changes:
+
+start(Name, Next) ->
+  %io:format("creating proc named ~p with next proc named ~p~n", [Name, Next]),
+  register(Name, spawn(fun() -> ring:startMe(Next) end)),
+  Name.
+
+startMe(Next) ->
+  receive
+    stop ->
+      void;
+    {Message, Initiator} ->
+      NextPid = whereis(Next),
+      if
+        is_pid(NextPid) ->
+          %io:format("sending message to next proc, ~p~n", [Next]),
+          NextPid ! {Message, Initiator},
+          startMe(Next);
+        true ->
+          %io:format("no next proc to send message to!~n"),
+          Initiator ! done,
+          startMe(Next)
+      end
+  end.
+
+generateProcs(Num) ->
+  if
+    Num > 0 ->
+      [start(ring:i_to_a(Num), ring:i_to_a(Num-1)) | ring:generateProcs(Num-1)];
+    true ->
+      []
+  end.
+
+stopProcs([H | T]) ->
+  HeadPid = whereis(H),
+  HeadPid ! stop,
+  stopProcs(T);
+stopProcs([]) ->
+    void.
+
+sendMessageToFirst([H | T], Message) ->
+  HeadPid = whereis(H),
+  HeadPid ! {Message, self()},
+  receive
+    done -> void
+  end.
+
+a_to_i(A) ->
+  list_to_integer(atom_to_list(A)).
+
+i_to_a(I) ->
+  list_to_atom(integer_to_list(I)).
+
+times(Num, Fun) ->
+  if Num > 0 ->
+    Fun(),
+    times(Num-1, Fun);
+  true ->
+    void
+  end.
+
+main() ->
+  statistics(wall_clock),
+  NumProcs = 10,
+  NumMessages = 1,
+  Processes = ring:generateProcs(NumProcs),
+  Message = "hello!",
+  times(NumMessages, fun() -> sendMessageToFirst(Processes, Message) end),
+  ring:stopProcs(Processes),
+  {_, TotalTime} = statistics(wall_clock),
+  TimeInMicroseconds = TotalTime * 1000,
+  io:format("total time for N = ~p, M = ~p, is ~p microseconds~n", [NumProcs, NumMessages, TimeInMicroseconds]),
+  exit('baz').
+
+BAZ
+
+    contents = TestHelper.get_random_alphanumeric(1024)
+    folder = "ring-#{TestHelper.get_random_alphanumeric}"
+    source = "ring.erl"
+
+    tmp_folder = "/tmp/#{folder}"
+    FileUtils.mkdir_p(tmp_folder)
+    compiled = "#{tmp_folder}-compiled"
+    compiled_code = "#{compiled}/ring.beam"
+
+    local = "#{tmp_folder}/#{source}"
+    TestHelper.write_file(local, ring_code)
+
+    output = TestHelper.get_output_location(folder, storage)
+
+    compile_erlang_code(tmp_folder, source, compiled)
+    start_erlang_code(compiled_code, output, storage)
+    get_erlang_output(output, expected_output, storage)
+
+    FileUtils.rm_rf(tmp_folder)
+    FileUtils.rm_rf(compiled)
+  end
+
+  def compile_erlang_code(location, main_file, compiled)
+    std_out, std_err = TestHelper.compile_code(location, main_file, compiled)
+
+    make = "HOME=/root erlc ring.erl"
+    msg = "The Erlang Ring code did not compile as expected. It should have " +
+      "compiled with the command [#{make}] instead of [#{std_out}]."
+    assert(std_out.include?(make), msg)
+
+    msg = "The Erlang Ring code did not compile successfully. It reported " +
+      "the following error: #{std_err}"
+    assert_nil(std_err, msg)
+  end
+
+  def start_erlang_code(code_location, output, storage)
+    status = TestHelper.start_job("erlang", code_location, output, storage)
+
+    msg = "Your job was not started successfully. The failure message " +
+      "reported was #{status[:msg]}"
+    assert_equal(status[:result], :success, msg)
+  end
+
+  def get_erlang_output(output, expected, storage)
+    result = TestHelper.get_job_output(output, storage)
+
+    msg = "The Erlang job you ran did not return the expected result. " +
+      "We expected to see [#{expected}] but instead saw [#{result}]"
+    out_contains = result.include?(expected)
+    assert(out_contains, msg)
+  end
+end 
+

data/test/tc_mapreduce.rb

@@ -0,0 +1,207 @@
+
+class TestMapReduce < Test::Unit::TestCase
+  def test_mr_streaming_code
+    STORAGE_TYPES.each { |storage|
+      run_streaming_code(storage)
+    }
+  end
+
+  def run_streaming_code(storage)
+    expected_output = "0: We have 1 processors"
+    input = <<BAZ
+1	32
+33	64
+65	96
+97	128
+BAZ
+
+    map_code = <<BAZ
+#!/usr/local/bin/ruby -w
+# Programmer: Chris Bunch
+# mapper-ruby.rb: Solves part of the EP parallel benchmark via the 
+# MapReduce framework as follows:
+# Input: Takes in ranges of k values to compute over STDIN.
+# Output: list [l, X_k, Y_k]
+
+A = 5 ** 13
+S = 271828183
+MIN_VAL = 2 ** -46
+MAX_VAL = 2 ** 46
+
+def generate_random(k)
+  xk = (A ** k) * S % MAX_VAL
+  MIN_VAL * xk
+end
+
+def ep(k)
+  k = Integer(k)
+  
+  xj = generate_random(k)
+  yj = generate_random(k+1)
+  
+  t = xj * xj + yj * yj
+  
+  if t <= 1
+    xk = xj * Math.sqrt(-2 * Math.log(t) / t)
+    yk = yj * Math.sqrt(-2 * Math.log(t) / t)
+    
+    max = [xk.abs, yk.abs].max
+    l = max.floor
+    puts l + " " + xk + " " + yk
+  end
+end
+
+loop {
+  input = STDIN.gets
+  break if input.nil?
+  start, fin = input.chomp.split
+  start = Integer(start)
+  fin = Integer(fin)
+  current = start
+  loop {
+    ep(current)
+    current = current + 2
+    break if current > fin
+  }
+}
+
+BAZ
+
+    red_code = <<BAZ
+#!/usr/local/bin/ruby -w
+# Programmer: Chris Bunch
+# reducer-ruby.rb: Solves part of the EP parallel benchmark via the 
+# MapReduce framework as follows:
+# Input: list [l, X_k, Y_k]
+# Output: [l, sum(X_k), sum(Y_k)]
+
+current_l = nil
+
+x_count = 0
+y_count = 0
+
+sum_x = 0.0
+sum_y = 0.0
+
+loop {
+  input = STDIN.gets
+  break if input.nil?
+  l, x, y = input.chomp.split
+  l = Integer(l)
+  x = Float(x)
+  y = Float(y)
+  
+  current_l = l if current_l.nil?
+  
+  if l != current_l
+    puts "bucket = " + current_l + ", |x| = " + x_count + ", |y| = " + y_count
+    current_l = l
+    x_count = 0
+    y_count = 0
+  end
+  
+  sum_x = sum_x + x
+  sum_y = sum_y + y
+
+  abs_x = x.abs
+  abs_y = y.abs
+
+  if abs_x > abs_y
+    x_count = x_count + 1
+  else
+    y_count = y_count + 1 
+  end
+}
+
+puts "bucket = " + current_l + ", |x| = " + x_count + ", |y| = " + y_count
+puts "sum x = " + sum_x + ", sum y = " + sum_y
+
+BAZ
+
+    contents = TestHelper.get_random_alphanumeric(1024)
+    folder = "ep-#{TestHelper.get_random_alphanumeric}"
+
+    input_name = "input"
+    map_source = "map.rb"
+    red_source = "reduce.rb"
+
+    tmp_folder = "/tmp/#{folder}"
+    FileUtils.mkdir_p(tmp_folder)
+    compiled = "#{tmp_folder}-compiled"
+    compiled_code = "#{compiled}/HelloWorld"
+
+    local_input = "#{tmp_folder}/#{input_name}"
+    local_map = "#{tmp_folder}/#{map_source}"
+    local_red = "#{tmp_folder}/#{red_source}"
+
+    TestHelper.write_file(local_input, input)
+    TestHelper.write_file(local_map, map_code)
+    TestHelper.write_file(local_red, red_code)
+
+    remote_input = TestHelper.get_output_location("#{folder}-input", storage)
+    remote_map = TestHelper.get_output_location("#{folder}-map.rb", storage, notxt=true)
+    remote_red = TestHelper.get_output_location("#{folder}-reduce.rb", storage, notxt=true)
+    remote_output = TestHelper.get_output_location("#{folder}-output", storage)
+
+    put_file_in_storage(local_input, remote_input, storage)
+    put_file_in_storage(local_map, remote_map, storage)
+    put_file_in_storage(local_red, remote_red, storage)
+
+    start_mr_streaming_code(remote_input, remote_output, remote_map, remote_red, storage)
+    get_mr_output(remote_output, expected_output, storage)
+
+    FileUtils.rm_rf(local_input)
+    FileUtils.rm_rf(local_map)
+    FileUtils.rm_rf(local_red)
+  end
+
+  def put_file_in_storage(local, remote, storage)
+    params = {
+      :type => "input",
+      :local => local,
+      :remote => remote
+    }.merge(TestHelper.get_storage_params(storage))
+
+    input_result = neptune(params)
+
+    msg = "We were unable to store a file in the database. We " +
+      " got back this: #{msg}"
+    assert(input_result, msg)
+  end
+
+  def start_mr_streaming_code(input, output, map, reduce, storage)
+    params = {
+      :type => "mapreduce",
+      :input  => input,
+      :output => output,
+      :map => map,
+      :reduce => reduce,
+      :nodes_to_use => 1
+    }.merge(TestHelper.get_storage_params(storage))
+
+    status = nil
+
+    loop {
+      status = neptune(params)
+      if status[:msg] =~ /not enough free nodes/
+        puts status[:msg]
+      else
+        break
+      end
+      sleep(5)
+    }
+
+    msg = "Your job was not started successfully. The failure message " +
+      "reported was #{status[:msg]}"
+    assert_equal(status[:result], :success, msg)
+  end
+
+  def get_mr_output(output, expected, storage)
+    result = TestHelper.get_job_output(output, storage)
+
+    msg = "The MapReduce job you ran did not return the expected result. " +
+      "We expected to see [#{expected}] but instead saw [#{result}]"
+    assert_equal(result, expected, msg)
+  end
+end 
+

data/test/tc_mpi.rb

@@ -0,0 +1,124 @@
+
+class TestMPI < Test::Unit::TestCase
+  def test_hello_world_code
+    num_procs = [1]
+
+    STORAGE_TYPES.each { |storage|
+      num_procs.each { |p|
+        run_hello_world_code(storage, p)
+      }
+    }
+  end
+
+  def run_hello_world_code(storage, num_procs)
+    expected_output = "0: We have 1 processors"
+    ring_code = <<BAZ
+ /*
+  "Hello World" MPI Test Program
+ */
+ #include <mpi.h>
+ #include <stdio.h>
+ #include <string.h>
+ 
+ #define BUFSIZE 128
+ #define TAG 0
+ 
+ int main(int argc, char *argv[])
+ {
+   char idstr[32];
+   char buff[BUFSIZE];
+   int numprocs;
+   int myid;
+   int i;
+   MPI_Status stat;
+ 
+   MPI_Init(&argc,&argv); /* all MPI programs start with MPI_Init; all 'N' processes exist thereafter */
+   MPI_Comm_size(MPI_COMM_WORLD,&numprocs); /* find out how big the SPMD world is */
+   MPI_Comm_rank(MPI_COMM_WORLD,&myid); /* and this processes' rank is */
+ 
+   /* At this point, all programs are running equivalently, the rank distinguishes
+      the roles of the programs in the SPMD model, with rank 0 often used specially... */
+   if(myid == 0)
+   {
+     printf("%d: We have %d processors", myid, numprocs);
+     for(i=1;i<numprocs;i++)
+     {
+       sprintf(buff, "Hello %d! ", i);
+       MPI_Send(buff, BUFSIZE, MPI_CHAR, i, TAG, MPI_COMM_WORLD);
+     }
+     for(i=1;i<numprocs;i++)
+     {
+       MPI_Recv(buff, BUFSIZE, MPI_CHAR, i, TAG, MPI_COMM_WORLD, &stat);
+       printf("%d: %s", myid, buff);
+     }
+   }
+   else
+   {
+     /* receive from rank 0: */
+     MPI_Recv(buff, BUFSIZE, MPI_CHAR, 0, TAG, MPI_COMM_WORLD, &stat);
+     sprintf(idstr, "Processor %d ", myid);
+     strncat(buff, idstr, BUFSIZE-1);
+     strncat(buff, "reporting for duty", BUFSIZE-1);
+     /* send to rank 0: */
+     MPI_Send(buff, BUFSIZE, MPI_CHAR, 0, TAG, MPI_COMM_WORLD);
+   }
+ 
+   MPI_Finalize(); /* MPI Programs end with MPI Finalize; this is a weak synchronization point */
+   return 0;
+ }
+
+BAZ
+
+    contents = TestHelper.get_random_alphanumeric(1024)
+    folder = "hello-world-#{TestHelper.get_random_alphanumeric}"
+    source = "HelloWorld.c"
+
+    tmp_folder = "/tmp/#{folder}"
+    FileUtils.mkdir_p(tmp_folder)
+    compiled = "#{tmp_folder}-compiled"
+    compiled_code = "#{compiled}/HelloWorld"
+
+    local = "#{tmp_folder}/#{source}"
+    TestHelper.write_file(local, ring_code)
+
+    output = TestHelper.get_output_location(folder, storage)
+
+    compile_mpi_code(tmp_folder, source, compiled)
+    start_mpi_code(compiled_code, num_procs, output, storage)
+    get_mpi_output(output, expected_output, storage)
+
+    FileUtils.rm_rf(tmp_folder)
+    FileUtils.rm_rf(compiled)
+  end
+
+  def compile_mpi_code(location, main_file, compiled)
+    std_out, std_err = TestHelper.compile_code(location, main_file, compiled)
+
+    make = "mpicc HelloWorld.c -o HelloWorld -Wall"
+    msg = "The MPI code did not compile as expected. It should have " +
+      "compiled with the command [#{make}] instead of [#{std_out}]."
+    assert_equal(std_out, make, msg)
+
+    msg = "The MPI code did not compile successfully. It reported " +
+      "the following error: #{std_err}"
+    assert_nil(std_err, msg)
+  end
+
+  def start_mpi_code(code_location, num_procs, output, storage)
+    params = { :procs_to_use => num_procs }
+    status = TestHelper.start_job("mpi", code_location, output, storage, params)
+
+    msg = "Your job was not started successfully. The failure message " +
+      "reported was #{status[:msg]}"
+    assert_equal(status[:result], :success, msg)
+  end
+
+  def get_mpi_output(output, expected, storage)
+    result = TestHelper.get_job_output(output, storage)
+
+    msg = "The MPI job you ran did not return the expected result. " +
+      "We expected to see [#{expected}] but instead saw [#{result}]"
+    assert_equal(result, expected, msg)
+  end
+end 
+