neptune 0.0.6 → 0.0.7

data/test/tc_storage.rb

@@ -0,0 +1,45 @@
+
+class TestStorage < Test::Unit::TestCase
+  def test_in_out
+    STORAGE_TYPES.each { |storage|
+      run_in_out(storage)
+    }
+  end
+
+  def run_in_out(storage)
+    contents = TestHelper.get_random_alphanumeric(1024)
+    suffix = "neptune-testfile-#{TestHelper.get_random_alphanumeric}"
+    local = "/tmp/#{suffix}"
+    TestHelper.write_file(local, contents)
+    
+    run_input(local, suffix, storage)
+    run_output(contents, suffix, storage)
+
+    FileUtils.rm_rf(local)
+  end
+
+  def run_input(local, suffix, storage)
+    params = {
+      :type => "input",
+      :local => local,
+      :remote => TestHelper.get_output_location(suffix, storage)
+    }.merge(TestHelper.get_storage_params(storage))
+
+    input_result = neptune(params)
+
+    msg = "We were unable to store a file in the database. We " +
+      " got back this: #{msg}"
+    assert(input_result, msg)
+  end
+
+  def run_output(local, suffix, storage)
+    output = TestHelper.get_output_location(suffix, storage)
+    remote = TestHelper.get_job_output(output, storage)
+
+    msg = "We were unable to verify that the remote file matches the " +
+      "local version. The local copy's contents are: " +
+      "[#{local}], while the remote copy's contents are [#{remote}]."
+    assert_equal(local, remote, msg)
+  end
+end 
+

data/test/tc_upc.rb

@@ -0,0 +1,75 @@
+
+class TestUPC < Test::Unit::TestCase
+  def test_hello_world_code
+    STORAGE_TYPES.each { |storage|
+      run_hello_world_code(storage)
+    }
+  end
+
+  def run_hello_world_code(storage)
+    expected_output = "Hello from thread 0"
+    ring_code = <<BAZ
+#include <upc_relaxed.h>
+#include <stdio.h>
+
+int main() {
+  printf("Hello from thread %i/%i", MYTHREAD, THREADS);
+  upc_barrier;
+  return 0;
+}
+
+BAZ
+
+    contents = TestHelper.get_random_alphanumeric(1024)
+    folder = "hello-world-#{TestHelper.get_random_alphanumeric}"
+    source = "HelloWorld.c"
+
+    tmp_folder = "/tmp/#{folder}"
+    FileUtils.mkdir_p(tmp_folder)
+    compiled = "#{tmp_folder}-compiled"
+    compiled_code = "#{compiled}/HelloWorld"
+
+    local = "#{tmp_folder}/#{source}"
+    TestHelper.write_file(local, ring_code)
+
+    output = TestHelper.get_output_location(folder, storage)
+
+    compile_upc_code(tmp_folder, source, compiled)
+    start_upc_code(compiled_code, output, storage)
+    get_upc_output(output, expected_output, storage)
+
+    FileUtils.rm_rf(tmp_folder)
+    FileUtils.rm_rf(compiled)
+  end
+
+  def compile_upc_code(location, main_file, compiled)
+    std_out, std_err = TestHelper.compile_code(location, main_file, compiled)
+
+    make = "/usr/local/berkeley_upc-2.12.1/upcc --network=mpi -o HelloWorld HelloWorld.c"
+    msg = "The UPC code did not compile as expected. It should have " +
+      "compiled with the command [#{make}] instead of [#{std_out}]."
+    assert_equal(std_out, make, msg)
+
+    msg = "The UPC code did not compile successfully. It reported " +
+      "the following error: #{std_err}"
+    assert_nil(std_err, msg)
+  end
+
+  def start_upc_code(code_location, output, storage)
+    status = TestHelper.start_job("upc", code_location, output, storage)
+
+    msg = "Your job was not started successfully. The failure message " +
+      "reported was #{status[:msg]}"
+    assert_equal(status[:result], :success, msg)
+  end
+
+  def get_upc_output(output, expected, storage)
+    result = TestHelper.get_job_output(output, storage)
+
+    msg = "The UPC job you ran did not return the expected result. " +
+      "We expected to see [#{expected}] but instead saw [#{result}]"
+    success = result.include?(expected)
+    assert(success, msg)
+  end
+end 
+

data/test/tc_x10.rb

@@ -0,0 +1,94 @@
+
+class TestX10 < Test::Unit::TestCase
+  def test_ring_code
+    STORAGE_TYPES.each { |storage|
+      run_ring_code(storage)
+    }
+  end
+
+  def run_ring_code(storage)
+    expected_output = "All done!"
+    ring_code = <<BAZ
+import x10.lang.Math;
+import x10.util.Timer;
+
+public class Ring {
+
+  static val NUM_MESSAGES = 1;
+
+  // A global datastructure with one integer cell per place
+  static A = PlaceLocalHandle.make[Cell[Long]](Dist.makeUnique(), ()=>new Cell[Long](-1));
+
+  public static def send (msg:Long, depth:Int) {
+    A()() = msg;
+    if (depth==0) return;
+    async at (here.next()) send(msg, depth-1);
+  }
+
+  public static def main(args:Array[String](1)) {
+
+    val startTime = Timer.milliTime();
+    finish send(42L, NUM_MESSAGES * Place.MAX_PLACES);
+    val endTime = Timer.milliTime();
+
+    val totalTime = (endTime - startTime) / 1000.0;
+
+    Console.OUT.printf("#{expected_output}");
+  }
+}
+
+BAZ
+
+    contents = TestHelper.get_random_alphanumeric(1024)
+    folder = "ring-#{TestHelper.get_random_alphanumeric}"
+    source = "Ring.x10"
+
+    tmp_folder = "/tmp/#{folder}"
+    FileUtils.mkdir_p(tmp_folder)
+    compiled = "#{tmp_folder}-compiled"
+    compiled_code = "#{compiled}/Ring"
+
+    local = "#{tmp_folder}/#{source}"
+    TestHelper.write_file(local, ring_code)
+
+    output = TestHelper.get_output_location(folder, storage)
+
+    compile_x10_code(tmp_folder, source, compiled)
+    start_x10_code(compiled_code, output, storage)
+    get_x10_output(output, expected_output, storage)
+
+    FileUtils.rm_rf(tmp_folder)
+    FileUtils.rm_rf(compiled)
+  end
+
+  def compile_x10_code(location, main_file, compiled)
+    std_out, std_err = TestHelper.compile_code(location, main_file, compiled)
+
+    make = "/usr/local/x10/x10.dist/bin/x10c++ -x10rt mpi -o Ring Ring.x10"
+    msg = "The X10 Ring code did not compile as expected. It should have " +
+      "compiled with the command [#{make}] instead of [#{std_out}]."
+    assert_equal(std_out, make, msg)
+
+    msg = "The X10 Ring code did not compile successfully. It reported " +
+      "the following error: #{std_err}"
+    compile_success = !std_err.include?("error")
+    assert(compile_success, msg)
+  end
+
+  def start_x10_code(code_location, output, storage)
+    status = TestHelper.start_job("x10", code_location, output, storage)
+
+    msg = "Your job was not started successfully. The failure message " +
+      "reported was #{status[:msg]}"
+    assert_equal(status[:result], :success, msg)
+  end
+
+  def get_x10_output(output, expected, storage)
+    result = TestHelper.get_job_output(output, storage)
+
+    msg = "The X10 job you ran did not return the expected result. " +
+      "We expected to see [#{expected}] but instead saw [#{result}]"
+    assert_equal(result, expected, msg)
+  end
+end 
+

data/test/test_helper.rb

@@ -0,0 +1,137 @@
+#!/usr/bin/ruby -w
+# Programmer: Chris Bunch
+
+module TestHelper
+  def self.compile_code(location, main_file, compiled_location)
+    result = neptune(
+      :type => "compile",
+      :code => location,
+      :main => main_file,
+      :output => "/baz",
+      :copy_to => compiled_location
+    )
+
+    puts "standard out is #{result[:out]}"
+    puts "standard err is #{result[:err]}"
+
+    return result[:out], result[:err]
+  end
+
+  def self.start_job(type, code_location, output, storage, extras={})
+    params = {
+      :type => type,
+      :code => code_location,
+      :output => output,
+      :nodes_to_use => 1
+    }.merge(TestHelper.get_storage_params(storage)).merge(extras)
+
+    status = nil
+
+    loop {
+      status = neptune(params)
+      if status[:msg] =~ /not enough free nodes/
+        puts status[:msg]
+      else
+        break
+      end
+      sleep(5)
+    }
+
+    return status
+  end
+
+  def self.get_job_output(output, storage)
+    result = ""
+
+    params = {
+      :type => "output",
+      :output => output
+    }.merge(TestHelper.get_storage_params(storage))
+
+    loop {
+      result = neptune(params)
+
+      break if result != "error: output does not exist"
+      puts "Waiting for job to complete..."
+      sleep(30)
+    }
+
+    return result
+  end
+
+  def self.get_output_location(file, storage="appdb", notxt=false)
+    output = "/neptune"
+
+    if storage == "walrus"
+      output << "_"
+    else
+      output << "-"
+    end
+
+    output << "testbin/#{file}"
+    if !notxt
+      output << ".txt"
+    end
+
+    return output
+  end
+
+  def self.get_storage_params(storage)
+    if storage == "gstorage"
+      return {
+        :storage => "gstorage",
+        :EC2_ACCESS_KEY => ENV['GSTORAGE_ACCESS_KEY'],
+        :EC2_SECRET_KEY => ENV['GSTORAGE_SECRET_KEY'],
+        :S3_URL => ENV['GSTORAGE_URL']
+      }
+    elsif storage == "s3"
+      return {
+        :storage => "s3",
+        :EC2_ACCESS_KEY => ENV['S3_ACCESS_KEY'],
+        :EC2_SECRET_KEY => ENV['S3_SECRET_KEY'],
+        :S3_URL => ENV['S3_URL']
+      }
+    elsif storage == "walrus"
+      return {
+        :storage => "s3",
+        :EC2_ACCESS_KEY => ENV['WALRUS_ACCESS_KEY'],
+        :EC2_SECRET_KEY => ENV['WALRUS_SECRET_KEY'],
+        :S3_URL => ENV['WALRUS_URL']
+      }
+    elsif storage == "appdb"
+      return {}
+      # nothing special to do
+    else
+      abort "Storage specified was not an acceptable value: #{storage}"
+    end
+  end
+
+  def self.write_file(location, contents)
+    File.open(location, "w+") { |file| file.write(contents) }
+  end
+
+  def self.read_file(location)
+    File.open(location) { |f| f.read.chomp! }
+  end
+
+  def self.get_random_alphanumeric(length=10)
+    random = ""
+    possible = "0123456789abcdefghijklmnopqrstuvxwyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    possibleLength = possible.length
+
+    length.times { |index|
+      random << possible[rand(possibleLength)]
+    }
+
+    return random
+  end
+
+  def self.is_appscale_running?(ip)
+    begin
+      Net::HTTP.get_response(URI.parse("http://#{ip}"))
+      return true
+    rescue Exception
+      return false
+    end
+  end
+end

data/test/ts_neptune.rb

@@ -0,0 +1,41 @@
+#STORAGE_TYPES = ["gstorage"]
+STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"]
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'neptune'
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "test")
+require 'test_helper'
+
+REQUIRED_CREDS = %w{ APPSCALE_HEAD_NODE
+GSTORAGE_ACCESS_KEY GSTORAGE_SECRET_KEY GSTORAGE_URL 
+S3_ACCESS_KEY S3_SECRET_KEY S3_URL 
+WALRUS_ACCESS_KEY WALRUS_SECRET_KEY WALRUS_URL }
+
+require 'test/unit'
+
+REQUIRED_CREDS.each { |cred|
+  msg = "The environment variable #{cred} was not set. Please " +
+    "set it and try again."
+  abort(msg) if ENV[cred].nil?
+}
+
+APPSCALE_HEAD_NODE_IP = ENV['APPSCALE_HEAD_NODE']
+msg = "AppScale is not currently running at " +
+  "#{APPSCALE_HEAD_NODE_IP}. Please start AppScale and try again."
+abort(msg) unless TestHelper.is_appscale_running?(APPSCALE_HEAD_NODE_IP)
+
+TEST_ALL_WORKING = true
+
+if TEST_ALL_WORKING
+  require 'tc_c'
+  require 'tc_dfsp'
+  require 'tc_dwssa'
+  require 'tc_erlang'
+  require 'tc_mpi'
+  require 'tc_storage'
+  require 'tc_upc'
+  require 'tc_x10'
+else
+  require 'tc_mapreduce'
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: neptune
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 7
+  version: 0.0.7
 platform: ruby
 authors: 
 - Chris Bunch
@@ -15,7 +15,7 @@ autorequire: neptune
 bindir: bin
 cert_chain: []
 
-date: 2011-03-28 00:00:00 -07:00
+date: 2011-04-02 00:00:00 -07:00
 default_executable: neptune
 dependencies: []
 
@@ -40,6 +40,7 @@ files:
 - doc/AppControllerClient.html
 - doc/LICENSE.html
 - doc/lib/job_rb.html
+- doc/lib/neptune_rb.html
 - doc/lib/app_controller_client_rb.html
 - doc/lib/common_functions_rb.html
 - doc/index.html
@@ -105,6 +106,17 @@ files:
 - lib/common_functions.rb
 - lib/app_controller_client.rb
 - lib/neptune.rb
+- test/tc_c.rb
+- test/tc_dfsp.rb
+- test/test_helper.rb
+- test/tc_mpi.rb
+- test/tc_dwssa.rb
+- test/tc_erlang.rb
+- test/tc_storage.rb
+- test/tc_x10.rb
+- test/tc_upc.rb
+- test/ts_neptune.rb
+- test/tc_mapreduce.rb
 - README
 - LICENSE
 has_rdoc: true