neptune 0.1.1 → 0.1.2

data/samples/x10/NQueensDist.x10

@@ -0,0 +1,112 @@
+/*
+ *  This file is part of the X10 project (http://x10-lang.org).
+ *
+ *  This file is licensed to You under the Eclipse Public License (EPL);
+ *  You may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *      http://www.opensource.org/licenses/eclipse-1.0.php
+ *
+ *  (C) Copyright IBM Corporation 2006-2010.
+ */
+
+import x10.io.Console;
+
+/**
+ * A distributed version of NQueens. Runs over NUM_PLACES.
+ * Identical to NQueensPar, except that it runs over multiple placs.
+ * Converted to 2.1 on 9/1/2010
+ */
+public class NQueensDist {
+    public static val expectedSolutions =
+        [0, 1, 0, 0, 2, 10, 4, 40, 92, 352, 724, 2680, 14200, 73712, 365596, 2279184, 14772512];
+
+    val N:Int;
+    val P:Int;
+    val results:DistArray[Int](1);
+
+    def this(N:Int, P:Int) { 
+	   this.N=N; 
+	   this.P=P;
+	   this.results = DistArray.make[Int](Dist.makeUnique(), (Point)=>0);
+    }
+    def start() {
+        new Board().search();
+    }
+    def run():Int {
+	   finish start();
+	   val result = results.reduce(Int.+,0);
+	   return result;
+    }
+
+    /**
+     * Return an array of P regions, which together block divide the 1-D region R.
+     */
+    public static def block(R: Region(1){rect}, P: Int): Array[Region(1){rect}](1) = {
+        assert P >= 0;
+        val low = R.min()(0), high = R.max()(0), count = high-low+1;
+        val baseSize = count/P, extra = count - baseSize*P;
+        new Array[Region(1){rect}](P, (i:int):Region(1){rect} => {
+            val start = low+i*baseSize+ (i < extra? i:extra);
+            start..start+baseSize+(i < extra?0:-1)
+        })
+    }
+
+    class Board {
+        val q: Array[Int](1);
+        def this() {
+            q = new Array[Int](0);
+        }
+        def this(old:Array[Int](1), newItem:Int) {
+            val n = old.size;
+            q = new Array[Int](n+1, (i:int)=> (i < n? old(i) : newItem));
+        }
+        def safe(j: int) {
+            val n = q.size;
+            for ([k] in 0..n-1) {
+                if (j == q(k) || Math.abs(n-k) == Math.abs(j-q(k)))
+                    return false;
+            }
+            return true;
+        }
+        /** Search for all solutions in parallel, on finding
+         * a solution update nSolutions.
+         */
+        def search(R: Region(1){rect}) {
+            for ([k] in R)
+                if (safe(k))
+                    new Board(q, k).search();
+        }
+
+        def search()  {
+            if (q.size == N) {
+                atomic NQueensDist.this.results(here.id)++;
+                return;
+            }
+            if (q.size == 0) {
+                val R = block(0..N-1, P);
+                ateach ([q] in Dist.makeUnique())
+                   // copy of this made across the at divide
+                  search(R(q));
+            } else search(0..N-1);
+        }
+    }
+
+    public static def main(args:Array[String](1))  {
+        val n = args.size > 0 ? Int.parse(args(0)) : 16;
+        println("N=" + n);
+        //warmup
+        //finish new NQueensPar(12, 1).start();
+        val P = Place.MAX_PLACES;
+	    val nq = new NQueensDist(n,P);
+	    var start:Long = -System.nanoTime();
+	    val answer = nq.run();
+	    val result = answer==expectedSolutions(n);
+	    start += System.nanoTime();
+	    start /= 1000000;
+	    println("NQueensPar " + nq.N + "(P=" + P +
+		   ") has " + answer + " solutions" +
+		   (result? " (ok)." : " (wrong).") + "time=" + start + "ms");
+    }
+
+    static def println(s:String) = Console.OUT.println(s);
+}

data/samples/x10/compile_x10_nqueens.rb

@@ -0,0 +1,7 @@
+neptune :type => "compile",
+  #:keyname => "cluster",
+  :code => "NQueensDist.x10",
+  :output => "/baz",
+  :lang => "x10",
+  :copy_to => "NQueensCompiled"
+

data/samples/x10/compile_x10_ring.rb

@@ -0,0 +1,12 @@
+result = neptune (
+  :type => "compile",
+  #:keyname => "cluster",
+  :code => "ring",
+  :main => "Ring.x10",
+  :output => "/baz",
+  :copy_to => "ring-compiled"
+)
+
+puts "out = #{result[:out]}"
+puts "err = #{result[:err]}"
+

data/samples/x10/get_x10_output.rb

@@ -0,0 +1,8 @@
+
+result = neptune ( 
+  :type => "output",
+  #:keyname => "neptune",
+  :output => "/baz.txt"
+)
+
+puts result

data/samples/x10/ring/Makefile

@@ -0,0 +1,3 @@
+all:
+	/usr/local/x10/x10.dist/bin/x10c++ -x10rt mpi -o Ring Ring.x10
+

data/samples/x10/ring/Ring.x10

@@ -0,0 +1,28 @@
+import x10.lang.Math;
+import x10.util.Timer;
+
+public class Ring {
+
+  static val NUM_MESSAGES = 10;
+
+  // A global datastructure with one integer cell per place
+  static A = PlaceLocalHandle.make[Cell[Long]](Dist.makeUnique(), ()=>new Cell[Long](-1));
+
+  public static def send (msg:Long, depth:Int) {
+    A()() = msg;
+    if (depth==0) return;
+    async at (here.next()) send(msg, depth-1);
+  }
+
+  public static def main(args:Array[String](1)) {
+
+    val startTime = Timer.milliTime();
+    finish send(42L, NUM_MESSAGES * Place.MAX_PLACES);
+    val endTime = Timer.milliTime();
+
+    val totalTime = (endTime - startTime) / 1000.0;
+
+    Console.OUT.printf("It took %f seconds\n", totalTime);
+  }
+}
+

data/samples/x10/ring/RingOld.x10

@@ -0,0 +1,68 @@
+import x10.lang.Math;
+import x10.util.Timer;
+
+public class Ring {
+
+  static val NUM_MESSAGES = 2;
+  static val P = Place.MAX_PLACES;
+
+  /* Can't do message passing directly - so we'll use shared memory
+   * to achieve the same goal. The basic idea is to give each processor
+   * a single integer and have them wait for it to change values */
+
+  static val R <: Region = (0..P-1);
+  static val D <: Dist = Dist.makeBlock(R);	
+  static val F <: (Point(1)) => Int = ([i]:Point(1)) => -1;	
+  static val A <: DistArray[Int] = DistArray.make[Int](D, F);
+
+  public static def send(target:Int, value:Int) {
+    at (Place.place(target)) {
+      A(target) = value;
+    }
+
+    return;
+  }
+
+  public static def recv(from:Int, value:Int) {
+    while (A(here.id) != value) {
+      Activity.sleep(10l);
+    }
+
+    return;
+  }
+	
+  public static def main(args:Array[String](1)) {
+    val startTime = Timer.milliTime();
+
+    for (var index : Int = 0; index < NUM_MESSAGES; index++) {
+      val i = index;
+      finish for (p in Place.places()) {
+        async at (p) {
+          if (p.id == 0) { 
+            Console.OUT.printf("master is sending message to node 1\n");
+            Ring.send(1, i);
+
+            Console.OUT.printf("master is waiting for message from node %d\n", P - 1);
+            Ring.recv(P - 1, i);
+          } else {
+            Ring.recv(p.id - 1, i);
+
+            if (p.id + 1 == P) {
+              Console.OUT.printf("node %d is sending a message to node 0\n", p.id);
+              Ring.send(0, i);
+            } else {
+              Console.OUT.printf("node %d is sending a message to node %d\n", p.id, p.id + 1);
+              Ring.send(p.id + 1, i);
+            }
+          }
+        }
+      }
+    }
+
+    val endTime = Timer.milliTime();
+    val totalTime = (endTime - startTime) / 1000.0;
+    	   
+    Console.OUT.printf("It took %f seconds\n", totalTime);
+  }
+}
+

data/samples/x10/run_x10_nqueens.rb

@@ -0,0 +1,6 @@
+neptune :type => "mpi",
+  #:keyname = "cluster",
+  :code => "NQueensCompiled",
+  :output => "/baz.txt",
+  :nodes_to_use => 1
+

data/samples/x10/run_x10_powermethod.rb

@@ -0,0 +1,7 @@
+neptune :type => "mpi",
+  :keyname => "cluster",
+  #:input => "baz",
+  #:output => "x10-output.txt",
+  :code => "MyPowerMethod",
+  :nodes_to_use => 16
+

data/samples/x10/run_x10_ring.rb

@@ -0,0 +1,6 @@
+neptune :type => "mpi",
+  #:keyname = "cluster",
+  :code => "ring-compiled/Ring",
+  :output => "/baz.txt",
+  :nodes_to_use => 4
+

data/test/{tc_c.rb → integration/tc_c.rb}

@@ -1,8 +1,8 @@
 
-$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+$:.unshift File.join(File.dirname(__FILE__), "..", "..", "lib")
 require 'neptune'
 
-$:.unshift File.join(File.dirname(__FILE__), "..", "test")
+$:.unshift File.join(File.dirname(__FILE__), "..", "test", "integration")
 require 'test_helper'
 
 require 'test/unit'

data/test/{ts_neptune.rb → integration/ts_neptune.rb}

@@ -1,9 +1,9 @@
 STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"] - ["appdb"]
 
-$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+$:.unshift File.join(File.dirname(__FILE__), "..", "..", "lib")
 require 'neptune'
 
-$:.unshift File.join(File.dirname(__FILE__), "..", "test")
+$:.unshift File.join(File.dirname(__FILE__), "..", "test", "integration")
 require 'test_helper'
 
 REQUIRED_CREDS = %w{ APPSCALE_HEAD_NODE

data/test/unit/test_app_controller_client.rb

@@ -0,0 +1,106 @@
+# Programmer: Chris Bunch (cgb@cs.ucsb.edu)
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "..", "lib")
+require 'app_controller_client'
+
+require 'test/unit'
+
+class FakeConnection
+  # Since all the methods we're faking take the same arguments and have
+  # the same semantics (return true or abort), just cover it all in one place.
+  def method_missing(id, *args, &block)
+    method_names = ["neptune_start_job", "neptune_put_input"] +
+      ["neptune_get_output", "neptune_get_acl", "neptune_set_acl"] +
+      ["neptune_compile_code"]
+
+    if method_names.include?(id.to_s)
+      job_data = args[0]
+      if job_data.include?("OK")
+        return true
+      else
+        return "Error:"
+      end
+    else
+      super
+    end
+  end
+end
+
+class TestAppControllerClient < Test::Unit::TestCase
+  def setup
+    @client = AppControllerClient.new("localhost", "secret")
+    @client.conn = FakeConnection.new()
+
+    @job_data_ok = ["OK"]
+    @job_data_err = ["ERR"]
+  end
+
+  def test_make_call
+    no_timeout = -1
+    retry_on_exception = true
+    no_retry_on_exception = false
+
+    call_number = 0
+    assert_nothing_raised(SystemExit) {
+      @client.make_call(no_timeout, retry_on_exception) {
+        call_number += 1
+        case call_number
+        when 1
+          raise Errno::ECONNREFUSED
+        when 2
+          raise OpenSSL::SSL::SSLError
+        when 3
+          raise Exception
+        else
+          0
+        end
+      }
+    }
+
+    assert_raise(SystemExit) {
+      @client.make_call(no_timeout, no_retry_on_exception) {
+        raise Errno::ECONNREFUSED
+      }
+    }
+
+    assert_raise(SystemExit) {
+      @client.make_call(no_timeout, no_retry_on_exception) {
+        raise Exception
+      }
+    }
+
+  end
+
+  # The remaining tests are identical since the implemented methods are all
+  # extremely similar - these methods all make a SOAP call and return the
+  # result unless it has 'Error:' in it. If it does, it aborts execution.
+  def test_start_neptune_job
+    assert(@client.start_neptune_job(@job_data_ok))
+    assert_raise(SystemExit) { @client.start_neptune_job(@job_data_err) }
+  end
+
+  def test_put_input
+    assert(@client.put_input(@job_data_ok))
+    assert_raise(SystemExit) { @client.put_input(@job_data_err) }
+  end
+
+  def test_get_output
+    assert(@client.get_output(@job_data_ok))
+    assert_raise(SystemExit) { @client.get_output(@job_data_err) }
+  end
+
+  def test_get_acl
+    assert(@client.get_acl(@job_data_ok))
+    assert_raise(SystemExit) { @client.get_acl(@job_data_err) }
+  end
+
+  def test_set_acl
+    assert(@client.set_acl(@job_data_ok))
+    assert_raise(SystemExit) { @client.set_acl(@job_data_err) }
+  end
+
+  def test_compile_code
+    assert(@client.compile_code(@job_data_ok))
+    assert_raise(SystemExit) { @client.compile_code(@job_data_err) }
+  end
+end

data/test/unit/test_common_functions.rb

@@ -0,0 +1,106 @@
+# Programmer: Chris Bunch (cgb@cs.ucsb.edu)
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "..", "lib")
+require 'common_functions'
+
+require 'test/unit'
+
+SECRET = "hey-its-a-secret"
+
+module FakeCommonFunctions
+  def self.get_from_yaml(a, b, c)
+  end
+
+  def self.scp_file(a, b, c, d, e)
+  end
+end
+
+class FakeFile
+  @@exists_checks_done = 0
+
+  def self.expand_path(path)
+    return path
+  end
+
+  def self.exists?(name)
+    if name.include?("OK") or name.include?("BAD-TAG") or name.include?("FAIL")
+      return true
+    elsif name.include?("retval")
+      # The first time around, tell the caller that the file didn't exist so
+      # that it sleeps. The next time around, tell it that the file does exist.
+      if @@exists_checks_done.zero?
+        @@exists_checks_done += 1
+        return false
+      else
+        return true
+      end
+    else
+      return false
+    end
+  end
+
+  def self.open(name, mode=nil)
+    if name.include?("retval")
+      return "0\n"
+    else
+      return "1\n"
+    end
+  end
+end
+
+module FakeFileUtils
+  def self.rm_f(name)
+    # Do nothing - faking out File means we don't have extra files to clean up
+  end
+end
+
+module FakeKernel
+  def `(command)
+    # Do nothing - we don't need the side-effects from exec'ing a command
+  end
+
+  #def sleep(time)
+    # Do nothing - we don't actually need to sleep the current thread
+  #end
+end
+
+module FakeYAML
+  def self.load_file(filename)
+    if filename.include?("OK")
+      return { :secret => SECRET, :shadow => SECRET }
+    elsif filename.include?("BAD-TAG")
+      return {}
+    else
+      raise ArgumentError
+    end
+  end
+end
+
+class TestCommonFunctions < Test::Unit::TestCase
+  def test_scp_to_shadow
+    assert_nothing_raised(Exception) {
+      CommonFunctions.scp_to_shadow("OK", "OK", "OK", "OK",
+        file=FakeFile,
+        get_from_yaml=FakeCommonFunctions.method(:get_from_yaml),
+        scp_file=FakeCommonFunctions.method(:scp_file))
+    }
+  end
+
+  def test_get_secret_key
+    assert_equal(SECRET, CommonFunctions.get_secret_key("OK", required=true,
+      FakeFile, FakeYAML))
+
+    assert_raise(SystemExit) { CommonFunctions.get_secret_key("BAD-TAG",
+      required=true, FakeFile, FakeYAML) }
+
+    assert_raise(SystemExit) { CommonFunctions.get_secret_key("FAIL",
+      required=true, FakeFile, FakeYAML) }
+    assert_nil(CommonFunctions.get_secret_key("FAIL", required=false, FakeFile,
+      FakeYAML))
+
+    assert_raise(SystemExit) {
+      CommonFunctions.get_secret_key("NOT-EXISTANT", required=true, FakeFile,
+        FakeYAML)
+    }
+  end
+end