RubyGems - neptune - Versions diffs - 0.0.1 - Mend

neptune 0.0.1

Files changed (79) hide show

data/LICENSE +39 -0
data/README +50 -0
data/bin/neptune +16 -0
data/doc/AppControllerClient.html +468 -0
data/doc/CommonFunctions.html +368 -0
data/doc/LICENSE.html +131 -0
data/doc/Object.html +431 -0
data/doc/README.html +135 -0
data/doc/bin/neptune.html +58 -0
data/doc/created.rid +7 -0
data/doc/images/brick.png +0 -0
data/doc/images/brick_link.png +0 -0
data/doc/images/bug.png +0 -0
data/doc/images/bullet_black.png +0 -0
data/doc/images/bullet_toggle_minus.png +0 -0
data/doc/images/bullet_toggle_plus.png +0 -0
data/doc/images/date.png +0 -0
data/doc/images/find.png +0 -0
data/doc/images/loadingAnimation.gif +0 -0
data/doc/images/macFFBgHack.png +0 -0
data/doc/images/package.png +0 -0
data/doc/images/page_green.png +0 -0
data/doc/images/page_white_text.png +0 -0
data/doc/images/page_white_width.png +0 -0
data/doc/images/plugin.png +0 -0
data/doc/images/ruby.png +0 -0
data/doc/images/tag_green.png +0 -0
data/doc/images/wrench.png +0 -0
data/doc/images/wrench_orange.png +0 -0
data/doc/images/zoom.png +0 -0
data/doc/index.html +90 -0
data/doc/js/darkfish.js +116 -0
data/doc/js/jquery.js +32 -0
data/doc/js/quicksearch.js +114 -0
data/doc/js/thickbox-compressed.js +10 -0
data/doc/lib/app_controller_client_rb.html +60 -0
data/doc/lib/common_functions_rb.html +68 -0
data/doc/lib/job_rb.html +55 -0
data/doc/scripts/appscale/add_appserver_rb.html +52 -0
data/doc/scripts/appscale/add_database_rb.html +52 -0
data/doc/scripts/appscale/add_loadbalancer_rb.html +52 -0
data/doc/scripts/appscale/add_slave_rb.html +52 -0
data/doc/scripts/cewssa/cewssa-output.html +104 -0
data/doc/scripts/cewssa/get_cewssa_output_rb.html +52 -0
data/doc/scripts/cewssa/run_cewssa_rb.html +52 -0
data/doc/scripts/cewssa/test_dfsp_rb.html +52 -0
data/doc/scripts/dfsp/get_dfsp_output_rb.html +52 -0
data/doc/scripts/dfsp/make_output_public_rb.html +52 -0
data/doc/scripts/dfsp/run_dfsp_rb.html +52 -0
data/doc/scripts/dfsp/see_acl_rb.html +52 -0
data/doc/scripts/dfsp/test_dfsp_rb.html +52 -0
data/doc/scripts/mapreduce/get_mapreduce_output_rb.html +52 -0
data/doc/scripts/mapreduce/input-10.html +114 -0
data/doc/scripts/mapreduce/input-30.html +114 -0
data/doc/scripts/mapreduce/input-7.html +104 -0
data/doc/scripts/mapreduce/map_rb.html +56 -0
data/doc/scripts/mapreduce/reduce_rb.html +56 -0
data/doc/scripts/mapreduce/run_java_mr_rb.html +52 -0
data/doc/scripts/mapreduce/run_mapreduce_rb.html +52 -0
data/doc/scripts/mpi/Makefile.html +125 -0
data/doc/scripts/mpi/get_mpi_output_rb.html +52 -0
data/doc/scripts/mpi/hw2_c.html +52 -0
data/doc/scripts/mpi/hw2harness_c.html +52 -0
data/doc/scripts/mpi/hw2harness_h.html +52 -0
data/doc/scripts/mpi/run_mpi_cpi_rb.html +52 -0
data/doc/scripts/mpi/run_mpi_powermethod_rb.html +52 -0
data/doc/scripts/mpi/test_rb.html +52 -0
data/doc/scripts/x10/Makefile.html +106 -0
data/doc/scripts/x10/get_x10_output_rb.html +52 -0
data/doc/scripts/x10/run_x10_kmeans_rb.html +52 -0
data/doc/scripts/x10/run_x10_montypi_rb.html +52 -0
data/doc/scripts/x10/run_x10_nqueens_rb.html +52 -0
data/doc/scripts/x10/run_x10_powermethod_rb.html +52 -0
data/doc/scripts/x10/run_x10_qsort_rb.html +52 -0
data/doc/scripts/x10/test_rb.html +52 -0
data/lib/app_controller_client.rb +135 -0
data/lib/common_functions.rb +107 -0
data/lib/job.rb +161 -0
metadata +145 -0

data/doc/scripts/x10/run_x10_qsort_rb.html ADDED Viewed

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+  <title>File: run_x10_qsort.rb [RDoc Documentation]</title>
+  <link type="text/css" media="screen" href="../../rdoc.css" rel="stylesheet" />
+  <script src="../../js/jquery.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/thickbox-compressed.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/quicksearch.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/darkfish.js" type="text/javascript"
+    charset="utf-8"></script>
+</head>
+<body class="file file-popup">
+  <div id="metadata">
+    <dl>
+      <dt class="modified-date">Last Modified</dt>
+      <dd class="modified-date">Mon Dec 27 19:27:35 -0800 2010</dd>
+      <dt class="requires">Requires</dt>
+      <dd class="requires">
+        <ul>
+        </ul>
+      </dd>
+    </dl>
+  </div>
+  <div id="documentation">
+    <div class="description">
+      <h2>Description</h2>
+    </div>
+  </div>
+</body>
+</html>

data/doc/scripts/x10/test_rb.html ADDED Viewed

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+  <title>File: test.rb [RDoc Documentation]</title>
+  <link type="text/css" media="screen" href="../../rdoc.css" rel="stylesheet" />
+  <script src="../../js/jquery.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/thickbox-compressed.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/quicksearch.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/darkfish.js" type="text/javascript"
+    charset="utf-8"></script>
+</head>
+<body class="file file-popup">
+  <div id="metadata">
+    <dl>
+      <dt class="modified-date">Last Modified</dt>
+      <dd class="modified-date">Wed Dec 29 14:05:27 -0800 2010</dd>
+      <dt class="requires">Requires</dt>
+      <dd class="requires">
+        <ul>
+        </ul>
+      </dd>
+    </dl>
+  </div>
+  <div id="documentation">
+    <div class="description">
+      <h2>Description</h2>
+    </div>
+  </div>
+</body>
+</html>

data/lib/app_controller_client.rb ADDED Viewed

@@ -0,0 +1,135 @@
+#!/usr/bin/ruby -w
+# Programmer: Chris Bunch
+require 'openssl'
+require 'soap/rpc/driver'
+require 'timeout'
+# Sometimes SOAP calls take a long time if large amounts of data are being
+# sent over the network: for this first version we don't want these calls to
+# endlessly timeout and retry, so as a hack, just don't let them timeout.
+# The next version should replace this and properly timeout and not use
+# long calls unless necessary.
+NO_TIMEOUT = -1
+# A client that uses SOAP messages to communicate with the underlying cloud
+# platform (here, AppScale). This client is similar to that used in the AppScale
+# Tools, but with non-Neptune SOAP calls removed.
+class AppControllerClient
+  attr_reader :conn, :ip, :secret
+  # A constructor that requires both the IP address of the machine to communicate
+  # with as well as the secret (string) needed to perform communication.
+  # AppControllers will reject SOAP calls if this secret (basically a password)
+  # is not present - it can be found in the user's .appscale directory, and a
+  # helper method is usually present to fetch this for us.
+  def initialize(ip, secret)
+    @ip = ip
+    @secret = secret
+    @conn = SOAP::RPC::Driver.new("https://#{@ip}:17443")
+    @conn.add_method("neptune_start_job", "job_data", "secret")
+    @conn.add_method("neptune_get_output", "job_data", "secret")
+    @conn.add_method("neptune_get_acl", "job_data", "secret")
+    @conn.add_method("neptune_set_acl", "job_data", "secret")
+  end
+  # A helper method to make SOAP calls for us. This method is mainly here to
+  # reduce code duplication: all SOAP calls expect a certain timeout and can
+  # tolerate certain exceptions, so we consolidate this code into this method.
+  # Here, the caller specifies the timeout for the SOAP call (or NO_TIMEOUT
+  # if an infinite timeout is required) as well as whether the call should
+  # be retried in the face of exceptions. Exceptions can occur if the machine
+  # is not yet running or is too busy to handle the request, so these exceptions
+  # are automatically retried regardless of the retry value. Typically
+  # callers set this to false to catch 'Connection Refused' exceptions or
+  # the like. Finally, the caller must provide a block of
+  # code that indicates the SOAP call to make: this is really all that differs
+  # between the calling methods. The result of the block is returned to the
+  # caller.
+  def make_call(time, retry_on_except)
+    begin
+      Timeout::timeout(time) {
+        yield if block_given?
+      }
+    rescue Errno::ECONNREFUSED
+      if retry_on_except
+        retry
+      else
+        abort("Connection was refused. Is the AppController running?")
+      end
+    rescue OpenSSL::SSL::SSLError, NotImplementedError, Timeout::Error
+      retry
+    rescue Exception => except
+      if retry_on_except
+        retry
+      else
+        abort("We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.")
+      end
+    end
+  end
+  # Initiates the start of a Neptune job, whether it be a HPC job (MPI, X10,
+  # or MapReduce), or a scaling job (e.g., for AppScale itself). This method
+  # should not be used for retrieving the output of a job or getting / setting
+  # output ACLs, but just for starting new HPC / scaling jobs. This method
+  # takes a hash containing the parameters of the job to run, and can abort if
+  # the AppController it calls returns an error (e.g., if a bad secret is used
+  # or the machine isn't running). Otherwise, the return value of this method
+  # is the result returned from the AppController.
+  def start_neptune_job(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_start_job(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+  # Retrieves the output of a Neptune job, stored in an underlying
+  # database. Within AppScale, a special application runs, referred to as the
+  # Repository, which provides a key-value interface to Neptune job data.
+  # Data is stored as though it were on a file system, therefore output
+  # be of the usual form /folder/filename . Currently the contents of the
+  # file is returned as a string to the caller, but as this is inefficient
+  # for non-trivial output jobs, the next version of Neptune will add an
+  # additional call to directly copy the output to a file on the local
+  # filesystem. See start_neptune_job for conditions by which this method
+  # can abort as well as the input format used for job_data.
+  def get_output(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_get_output(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+  # Returns the ACL associated with the named piece of data stored
+  # in the underlying cloud platform. Right now, data can only be
+  # public or private, but future versions will add individual user
+  # support. Input, output, and exceptions mirror that of
+  # start_neptune_job.
+  def get_acl(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_get_acl(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+  # Sets the ACL of a specified pieces of data stored in the underlying
+  # cloud platform. As is the case with get_acl, ACLs can be either
+  # public or private right now, but this will be expanded upon in
+  # the future. As with the other SOAP calls, input, output, and exceptions
+  # mirror that of start_neptune_job.
+  def set_acl(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_set_acl(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+end

data/lib/common_functions.rb ADDED Viewed

@@ -0,0 +1,107 @@
+#!/usr/bin/ruby -w
+# Programmer: Chris Bunch
+require 'digest/sha1'
+require 'fileutils'
+require 'net/http'
+require 'openssl'
+require 'socket'
+require 'timeout'
+require 'yaml'
+# A helper module that aggregates functions that are not part of Neptune's
+# core functionality. Specifically, this module contains methods to scp
+# files to other machines and the ability to read YAML files, which are
+# often needed to determine which machine should be used for computation
+# or to copy over code and input files.
+module CommonFunctions
+  # Copies a file to the Shadow node (head node) within AppScale.
+  # The caller specifies
+  # the local file location, the destination where the file should be
+  # placed, and the name of the key to use. The keyname is typically
+  # specified by the Neptune job given, but defaults to ''appscale''
+  # if not provided.
+  def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname)
+    shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+    ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
+    self.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key)
+  end
+  # Performs the actual remote copying of files: given the IP address
+  # and other information from scp_to_shadow, attempts to use scp
+  # to copy the file over. Aborts if the scp fails, which can occur
+  # if the network is down, if a bad keyname is provided, or if the
+  # wrong IP is given.
+  def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc)
+    cmd = ""
+    local_file_loc = File.expand_path(local_file_loc)
+    if public_key_loc.class == Array
+      public_key_loc.each { |key|
+        key = File.expand_path(key)
+      }
+      cmd = "scp -i #{public_key_loc.join(' -i ')} -o StrictHostkeyChecking=no 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
+    else
+      public_key_loc = File.expand_path(public_key_loc)
+      cmd = "scp -i #{public_key_loc} -o StrictHostkeyChecking=no 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
+    end
+    cmd << "; echo $? >> ~/.appscale/retval"
+    retval_loc = File.expand_path("~/.appscale/retval")
+    FileUtils.rm_f(retval_loc)
+    begin
+      Timeout::timeout(-1) { `#{cmd}` }
+    rescue Timeout::Error
+      abort("Remotely copying over files failed. Is the destination machine on and reachable from this computer? We tried the following command:\n\n#{cmd}")
+    end
+    loop {
+      break if File.exists?(retval_loc)
+      sleep(5)
+    }
+    retval = (File.open(retval_loc) { |f| f.read }).chomp
+    abort("\n\n[#{cmd}] returned #{retval} instead of 0 as expected. Is your environment set up properly?") if retval != "0"
+    return cmd
+  end
+  # Given the AppScale keyname, reads the associated YAML file and returns
+  # the contents of the given tag. The required flag (default value is true)
+  # indicates whether a value must exist for this tag: if set to true, this
+  # method aborts if the value doesn't exist or the YAML file is malformed.
+  # If the required flag is set to false, it returns nil in either scenario
+  # instead.
+  def self.get_from_yaml(keyname, tag, required=true)
+    location_file = File.expand_path("~/.appscale/locations-#{keyname}.yaml")
+    abort("An AppScale instance is not currently running with the provided keyname, \"#{keyname}\".") unless File.exists?(location_file)
+    begin
+      tree = YAML.load_file(location_file)
+    rescue ArgumentError
+      if required
+        abort("The yaml file you provided was malformed. Please correct any errors in it and try again.")
+      else
+        return nil
+      end
+    end
+    value = tree[tag]
+    bad_yaml_format_msg = "The file #{location_file} is in the wrong format and doesn't contain a #{tag} tag. Please make sure the file is in the correct format and try again"
+    abort(bad_yaml_format_msg) if value.nil? and required
+    return value
+  end
+  # Returns the secret key needed for communication with AppScale's
+  # Shadow node. This method is a nice frontend to the get_from_yaml
+  # function, as the secret is stored in a YAML file.
+  def self.get_secret_key(keyname, required=true)
+    return CommonFunctions.get_from_yaml(keyname, :secret)
+  end
+end

data/lib/job.rb ADDED Viewed

@@ -0,0 +1,161 @@
+#!/usr/bin/ruby
+# Setting verbose to nil here suppresses the otherwise
+# excessive SSL cert warning messages that will pollute
+# stderr and worry users unnecessarily.
+$VERBOSE = nil
+#MPI_RUN_JOB_REQUIRED = %w{ input output code filesystem }
+#MPI_REQUIRED = %w{ output }
+#X10_RUN_JOB_REQUIRED = %w{ input output code filesystem }
+#X10_REQUIRED = %w{ output }
+#DFSP_RUN_JOB_REQUIRED = %w{ output simulations }
+#DFSP_REQUIRED = %w{ output }
+#CEWSSA_RUN_JOB_REQUIRED = %w{ output simulations }
+#CEWSSA_REQUIRED = %w{ output }
+#MR_RUN_JOB_REQUIRED = %w{ }
+#MR_REQUIRED = %w{ output }
+# A list of jobs that require some kind of work to be done before
+# the actual computation can be performed.
+NEED_PREPROCESSING = %w{ mpi mapreduce }
+# A set of methods and constants that we've monkey-patched to enable Neptune
+# support. In the future, it is likely that the only exposed / monkey-patched
+# method should be job, while the others could probably be folded into either
+# a Neptune-specific class or into CommonFunctions.
+class Object
+end
+# Certain types of jobs need steps to be taken before they
+# can be started (e.g., copying input data or code over).
+# This method dispatches the right method to use based
+# on the type of the job that the user has asked to run.
+def do_preprocessing(job_data)
+  job_type = job_data["@type"]
+  return unless NEED_PREPROCESSING.include?(job_type)
+  preprocess = "preprocess_#{job_type}".to_sym
+  send(preprocess, job_data)
+end
+# This preprocessing method copies over the user's MPI
+# code to the master node in AppScale - this node will
+# then copy it to whoever will run the MPI job.
+def preprocess_mpi(job_data)
+  source_code = File.expand_path(job_data["@code"])
+  unless File.exists?(source_code)
+    file_not_found = "The specified code, #{job_data['@code']}," +
+      " didn't exist. Please specify one that exists and try again"
+    abort(file_not_found)
+  end
+  dest_code = "/tmp/thempicode"
+  keyname = job_data["@keyname"]
+  CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
+end
+# This preprocessing method handles copying data for regular
+# Hadoop MapReduce and Hadoop MapReduce Streaming. For the former
+# case, we copy over just the JAR the user has given us, and
+# in the latter case, we copy over the Map and Reduce files
+# that have been specified. In either case, if the user has
+# specified to us to copy over an input file, we do that as well:
+# AppScale will copy it into HDFS for us.
+def preprocess_mapreduce(job_data)
+  items_to_copy = ["@map", "@reduce"] if job_data["@map"] and job_data["@reduce"]
+  items_to_copy = ["@mapreducejar"] if job_data["@mapreducejar"]
+  items_to_copy << "@input" if job_data["@copy_input"]
+  items_to_copy.each { |item|
+    source = File.expand_path(job_data[item])
+    unless File.exists?(source)
+      abort("The #{item} file #{source} does not exist.")
+    end
+    suffix = source.split('/')[-1]
+    dest = "/tmp/#{suffix}"
+    keyname = job_data["@keyname"]
+    CommonFunctions.scp_to_shadow(source, dest, keyname)
+    job_data[item] = dest
+  }
+end
+# TODO: actually use me!
+#def validate_args(list)
+#  list.each do |item|
+#    val = instance_variable_get("@#{item}".to_sym)
+#    abort("FATAL: #{item} was not defined") if val.nil?
+#  end
+#end
+# This method is the heart of Neptune - here, we take
+# blocks of code that the user has written and convert them
+# into HPC job requests. At a high level, the user can
+# request to run a job, retrieve a job's output, or
+# modify the access policy (ACL) for the output of a
+# job. By default, job data is private, but a Neptune
+# job can be used to set it to public later (and
+# vice-versa).
+def job(name, &block)
+  puts "Received a request to run a job."
+  puts name
+  block.call()
+  if @keyname.nil?
+    keyname = "appscale"
+  else
+    keyname = @keyname
+  end
+  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+  secret = CommonFunctions.get_secret_key(keyname)
+  controller = AppControllerClient.new(shadow_ip, secret)
+  ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
+  job_data = {}
+  instance_variables.each { |var|
+    job_data[var] = instance_variable_get(var)
+  }
+  job_data["@job"] = nil
+  job_data["@type"] = name
+  job_data["@keyname"] = keyname
+  if job_data["@output"].nil? or job_data["@output"] == ""
+    abort("Job output must be specified")
+  end
+  if job_data["@output"][0].chr != "/"
+    abort("Job output must begin with a slash ('/')")
+  end
+  #if job_data["@can_run_on"].class == Range
+  #  job_data["@can_run_on"] = job_data["@can_run_on"].to_a
+  #elsif job_data["@can_run_on"].class == Fixnum
+  #  job_data["@can_run_on"] = [job_data["@can_run_on"]]
+  #end
+  puts "job data = #{job_data.inspect}"
+  do_preprocessing(job_data)
+  if job_data["@type"] == "output"
+    return controller.get_output(job_data)
+  elsif job_data["@type"] == "get-acl"
+    job_data["@type"] = "acl"
+    return controller.get_acl(job_data)
+  elsif job_data["@type"] == "set-acl"
+    job_data["@type"] = "acl"
+    return controller.set_acl(job_data)
+  else
+    result = controller.start_neptune_job(job_data)
+    if result =~ /job is now running\Z/
+      return :success
+    else
+      return :failure
+    end
+  end
+end