RubyGems - neptune - Versions diffs - 0.0.1 - Mend

neptune 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (79) hide show

data/LICENSE +39 -0
data/README +50 -0
data/bin/neptune +16 -0
data/doc/AppControllerClient.html +468 -0
data/doc/CommonFunctions.html +368 -0
data/doc/LICENSE.html +131 -0
data/doc/Object.html +431 -0
data/doc/README.html +135 -0
data/doc/bin/neptune.html +58 -0
data/doc/created.rid +7 -0
data/doc/images/brick.png +0 -0
data/doc/images/brick_link.png +0 -0
data/doc/images/bug.png +0 -0
data/doc/images/bullet_black.png +0 -0
data/doc/images/bullet_toggle_minus.png +0 -0
data/doc/images/bullet_toggle_plus.png +0 -0
data/doc/images/date.png +0 -0
data/doc/images/find.png +0 -0
data/doc/images/loadingAnimation.gif +0 -0
data/doc/images/macFFBgHack.png +0 -0
data/doc/images/package.png +0 -0
data/doc/images/page_green.png +0 -0
data/doc/images/page_white_text.png +0 -0
data/doc/images/page_white_width.png +0 -0
data/doc/images/plugin.png +0 -0
data/doc/images/ruby.png +0 -0
data/doc/images/tag_green.png +0 -0
data/doc/images/wrench.png +0 -0
data/doc/images/wrench_orange.png +0 -0
data/doc/images/zoom.png +0 -0
data/doc/index.html +90 -0
data/doc/js/darkfish.js +116 -0
data/doc/js/jquery.js +32 -0
data/doc/js/quicksearch.js +114 -0
data/doc/js/thickbox-compressed.js +10 -0
data/doc/lib/app_controller_client_rb.html +60 -0
data/doc/lib/common_functions_rb.html +68 -0
data/doc/lib/job_rb.html +55 -0
data/doc/scripts/appscale/add_appserver_rb.html +52 -0
data/doc/scripts/appscale/add_database_rb.html +52 -0
data/doc/scripts/appscale/add_loadbalancer_rb.html +52 -0
data/doc/scripts/appscale/add_slave_rb.html +52 -0
data/doc/scripts/cewssa/cewssa-output.html +104 -0
data/doc/scripts/cewssa/get_cewssa_output_rb.html +52 -0
data/doc/scripts/cewssa/run_cewssa_rb.html +52 -0
data/doc/scripts/cewssa/test_dfsp_rb.html +52 -0
data/doc/scripts/dfsp/get_dfsp_output_rb.html +52 -0
data/doc/scripts/dfsp/make_output_public_rb.html +52 -0
data/doc/scripts/dfsp/run_dfsp_rb.html +52 -0
data/doc/scripts/dfsp/see_acl_rb.html +52 -0
data/doc/scripts/dfsp/test_dfsp_rb.html +52 -0
data/doc/scripts/mapreduce/get_mapreduce_output_rb.html +52 -0
data/doc/scripts/mapreduce/input-10.html +114 -0
data/doc/scripts/mapreduce/input-30.html +114 -0
data/doc/scripts/mapreduce/input-7.html +104 -0
data/doc/scripts/mapreduce/map_rb.html +56 -0
data/doc/scripts/mapreduce/reduce_rb.html +56 -0
data/doc/scripts/mapreduce/run_java_mr_rb.html +52 -0
data/doc/scripts/mapreduce/run_mapreduce_rb.html +52 -0
data/doc/scripts/mpi/Makefile.html +125 -0
data/doc/scripts/mpi/get_mpi_output_rb.html +52 -0
data/doc/scripts/mpi/hw2_c.html +52 -0
data/doc/scripts/mpi/hw2harness_c.html +52 -0
data/doc/scripts/mpi/hw2harness_h.html +52 -0
data/doc/scripts/mpi/run_mpi_cpi_rb.html +52 -0
data/doc/scripts/mpi/run_mpi_powermethod_rb.html +52 -0
data/doc/scripts/mpi/test_rb.html +52 -0
data/doc/scripts/x10/Makefile.html +106 -0
data/doc/scripts/x10/get_x10_output_rb.html +52 -0
data/doc/scripts/x10/run_x10_kmeans_rb.html +52 -0
data/doc/scripts/x10/run_x10_montypi_rb.html +52 -0
data/doc/scripts/x10/run_x10_nqueens_rb.html +52 -0
data/doc/scripts/x10/run_x10_powermethod_rb.html +52 -0
data/doc/scripts/x10/run_x10_qsort_rb.html +52 -0
data/doc/scripts/x10/test_rb.html +52 -0
data/lib/app_controller_client.rb +135 -0
data/lib/common_functions.rb +107 -0
data/lib/job.rb +161 -0
metadata +145 -0

data/doc/scripts/x10/run_x10_qsort_rb.html ADDED Viewed

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+  <title>File: run_x10_qsort.rb [RDoc Documentation]</title>
+  <link type="text/css" media="screen" href="../../rdoc.css" rel="stylesheet" />
+  <script src="../../js/jquery.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/thickbox-compressed.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/quicksearch.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/darkfish.js" type="text/javascript"
+    charset="utf-8"></script>
+</head>
+<body class="file file-popup">
+  <div id="metadata">
+    <dl>
+      <dt class="modified-date">Last Modified</dt>
+      <dd class="modified-date">Mon Dec 27 19:27:35 -0800 2010</dd>
+      <dt class="requires">Requires</dt>
+      <dd class="requires">
+        <ul>
+        </ul>
+      </dd>
+    </dl>
+  </div>
+  <div id="documentation">
+    <div class="description">
+      <h2>Description</h2>
+    </div>
+  </div>
+</body>
+</html>

data/doc/scripts/x10/test_rb.html ADDED Viewed

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+  <title>File: test.rb [RDoc Documentation]</title>
+  <link type="text/css" media="screen" href="../../rdoc.css" rel="stylesheet" />
+  <script src="../../js/jquery.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/thickbox-compressed.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/quicksearch.js" type="text/javascript"
+    charset="utf-8"></script>
+  <script src="../../js/darkfish.js" type="text/javascript"
+    charset="utf-8"></script>
+</head>
+<body class="file file-popup">
+  <div id="metadata">
+    <dl>
+      <dt class="modified-date">Last Modified</dt>
+      <dd class="modified-date">Wed Dec 29 14:05:27 -0800 2010</dd>
+      <dt class="requires">Requires</dt>
+      <dd class="requires">
+        <ul>
+        </ul>
+      </dd>
+    </dl>
+  </div>
+  <div id="documentation">
+    <div class="description">
+      <h2>Description</h2>
+    </div>
+  </div>
+</body>
+</html>

data/lib/app_controller_client.rb ADDED Viewed

@@ -0,0 +1,135 @@
+#!/usr/bin/ruby -w
+# Programmer: Chris Bunch
+require 'openssl'
+require 'soap/rpc/driver'
+require 'timeout'
+# Sometimes SOAP calls take a long time if large amounts of data are being
+# sent over the network: for this first version we don't want these calls to
+# endlessly timeout and retry, so as a hack, just don't let them timeout.
+# The next version should replace this and properly timeout and not use
+# long calls unless necessary.
+NO_TIMEOUT = -1
+# A client that uses SOAP messages to communicate with the underlying cloud
+# platform (here, AppScale). This client is similar to that used in the AppScale
+# Tools, but with non-Neptune SOAP calls removed.
+class AppControllerClient
+  attr_reader :conn, :ip, :secret
+  # A constructor that requires both the IP address of the machine to communicate
+  # with as well as the secret (string) needed to perform communication.
+  # AppControllers will reject SOAP calls if this secret (basically a password)
+  # is not present - it can be found in the user's .appscale directory, and a
+  # helper method is usually present to fetch this for us.
+  def initialize(ip, secret)
+    @ip = ip
+    @secret = secret
+    @conn = SOAP::RPC::Driver.new("https://#{@ip}:17443")
+    @conn.add_method("neptune_start_job", "job_data", "secret")
+    @conn.add_method("neptune_get_output", "job_data", "secret")
+    @conn.add_method("neptune_get_acl", "job_data", "secret")
+    @conn.add_method("neptune_set_acl", "job_data", "secret")
+  end
+  # A helper method to make SOAP calls for us. This method is mainly here to
+  # reduce code duplication: all SOAP calls expect a certain timeout and can
+  # tolerate certain exceptions, so we consolidate this code into this method.
+  # Here, the caller specifies the timeout for the SOAP call (or NO_TIMEOUT
+  # if an infinite timeout is required) as well as whether the call should
+  # be retried in the face of exceptions. Exceptions can occur if the machine
+  # is not yet running or is too busy to handle the request, so these exceptions
+  # are automatically retried regardless of the retry value. Typically
+  # callers set this to false to catch 'Connection Refused' exceptions or
+  # the like. Finally, the caller must provide a block of
+  # code that indicates the SOAP call to make: this is really all that differs
+  # between the calling methods. The result of the block is returned to the
+  # caller.
+  def make_call(time, retry_on_except)
+    begin
+      Timeout::timeout(time) {
+        yield if block_given?
+      }
+    rescue Errno::ECONNREFUSED
+      if retry_on_except
+        retry
+      else
+        abort("Connection was refused. Is the AppController running?")
+      end
+    rescue OpenSSL::SSL::SSLError, NotImplementedError, Timeout::Error
+      retry
+    rescue Exception => except
+      if retry_on_except
+        retry
+      else
+        abort("We saw an unexpected error of the type #{except.class} with the following message:\n#{except}.")
+      end
+    end
+  end
+  # Initiates the start of a Neptune job, whether it be a HPC job (MPI, X10,
+  # or MapReduce), or a scaling job (e.g., for AppScale itself). This method
+  # should not be used for retrieving the output of a job or getting / setting
+  # output ACLs, but just for starting new HPC / scaling jobs. This method
+  # takes a hash containing the parameters of the job to run, and can abort if
+  # the AppController it calls returns an error (e.g., if a bad secret is used
+  # or the machine isn't running). Otherwise, the return value of this method
+  # is the result returned from the AppController.
+  def start_neptune_job(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_start_job(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+  # Retrieves the output of a Neptune job, stored in an underlying
+  # database. Within AppScale, a special application runs, referred to as the
+  # Repository, which provides a key-value interface to Neptune job data.
+  # Data is stored as though it were on a file system, therefore output
+  # be of the usual form /folder/filename . Currently the contents of the
+  # file is returned as a string to the caller, but as this is inefficient
+  # for non-trivial output jobs, the next version of Neptune will add an
+  # additional call to directly copy the output to a file on the local
+  # filesystem. See start_neptune_job for conditions by which this method
+  # can abort as well as the input format used for job_data.
+  def get_output(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_get_output(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+  # Returns the ACL associated with the named piece of data stored
+  # in the underlying cloud platform. Right now, data can only be
+  # public or private, but future versions will add individual user
+  # support. Input, output, and exceptions mirror that of
+  # start_neptune_job.
+  def get_acl(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_get_acl(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+  # Sets the ACL of a specified pieces of data stored in the underlying
+  # cloud platform. As is the case with get_acl, ACLs can be either
+  # public or private right now, but this will be expanded upon in
+  # the future. As with the other SOAP calls, input, output, and exceptions
+  # mirror that of start_neptune_job.
+  def set_acl(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) {
+      result = conn.neptune_set_acl(job_data, @secret)
+    }
+    abort(result) if result =~ /Error:/
+    return result
+  end
+end

data/lib/common_functions.rb ADDED Viewed

@@ -0,0 +1,107 @@
+#!/usr/bin/ruby -w
+# Programmer: Chris Bunch
+require 'digest/sha1'
+require 'fileutils'
+require 'net/http'
+require 'openssl'
+require 'socket'
+require 'timeout'
+require 'yaml'
+# A helper module that aggregates functions that are not part of Neptune's
+# core functionality. Specifically, this module contains methods to scp
+# files to other machines and the ability to read YAML files, which are
+# often needed to determine which machine should be used for computation
+# or to copy over code and input files.
+module CommonFunctions
+  # Copies a file to the Shadow node (head node) within AppScale.
+  # The caller specifies
+  # the local file location, the destination where the file should be
+  # placed, and the name of the key to use. The keyname is typically
+  # specified by the Neptune job given, but defaults to ''appscale''
+  # if not provided.
+  def self.scp_to_shadow(local_file_loc, remote_file_loc, keyname)
+    shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+    ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
+    self.scp_file(local_file_loc, remote_file_loc, shadow_ip, ssh_key)
+  end
+  # Performs the actual remote copying of files: given the IP address
+  # and other information from scp_to_shadow, attempts to use scp
+  # to copy the file over. Aborts if the scp fails, which can occur
+  # if the network is down, if a bad keyname is provided, or if the
+  # wrong IP is given.
+  def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc)
+    cmd = ""
+    local_file_loc = File.expand_path(local_file_loc)
+    if public_key_loc.class == Array
+      public_key_loc.each { |key|
+        key = File.expand_path(key)
+      }
+      cmd = "scp -i #{public_key_loc.join(' -i ')} -o StrictHostkeyChecking=no 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
+    else
+      public_key_loc = File.expand_path(public_key_loc)
+      cmd = "scp -i #{public_key_loc} -o StrictHostkeyChecking=no 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
+    end
+    cmd << "; echo $? >> ~/.appscale/retval"
+    retval_loc = File.expand_path("~/.appscale/retval")
+    FileUtils.rm_f(retval_loc)
+    begin
+      Timeout::timeout(-1) { `#{cmd}` }
+    rescue Timeout::Error
+      abort("Remotely copying over files failed. Is the destination machine on and reachable from this computer? We tried the following command:\n\n#{cmd}")
+    end
+    loop {
+      break if File.exists?(retval_loc)
+      sleep(5)
+    }
+    retval = (File.open(retval_loc) { |f| f.read }).chomp
+    abort("\n\n[#{cmd}] returned #{retval} instead of 0 as expected. Is your environment set up properly?") if retval != "0"
+    return cmd
+  end
+  # Given the AppScale keyname, reads the associated YAML file and returns
+  # the contents of the given tag. The required flag (default value is true)
+  # indicates whether a value must exist for this tag: if set to true, this
+  # method aborts if the value doesn't exist or the YAML file is malformed.
+  # If the required flag is set to false, it returns nil in either scenario
+  # instead.
+  def self.get_from_yaml(keyname, tag, required=true)
+    location_file = File.expand_path("~/.appscale/locations-#{keyname}.yaml")
+    abort("An AppScale instance is not currently running with the provided keyname, \"#{keyname}\".") unless File.exists?(location_file)
+    begin
+      tree = YAML.load_file(location_file)
+    rescue ArgumentError
+      if required
+        abort("The yaml file you provided was malformed. Please correct any errors in it and try again.")
+      else
+        return nil
+      end
+    end
+    value = tree[tag]
+    bad_yaml_format_msg = "The file #{location_file} is in the wrong format and doesn't contain a #{tag} tag. Please make sure the file is in the correct format and try again"
+    abort(bad_yaml_format_msg) if value.nil? and required
+    return value
+  end
+  # Returns the secret key needed for communication with AppScale's
+  # Shadow node. This method is a nice frontend to the get_from_yaml
+  # function, as the secret is stored in a YAML file.
+  def self.get_secret_key(keyname, required=true)
+    return CommonFunctions.get_from_yaml(keyname, :secret)
+  end
+end

data/lib/job.rb ADDED Viewed

@@ -0,0 +1,161 @@
+#!/usr/bin/ruby
+# Setting verbose to nil here suppresses the otherwise
+# excessive SSL cert warning messages that will pollute
+# stderr and worry users unnecessarily.
+$VERBOSE = nil
+#MPI_RUN_JOB_REQUIRED = %w{ input output code filesystem }
+#MPI_REQUIRED = %w{ output }
+#X10_RUN_JOB_REQUIRED = %w{ input output code filesystem }
+#X10_REQUIRED = %w{ output }
+#DFSP_RUN_JOB_REQUIRED = %w{ output simulations }
+#DFSP_REQUIRED = %w{ output }
+#CEWSSA_RUN_JOB_REQUIRED = %w{ output simulations }
+#CEWSSA_REQUIRED = %w{ output }
+#MR_RUN_JOB_REQUIRED = %w{ }
+#MR_REQUIRED = %w{ output }
+# A list of jobs that require some kind of work to be done before
+# the actual computation can be performed.
+NEED_PREPROCESSING = %w{ mpi mapreduce }
+# A set of methods and constants that we've monkey-patched to enable Neptune
+# support. In the future, it is likely that the only exposed / monkey-patched
+# method should be job, while the others could probably be folded into either
+# a Neptune-specific class or into CommonFunctions.
+class Object
+end
+# Certain types of jobs need steps to be taken before they
+# can be started (e.g., copying input data or code over).
+# This method dispatches the right method to use based
+# on the type of the job that the user has asked to run.
+def do_preprocessing(job_data)
+  job_type = job_data["@type"]
+  return unless NEED_PREPROCESSING.include?(job_type)
+  preprocess = "preprocess_#{job_type}".to_sym
+  send(preprocess, job_data)
+end
+# This preprocessing method copies over the user's MPI
+# code to the master node in AppScale - this node will
+# then copy it to whoever will run the MPI job.
+def preprocess_mpi(job_data)
+  source_code = File.expand_path(job_data["@code"])
+  unless File.exists?(source_code)
+    file_not_found = "The specified code, #{job_data['@code']}," +
+      " didn't exist. Please specify one that exists and try again"
+    abort(file_not_found)
+  end
+  dest_code = "/tmp/thempicode"
+  keyname = job_data["@keyname"]
+  CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
+end
+# This preprocessing method handles copying data for regular
+# Hadoop MapReduce and Hadoop MapReduce Streaming. For the former
+# case, we copy over just the JAR the user has given us, and
+# in the latter case, we copy over the Map and Reduce files
+# that have been specified. In either case, if the user has
+# specified to us to copy over an input file, we do that as well:
+# AppScale will copy it into HDFS for us.
+def preprocess_mapreduce(job_data)
+  items_to_copy = ["@map", "@reduce"] if job_data["@map"] and job_data["@reduce"]
+  items_to_copy = ["@mapreducejar"] if job_data["@mapreducejar"]
+  items_to_copy << "@input" if job_data["@copy_input"]
+  items_to_copy.each { |item|
+    source = File.expand_path(job_data[item])
+    unless File.exists?(source)
+      abort("The #{item} file #{source} does not exist.")
+    end
+    suffix = source.split('/')[-1]
+    dest = "/tmp/#{suffix}"
+    keyname = job_data["@keyname"]
+    CommonFunctions.scp_to_shadow(source, dest, keyname)
+    job_data[item] = dest
+  }
+end
+# TODO: actually use me!
+#def validate_args(list)
+#  list.each do |item|
+#    val = instance_variable_get("@#{item}".to_sym)
+#    abort("FATAL: #{item} was not defined") if val.nil?
+#  end
+#end
+# This method is the heart of Neptune - here, we take
+# blocks of code that the user has written and convert them
+# into HPC job requests. At a high level, the user can
+# request to run a job, retrieve a job's output, or
+# modify the access policy (ACL) for the output of a
+# job. By default, job data is private, but a Neptune
+# job can be used to set it to public later (and
+# vice-versa).
+def job(name, &block)
+  puts "Received a request to run a job."
+  puts name
+  block.call()
+  if @keyname.nil?
+    keyname = "appscale"
+  else
+    keyname = @keyname
+  end
+  shadow_ip = CommonFunctions.get_from_yaml(keyname, :shadow)
+  secret = CommonFunctions.get_secret_key(keyname)
+  controller = AppControllerClient.new(shadow_ip, secret)
+  ssh_key = File.expand_path("~/.appscale/#{keyname}.key")
+  job_data = {}
+  instance_variables.each { |var|
+    job_data[var] = instance_variable_get(var)
+  }
+  job_data["@job"] = nil
+  job_data["@type"] = name
+  job_data["@keyname"] = keyname
+  if job_data["@output"].nil? or job_data["@output"] == ""
+    abort("Job output must be specified")
+  end
+  if job_data["@output"][0].chr != "/"
+    abort("Job output must begin with a slash ('/')")
+  end
+  #if job_data["@can_run_on"].class == Range
+  #  job_data["@can_run_on"] = job_data["@can_run_on"].to_a
+  #elsif job_data["@can_run_on"].class == Fixnum
+  #  job_data["@can_run_on"] = [job_data["@can_run_on"]]
+  #end
+  puts "job data = #{job_data.inspect}"
+  do_preprocessing(job_data)
+  if job_data["@type"] == "output"
+    return controller.get_output(job_data)
+  elsif job_data["@type"] == "get-acl"
+    job_data["@type"] = "acl"
+    return controller.get_acl(job_data)
+  elsif job_data["@type"] == "set-acl"
+    job_data["@type"] = "acl"
+    return controller.set_acl(job_data)
+  else
+    result = controller.start_neptune_job(job_data)
+    if result =~ /job is now running\Z/
+      return :success
+    else
+      return :failure
+    end
+  end
+end