neptune 0.1.2 → 0.1.3

data/README

@@ -66,6 +66,10 @@ Here's a link</a> to the abstract of the paper and the PDF.
 
 Version History:
 
+December 4, 2011 - 0.1.3 released, adding support for KDT (Knowledge Discovery
+Toolkit) and Cicero, a framework for automatic task execution over Google
+App Engine and AppScale.
+
 November 10, 2011 - 0.1.2 released, adding unit tests and
 refactoring all around.
 

data/doc/AppControllerClient.html

@@ -96,17 +96,6 @@
     <div id="project-metadata">
       
       
-      <div id="fileindex-section" class="section project-section">
-        <h3 class="section-header">Files</h3>
-        <ul>
-        
-          <li class="file"><a href="./LICENSE.html">LICENSE</a></li>
-        
-          <li class="file"><a href="./README.html">README</a></li>
-        
-        </ul>
-      </div>
-      
 
       <div id="classindex-section" class="section project-section">
         <h3 class="section-header">Class/Module Index

data/doc/CommonFunctions.html

@@ -81,17 +81,6 @@
     <div id="project-metadata">
       
       
-      <div id="fileindex-section" class="section project-section">
-        <h3 class="section-header">Files</h3>
-        <ul>
-        
-          <li class="file"><a href="./LICENSE.html">LICENSE</a></li>
-        
-          <li class="file"><a href="./README.html">README</a></li>
-        
-        </ul>
-      </div>
-      
 
       <div id="classindex-section" class="section project-section">
         <h3 class="section-header">Class/Module Index

data/doc/Kernel.html

@@ -75,17 +75,6 @@
     <div id="project-metadata">
       
       
-      <div id="fileindex-section" class="section project-section">
-        <h3 class="section-header">Files</h3>
-        <ul>
-        
-          <li class="file"><a href="./LICENSE.html">LICENSE</a></li>
-        
-          <li class="file"><a href="./README.html">README</a></li>
-        
-        </ul>
-      </div>
-      
 
       <div id="classindex-section" class="section project-section">
         <h3 class="section-header">Class/Module Index

data/doc/Object.html

@@ -94,6 +94,8 @@
           
           <li><a href="#method-i-run_job">#run_job</a></li>
           
+          <li><a href="#method-i-upload_app_for_cicero">#upload_app_for_cicero</a></li>
+          
           <li><a href="#method-i-validate_storage_params">#validate_storage_params</a></li>
           
           <li><a href="#method-i-wait_for_compilation_to_finish">#wait_for_compilation_to_finish</a></li>
@@ -109,17 +111,6 @@
     <div id="project-metadata">
       
       
-      <div id="fileindex-section" class="section project-section">
-        <h3 class="section-header">Files</h3>
-        <ul>
-        
-          <li class="file"><a href="./LICENSE.html">LICENSE</a></li>
-        
-          <li class="file"><a href="./README.html">README</a></li>
-        
-        </ul>
-      </div>
-      
 
       <div id="classindex-section" class="section project-section">
         <h3 class="section-header">Class/Module Index
@@ -242,7 +233,7 @@ indicates whether or not the compilation was successful.</p>
           <div class="method-source-code"
             id="compile_code-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 281</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 290</span>
 def compile_code(job_data, ssh_args, shadow_ip, shell=<span class="ruby-constant">Kernel</span>.method(:`))
   compiled_location = controller.compile_code(job_data)
 
@@ -342,7 +333,7 @@ job succeeded and if it failed, the reason for it.</p>
           <div class="method-source-code"
             id="get_input-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 227</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 236</span>
 def get_input(job_data, ssh_args, shadow_ip, controller, file=<span class="ruby-constant">File</span>,
   shell=<span class="ruby-constant">Kernel</span>.method(:`))
   result = {:result =&gt; :success}
@@ -422,6 +413,15 @@ def get_job_data(params)
     type = <span class="ruby-string">&quot;mpi&quot;</span>
   end
 
+  <span class="ruby-comment"># kdt jobs also run as mpi jobs, but need to pass along an executable</span>
+  <span class="ruby-comment"># parameter to let mpiexec know to use python to exec it</span>
+  if type == <span class="ruby-string">&quot;kdt&quot;</span>
+    job_data[<span class="ruby-string">&quot;@type&quot;</span>] = <span class="ruby-string">&quot;mpi&quot;</span>
+    type = <span class="ruby-string">&quot;mpi&quot;</span>
+
+    job_data[<span class="ruby-string">&quot;@executable&quot;</span>] = <span class="ruby-string">&quot;python&quot;</span>
+  end
+
   if job_data[<span class="ruby-string">&quot;@nodes_to_use&quot;</span>].class == <span class="ruby-constant">Hash</span>
     job_data[<span class="ruby-string">&quot;@nodes_to_use&quot;</span>] = job_data[<span class="ruby-string">&quot;@nodes_to_use&quot;</span>].to_a.flatten
   end
@@ -470,7 +470,7 @@ not the job completed successfully (success = no errors).</p>
           <div class="method-source-code"
             id="get_std_out_and_err-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 310</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 319</span>
 def get_std_out_and_err(location)
   result = {}
 
@@ -523,7 +523,7 @@ vice-versa).</p>
           <div class="method-source-code"
             id="neptune-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 368</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 408</span>
 def neptune(params)
   puts <span class="ruby-string">&quot;Received a request to run a job.&quot;</span>
   puts params[:type]
@@ -778,7 +778,7 @@ as well as information about the node to send the request to.</p>
           <div class="method-source-code"
             id="run_job-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 330</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 365</span>
 def run_job(job_data, ssh_args, shadow_ip, secret,
   controller=<span class="ruby-constant">AppControllerClient</span>, file=<span class="ruby-constant">File</span>)
   controller = controller.new(shadow_ip, secret)
@@ -800,6 +800,11 @@ def run_job(job_data, ssh_args, shadow_ip, secret,
     result[:acl] = controller.set_acl(job_data)
   when <span class="ruby-string">&quot;compile&quot;</span>
     result = compile_code(job_data, ssh_args, shadow_ip)
+  when <span class="ruby-string">&quot;cicero&quot;</span>
+    upload_app_for_cicero(job_data)
+    msg = controller.start_neptune_job(job_data)
+    result[:msg] = msg
+    result[:result] = :failure if result[:msg] !~ <span class="ruby-regexp">/job is now running\Z/</span>
   else
     msg = controller.start_neptune_job(job_data)
     result[:msg] = msg
@@ -815,6 +820,62 @@ end</pre>
         
 
         
+      </div>
+
+    
+      <div id="upload_app_for_cicero-method" class="method-detail ">
+        <a name="method-i-upload_app_for_cicero"></a>
+
+        
+        <div class="method-heading">
+          <span class="method-name">upload_app_for_cicero</span><span
+            class="method-args">(job_data)</span>
+          <span class="method-click-advice">click to toggle source</span>
+        </div>
+        
+
+        <div class="method-description">
+          
+          
+          
+
+          
+          <div class="method-source-code"
+            id="upload_app_for_cicero-source">
+<pre>
+<span class="ruby-comment"># File lib/neptune.rb, line 337</span>
+def upload_app_for_cicero(job_data)
+  if !job_data[<span class="ruby-string">&quot;@app&quot;</span>]
+    puts <span class="ruby-string">&quot;No app specified, not uploading...&quot;</span> 
+    return
+  end
+
+  app_location = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@app&quot;</span>])
+  if !<span class="ruby-constant">File</span>.exists?(app_location)
+    abort(&quot;The app you specified, #{app_location}, does not exist.&quot; + 
+      <span class="ruby-string">&quot;Please specify one that does and try again.&quot;</span>)
+  end
+
+  keyname = job_data[<span class="ruby-string">&quot;@keyname&quot;</span>] || <span class="ruby-string">&quot;appscale&quot;</span>
+  if job_data[<span class="ruby-string">&quot;@appscale_tools&quot;</span>]
+    upload_app = <span class="ruby-constant">File</span>.expand_path(job_data[<span class="ruby-string">&quot;@appscale_tools&quot;</span>]) +
+      <span class="ruby-constant">File</span>::<span class="ruby-constant">SEPARATOR</span> + <span class="ruby-string">&quot;bin&quot;</span> + <span class="ruby-constant">File</span>::<span class="ruby-constant">SEPARATOR</span> + <span class="ruby-string">&quot;appscale-upload-app&quot;</span>
+  else
+    upload_app = <span class="ruby-string">&quot;appscale-upload-app&quot;</span>
+  end
+
+  puts &quot;Uploading AppEngine app at #{app_location}&quot;
+  upload_command = &quot;#{upload_app} --file #{app_location} --test --keyname #{keyname}&quot;
+  puts upload_command
+  puts `#{upload_command}`
+end</pre>
+          </div>
+          
+        </div>
+
+        
+
+        
       </div>
 
     
@@ -838,7 +899,7 @@ end</pre>
           <div class="method-source-code"
             id="validate_storage_params-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 186</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 195</span>
 def validate_storage_params(job_data)
   if !job_data[<span class="ruby-string">&quot;@storage&quot;</span>]
     job_data[<span class="ruby-string">&quot;@storage&quot;</span>] = <span class="ruby-string">&quot;appdb&quot;</span>
@@ -908,7 +969,7 @@ location.</p>
           <div class="method-source-code"
             id="wait_for_compilation_to_finish-source">
 <pre>
-<span class="ruby-comment"># File lib/neptune.rb, line 260</span>
+<span class="ruby-comment"># File lib/neptune.rb, line 269</span>
 def wait_for_compilation_to_finish(ssh_args, shadow_ip, compiled_location,
   shell=<span class="ruby-constant">Kernel</span>.method(:`))
   loop {

data/doc/created.rid

@@ -1,7 +1,5 @@
-Thu, 10 Nov 2011 12:27:04 -0800
-./lib/common_functions.rb	Wed, 26 Oct 2011 10:37:34 -0700
-./LICENSE	Thu, 27 Jan 2011 13:24:30 -0800
-./lib/app_controller_client.rb	Tue, 25 Oct 2011 23:47:47 -0700
-./lib/neptune.rb	Wed, 09 Nov 2011 10:54:28 -0800
-./README	Thu, 10 Nov 2011 11:48:11 -0800
-./bin/neptune	Sun, 15 May 2011 22:52:56 -0700
+Sun, 04 Dec 2011 11:46:34 -0800
+bin/neptune	Sun, 15 May 2011 22:52:56 -0700
+lib/neptune.rb	Sat, 03 Dec 2011 23:03:43 -0800
+lib/app_controller_client.rb	Tue, 25 Oct 2011 23:47:47 -0700
+lib/common_functions.rb	Wed, 26 Oct 2011 10:37:34 -0700

data/doc/index.html

@@ -27,15 +27,6 @@
 
   
   
-  <h2>Files</h2>
-  <ul>
-    
-      <li class="file"><a href="LICENSE.html">LICENSE</a></li>
-    
-      <li class="file"><a href="README.html">README</a></li>
-    
-  </ul>
-  
 
   <h2 id="classes">Classes/Modules</h2>
   <ul>
@@ -63,10 +54,10 @@
     
       <li><a href="CommonFunctions.html#method-c-scp_to_shadow">::scp_to_shadow &mdash; CommonFunctions</a></li>
     
-      <li><a href="AppControllerClient.html#method-i-compile_code">#compile_code &mdash; AppControllerClient</a></li>
-    
       <li><a href="Object.html#method-i-compile_code">#compile_code &mdash; Object</a></li>
     
+      <li><a href="AppControllerClient.html#method-i-compile_code">#compile_code &mdash; AppControllerClient</a></li>
+    
       <li><a href="Object.html#method-i-do_preprocessing">#do_preprocessing &mdash; Object</a></li>
     
       <li><a href="AppControllerClient.html#method-i-get_acl">#get_acl &mdash; AppControllerClient</a></li>
@@ -101,6 +92,8 @@
     
       <li><a href="AppControllerClient.html#method-i-start_neptune_job">#start_neptune_job &mdash; AppControllerClient</a></li>
     
+      <li><a href="Object.html#method-i-upload_app_for_cicero">#upload_app_for_cicero &mdash; Object</a></li>
+    
       <li><a href="Object.html#method-i-validate_storage_params">#validate_storage_params &mdash; Object</a></li>
     
       <li><a href="Object.html#method-i-wait_for_compilation_to_finish">#wait_for_compilation_to_finish &mdash; Object</a></li>

data/doc/lib/neptune_rb.html

@@ -24,7 +24,7 @@
   <div id="metadata">
     <dl>
       <dt class="modified-date">Last Modified</dt>
-      <dd class="modified-date">Wed Nov 09 10:54:28 -0800 2011</dd>
+      <dd class="modified-date">Sat Dec 03 23:03:43 -0800 2011</dd>
 
       
       <dt class="requires">Requires</dt>

data/lib/neptune.rb

@@ -166,6 +166,15 @@ def get_job_data(params)
     type = "mpi"
   end
 
+  # kdt jobs also run as mpi jobs, but need to pass along an executable
+  # parameter to let mpiexec know to use python to exec it
+  if type == "kdt"
+    job_data["@type"] = "mpi"
+    type = "mpi"
+
+    job_data["@executable"] = "python"
+  end
+
   if job_data["@nodes_to_use"].class == Hash
     job_data["@nodes_to_use"] = job_data["@nodes_to_use"].to_a.flatten
   end
@@ -325,6 +334,32 @@ def get_std_out_and_err(location)
   return result
 end
 
+def upload_app_for_cicero(job_data)
+  if !job_data["@app"]
+    puts "No app specified, not uploading..." 
+    return
+  end
+
+  app_location = File.expand_path(job_data["@app"])
+  if !File.exists?(app_location)
+    abort("The app you specified, #{app_location}, does not exist." + 
+      "Please specify one that does and try again.")
+  end
+
+  keyname = job_data["@keyname"] || "appscale"
+  if job_data["@appscale_tools"]
+    upload_app = File.expand_path(job_data["@appscale_tools"]) +
+      File::SEPARATOR + "bin" + File::SEPARATOR + "appscale-upload-app"
+  else
+    upload_app = "appscale-upload-app"
+  end
+
+  puts "Uploading AppEngine app at #{app_location}"
+  upload_command = "#{upload_app} --file #{app_location} --test --keyname #{keyname}"
+  puts upload_command
+  puts `#{upload_command}`
+end
+
 # This method actually runs the Neptune job, given information about the job
 # as well as information about the node to send the request to.
 def run_job(job_data, ssh_args, shadow_ip, secret,
@@ -348,6 +383,11 @@ def run_job(job_data, ssh_args, shadow_ip, secret,
     result[:acl] = controller.set_acl(job_data)
   when "compile"
     result = compile_code(job_data, ssh_args, shadow_ip)
+  when "cicero"
+    upload_app_for_cicero(job_data)
+    msg = controller.start_neptune_job(job_data)
+    result[:msg] = msg
+    result[:result] = :failure if result[:msg] !~ /job is now running\Z/
   else
     msg = controller.start_neptune_job(job_data)
     result[:msg] = msg

data/samples/cicero/run_appengine_java_dfsp.rb

@@ -0,0 +1,7 @@
+response = neptune :type => :cicero,
+                   :nodes_to_use => {"cloud1" => 1},
+                   :tasks => 10,
+                   :function => "map",
+                   :output => "/output/mr/"
+
+puts response.inspect

data/samples/cicero/run_appengine_mapreduce.rb

@@ -0,0 +1,8 @@
+response = neptune :type => :cicero,
+                   :nodes_to_use => {"cloud1" => "http://appscale-benchmark4.appspot.com"},
+                   :tasks => 10,
+                   :function => "map",
+                   :input1 => "boo1",
+                   :output => "/output/mr/"
+
+puts response.inspect

data/samples/cicero/run_appengine_mapreduce_with_app.rb

@@ -0,0 +1,11 @@
+response = neptune :type => :cicero,
+                   :nodes_to_use => {"cloud1" => 1},
+                   :tasks => 10,
+                   :function => "map",
+                   :input1 => "boo1",
+                   :output => "/output/mr/",
+                   :app => "~/mapreduce-taskq",
+                   :appscale_tools => "~/clients/tools/trunk-tools",
+                   :app_name => "appscale-benchmark4"
+
+puts response.inspect

data/samples/cicero/run_appengine_python_dfsp_with_app.rb

@@ -0,0 +1,10 @@
+response = neptune :type => :cicero,
+                   :nodes_to_use => {"cloud1" => 1},
+                   :tasks => 10,
+                   :function => "dfsp",
+                   :output => "/output/mr/",
+                   :app => "~/dfsp-taskq",
+                   :appscale_tools => "~/clients/tools/trunk-tools",
+                   :app_name => "appscale-benchmark0"
+
+puts response.inspect

data/samples/cicero/run_appengine_python_dfsp_with_two_nodes.rb

@@ -0,0 +1,8 @@
+response = neptune :type => :cicero,
+                   :keyname => "booscale2",
+                   :nodes_to_use => {"cloud1" => "http://ec2-107-20-95-123.compute-1.amazonaws.com:8080"},
+                   :tasks => 100,
+                   :function => "dfsp",
+                   :output => "/output/mr/"
+
+puts response.inspect

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: neptune
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 29
   prerelease: 
   segments: 
   - 0
   - 1
-  - 2
-  version: 0.1.2
+  - 3
+  version: 0.1.3
 platform: ruby
 authors: 
 - Chris Bunch
@@ -15,7 +15,7 @@ autorequire: neptune
 bindir: bin
 cert_chain: []
 
-date: 2011-11-10 00:00:00 -08:00
+date: 2011-12-04 00:00:00 -08:00
 default_executable: neptune
 dependencies: []
 
@@ -30,7 +30,6 @@ extra_rdoc_files:
 - LICENSE
 files: 
 - bin/neptune
-- doc/README.html
 - doc/CommonFunctions.html
 - doc/Object.html
 - doc/js/jquery.js
@@ -38,8 +37,6 @@ files:
 - doc/js/quicksearch.js
 - doc/js/thickbox-compressed.js
 - doc/AppControllerClient.html
-- doc/LICENSE.html
-- doc/lib/job_rb.html
 - doc/lib/neptune_rb.html
 - doc/lib/app_controller_client_rb.html
 - doc/lib/common_functions_rb.html
@@ -65,43 +62,6 @@ files:
 - doc/images/bullet_toggle_plus.png
 - doc/images/package.png
 - doc/images/bullet_toggle_minus.png
-- doc/scripts/cewssa/run_cewssa_rb.html
-- doc/scripts/cewssa/test_dfsp_rb.html
-- doc/scripts/cewssa/get_cewssa_output_rb.html
-- doc/scripts/cewssa/cewssa-output.html
-- doc/scripts/dfsp/get_dfsp_output_rb.html
-- doc/scripts/dfsp/test_dfsp_rb.html
-- doc/scripts/dfsp/make_output_public_rb.html
-- doc/scripts/dfsp/run_dfsp_rb.html
-- doc/scripts/dfsp/see_acl_rb.html
-- doc/scripts/appscale/add_slave_rb.html
-- doc/scripts/appscale/add_appserver_rb.html
-- doc/scripts/appscale/add_loadbalancer_rb.html
-- doc/scripts/appscale/add_database_rb.html
-- doc/scripts/x10/run_x10_qsort_rb.html
-- doc/scripts/x10/run_x10_nqueens_rb.html
-- doc/scripts/x10/Makefile.html
-- doc/scripts/x10/run_x10_montypi_rb.html
-- doc/scripts/x10/run_x10_powermethod_rb.html
-- doc/scripts/x10/test_rb.html
-- doc/scripts/x10/get_x10_output_rb.html
-- doc/scripts/x10/run_x10_kmeans_rb.html
-- doc/scripts/mapreduce/run_mapreduce_rb.html
-- doc/scripts/mapreduce/run_java_mr_rb.html
-- doc/scripts/mapreduce/input-7.html
-- doc/scripts/mapreduce/input-30.html
-- doc/scripts/mapreduce/get_mapreduce_output_rb.html
-- doc/scripts/mapreduce/reduce_rb.html
-- doc/scripts/mapreduce/input-10.html
-- doc/scripts/mapreduce/map_rb.html
-- doc/scripts/mpi/hw2harness_h.html
-- doc/scripts/mpi/Makefile.html
-- doc/scripts/mpi/hw2harness_c.html
-- doc/scripts/mpi/get_mpi_output_rb.html
-- doc/scripts/mpi/hw2_c.html
-- doc/scripts/mpi/test_rb.html
-- doc/scripts/mpi/run_mpi_powermethod_rb.html
-- doc/scripts/mpi/run_mpi_cpi_rb.html
 - doc/created.rid
 - doc/Kernel.html
 - lib/common_functions.rb
@@ -136,6 +96,11 @@ files:
 - samples/go/hello/hello.go
 - samples/go/compile_hello.rb
 - samples/go/put_input.rb
+- samples/cicero/run_appengine_java_dfsp.rb
+- samples/cicero/run_appengine_mapreduce.rb
+- samples/cicero/run_appengine_python_dfsp_with_two_nodes.rb
+- samples/cicero/run_appengine_python_dfsp_with_app.rb
+- samples/cicero/run_appengine_mapreduce_with_app.rb
 - samples/r/run_hello.rb
 - samples/r/get_hello_output.rb
 - samples/r/hello/hello.r