neptune 0.2.2 → 0.2.3

data/doc/ExodusHelper.html

@@ -170,6 +170,18 @@ cluttering up <a href="Object.html">Object</a> or Kernel’s namespace.</p>
         <dd class="description">
         
       
+        <dt id="GET_CPU_INFO">GET_CPU_INFO
+        
+        <dd class="description"><p>The command that we can run to get information about the number and speed
+of CPUs on this machine.</p>
+        
+      
+        <dt id="NEPTUNE_DATA_DIR">NEPTUNE_DATA_DIR
+        
+        <dd class="description"><p>The location on this machine where we can read and write profiling
+information about jobs.</p>
+        
+      
         <dt id="OPTIMIZE_FOR_CHOICES">OPTIMIZE_FOR_CHOICES
         
         <dd class="description">
@@ -208,7 +220,7 @@ cluttering up <a href="Object.html">Object</a> or Kernel’s namespace.</p>
 
           
           <div class="method-source-code" id="average-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 268</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 316</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">average</span>(<span class="ruby-identifier">vals</span>)
   <span class="ruby-identifier">sum</span> = <span class="ruby-identifier">vals</span>.<span class="ruby-identifier">reduce</span>(<span class="ruby-value">0.0</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">running_total</span>, <span class="ruby-identifier">val</span><span class="ruby-operator">|</span>
     <span class="ruby-identifier">running_total</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">val</span>
@@ -246,7 +258,7 @@ href="BadConfigurationException.html">BadConfigurationException</a> if
 
           
           <div class="method-source-code" id="convert_clouds_to_use_to_array-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 123</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 138</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">convert_clouds_to_use_to_array</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-identifier">clouds_class</span> = <span class="ruby-identifier">job</span>[<span class="ruby-value">:clouds_to_use</span>].<span class="ruby-identifier">class</span>
   <span class="ruby-keyword">if</span> <span class="ruby-identifier">clouds_class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Symbol</span>
@@ -290,7 +302,7 @@ standard format for Neptune jobs.</p>
 
           
           <div class="method-source-code" id="ensure_all_jobs_are_hashes-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 95</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 110</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">ensure_all_jobs_are_hashes</span>(<span class="ruby-identifier">jobs</span>)
   <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">job</span><span class="ruby-operator">|</span>
     <span class="ruby-keyword">if</span> <span class="ruby-identifier">job</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">!=</span> <span class="ruby-constant">Hash</span>
@@ -327,7 +339,7 @@ missing params.</p>
 
           
           <div class="method-source-code" id="ensure_all_params_are_present-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 107</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 122</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">ensure_all_params_are_present</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-keyword">if</span> <span class="ruby-identifier">job</span>[<span class="ruby-value">:clouds_to_use</span>].<span class="ruby-identifier">nil?</span>
     <span class="ruby-identifier">raise</span> <span class="ruby-constant">BadConfigurationException</span>.<span class="ruby-identifier">new</span>(<span class="ruby-string">&quot;:clouds_to_use was not specified&quot;</span>)
@@ -364,7 +376,7 @@ missing params.</p>
 
           
           <div class="method-source-code" id="ensure_credentials_are_in_correct_format-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 165</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 180</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">ensure_credentials_are_in_correct_format</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-keyword">if</span> <span class="ruby-identifier">job</span>[<span class="ruby-value">:credentials</span>].<span class="ruby-identifier">nil?</span>
     <span class="ruby-identifier">raise</span> <span class="ruby-constant">BadConfigurationException</span>.<span class="ruby-identifier">new</span>(<span class="ruby-string">&quot;No credentials were specified.&quot;</span>)
@@ -401,7 +413,7 @@ missing params.</p>
 
           
           <div class="method-source-code" id="find_optimal_cloud_for_task-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 277</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 325</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">find_optimal_cloud_for_task</span>(<span class="ruby-identifier">job</span>, <span class="ruby-identifier">profiling_info</span>)
   <span class="ruby-identifier">raise</span> <span class="ruby-constant">NotImplementedError</span>
 <span class="ruby-keyword">end</span></pre>
@@ -431,7 +443,7 @@ missing params.</p>
 
           
           <div class="method-source-code" id="generate_babel_tasks-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 282</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 330</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">generate_babel_tasks</span>(<span class="ruby-identifier">job</span>, <span class="ruby-identifier">clouds_to_run_task_on</span>)
   <span class="ruby-identifier">tasks</span> = []
 
@@ -480,7 +492,7 @@ missing params.</p>
 
           
           <div class="method-source-code" id="get_clouds_to_run_task_on-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 227</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 275</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_clouds_to_run_task_on</span>(<span class="ruby-identifier">job</span>, <span class="ruby-identifier">profiling_info</span>)
   <span class="ruby-identifier">optimize_for</span> = <span class="ruby-identifier">job</span>[<span class="ruby-value">:optimize_for</span>]
   <span class="ruby-keyword">if</span> <span class="ruby-identifier">optimize_for</span> <span class="ruby-operator">==</span> <span class="ruby-value">:performance</span> <span class="ruby-keyword">or</span> <span class="ruby-identifier">optimize_for</span> <span class="ruby-operator">==</span> <span class="ruby-value">:cost</span>
@@ -515,9 +527,9 @@ missing params.</p>
 
           
           <div class="method-source-code" id="get_key_from_job_data-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 222</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 270</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_key_from_job_data</span>(<span class="ruby-identifier">job</span>)
-  <span class="ruby-keyword">return</span> <span class="ruby-identifier">job</span>[<span class="ruby-value">:code</span>]
+  <span class="ruby-keyword">return</span> <span class="ruby-identifier">job</span>[<span class="ruby-value">:code</span>].<span class="ruby-identifier">gsub</span>(<span class="ruby-regexp">%r[\/\.]/</span>, <span class="ruby-string">&quot;&quot;</span>)
 <span class="ruby-keyword">end</span></pre>
           </div><!-- get_key_from_job_data-source -->
           
@@ -545,7 +557,7 @@ missing params.</p>
 
           
           <div class="method-source-code" id="get_minimum_val_in_data-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 237</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 285</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_minimum_val_in_data</span>(<span class="ruby-identifier">job</span>, <span class="ruby-identifier">profiling_info</span>)
   <span class="ruby-identifier">min_cloud</span> = <span class="ruby-keyword">nil</span>
   <span class="ruby-identifier">min_val</span> = <span class="ruby-value">1_000_000</span>  <span class="ruby-comment"># infinity</span>
@@ -600,11 +612,44 @@ missing params.</p>
 
           
           <div class="method-source-code" id="get_profiling_info-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 214</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 229</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_profiling_info</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-identifier">key</span> = <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_key_from_job_data</span>(<span class="ruby-identifier">job</span>)
-  <span class="ruby-identifier">neptune_manager</span> = <span class="ruby-constant">BabelHelper</span>.<span class="ruby-identifier">get_neptune_manager_client</span>(<span class="ruby-identifier">job</span>)
-  <span class="ruby-keyword">return</span> <span class="ruby-identifier">neptune_manager</span>.<span class="ruby-identifier">get_profiling_info</span>(<span class="ruby-identifier">key</span>)
+
+  <span class="ruby-keyword">if</span> <span class="ruby-operator">!</span><span class="ruby-constant">File</span>.<span class="ruby-identifier">exists?</span>(<span class="ruby-constant">NEPTUNE_DATA_DIR</span>)
+    <span class="ruby-constant">FileUtils</span>.<span class="ruby-identifier">mkdir</span>(<span class="ruby-constant">NEPTUNE_DATA_DIR</span>)
+  <span class="ruby-keyword">end</span>
+
+  <span class="ruby-identifier">profiling_info_file</span> = <span class="ruby-node">&quot;#{NEPTUNE_DATA_DIR}/#{key}.json&quot;</span>
+  <span class="ruby-keyword">if</span> <span class="ruby-constant">File</span>.<span class="ruby-identifier">exists?</span>(<span class="ruby-identifier">profiling_info_file</span>)
+    <span class="ruby-identifier">contents</span> = <span class="ruby-constant">File</span>.<span class="ruby-identifier">open</span>(<span class="ruby-identifier">profiling_info_file</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">f</span><span class="ruby-operator">|</span> <span class="ruby-identifier">f</span>.<span class="ruby-identifier">read</span>() }
+    <span class="ruby-keyword">return</span> <span class="ruby-constant">JSON</span>.<span class="ruby-identifier">load</span>(<span class="ruby-identifier">contents</span>)
+  <span class="ruby-keyword">end</span>
+
+  <span class="ruby-comment"># If we don't have any profiling info on this job, run it locally and</span>
+  <span class="ruby-comment"># gather the data ourselves.</span>
+
+  <span class="ruby-identifier">start_time</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
+  <span class="ruby-comment"># TODO(cgb): exec the user's code</span>
+  <span class="ruby-identifier">end_time</span> = <span class="ruby-constant">Time</span>.<span class="ruby-identifier">now</span>
+  
+  <span class="ruby-comment"># To find out how fast this computer is, just check the file that has</span>
+  <span class="ruby-comment"># this info on it and take the first processor. This should be fine</span>
+  <span class="ruby-comment"># since we assume the user's code is not-multi-core aware and that</span>
+  <span class="ruby-comment"># all processors on this box are the same speed.</span>
+  <span class="ruby-identifier">cpu_speed</span> = <span class="ruby-constant">Float</span>(<span class="ruby-constant">CommonFunctions</span>.<span class="ruby-identifier">shell</span>(<span class="ruby-constant">GET_CPU_INFO</span>).
+    <span class="ruby-identifier">scan</span>(<span class="ruby-regexp">%rcpu MHz\s*:\s*(\d+\.\d+)/</span>).<span class="ruby-identifier">flatten</span>[<span class="ruby-value">0</span>])
+
+  <span class="ruby-identifier">json_info</span> = {
+    <span class="ruby-string">&quot;total_execution_time&quot;</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-identifier">end_time</span> <span class="ruby-operator">-</span> <span class="ruby-identifier">start_time</span>,
+    <span class="ruby-string">&quot;cpu_speed&quot;</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-identifier">cpu_speed</span>
+  }
+
+  <span class="ruby-constant">File</span>.<span class="ruby-identifier">open</span>(<span class="ruby-identifier">profiling_info_file</span>, <span class="ruby-string">&quot;w+&quot;</span>) { <span class="ruby-operator">|</span><span class="ruby-identifier">file</span><span class="ruby-operator">|</span> 
+    <span class="ruby-identifier">file</span>.<span class="ruby-identifier">write</span>(<span class="ruby-constant">JSON</span>.<span class="ruby-identifier">dump</span>(<span class="ruby-identifier">json_info</span>)) 
+  }
+
+  <span class="ruby-keyword">return</span> <span class="ruby-identifier">json_info</span>
 <span class="ruby-keyword">end</span></pre>
           </div><!-- get_profiling_info-source -->
           
@@ -634,7 +679,7 @@ the job does not specify it.</p>
 
           
           <div class="method-source-code" id="propogate_credentials_from_environment-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 180</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 195</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">propogate_credentials_from_environment</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-constant">CLOUD_CREDENTIALS</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">cloud_name</span>, <span class="ruby-identifier">credential_list</span><span class="ruby-operator">|</span>
     <span class="ruby-identifier">credential_list</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">cred</span><span class="ruby-operator">|</span>
@@ -670,7 +715,7 @@ the job does not specify it.</p>
 
           
           <div class="method-source-code" id="run_job-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 306</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 354</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">tasks_to_run</span>)
   <span class="ruby-keyword">return</span> <span class="ruby-identifier">babel</span>(<span class="ruby-identifier">tasks_to_run</span>)
 <span class="ruby-keyword">end</span></pre>
@@ -702,7 +747,7 @@ credentials needed to use that cloud.</p>
 
           
           <div class="method-source-code" id="validate_clouds_to_use-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 144</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 159</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">validate_clouds_to_use</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-keyword">self</span>.<span class="ruby-identifier">ensure_credentials_are_in_correct_format</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-keyword">self</span>.<span class="ruby-identifier">propogate_credentials_from_environment</span>(<span class="ruby-identifier">job</span>)
@@ -748,7 +793,7 @@ credentials needed to use that cloud.</p>
 
           
           <div class="method-source-code" id="validate_files_argv_executable-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 205</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 220</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">validate_files_argv_executable</span>(<span class="ruby-identifier">job</span>)
   [<span class="ruby-value">:code</span>, <span class="ruby-value">:argv</span>, <span class="ruby-value">:executable</span>].<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">param</span><span class="ruby-operator">|</span>
     <span class="ruby-keyword">if</span> <span class="ruby-identifier">job</span>[<span class="ruby-identifier">param</span>].<span class="ruby-identifier">nil?</span>
@@ -782,7 +827,7 @@ credentials needed to use that cloud.</p>
 
           
           <div class="method-source-code" id="validate_optimize_for_param-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 191</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 206</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">validate_optimize_for_param</span>(<span class="ruby-identifier">job</span>)
   <span class="ruby-keyword">if</span> <span class="ruby-identifier">job</span>[<span class="ruby-value">:optimize_for</span>].<span class="ruby-identifier">nil?</span>
     <span class="ruby-identifier">raise</span> <span class="ruby-constant">BadConfigurationException</span>.<span class="ruby-identifier">new</span>(<span class="ruby-string">&quot;:optimize_for needs to be &quot;</span> <span class="ruby-operator">+</span>

data/doc/NeptuneHelper.html

@@ -186,7 +186,7 @@ indicates whether or not the compilation was successful.</p>
 
           
           <div class="method-source-code" id="compile_code-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 458</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 459</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">compile_code</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">ssh_args</span>, <span class="ruby-identifier">shadow_ip</span>)
   <span class="ruby-identifier">compiled_location</span> = <span class="ruby-identifier">controller</span>.<span class="ruby-identifier">compile_code</span>(<span class="ruby-identifier">job_data</span>)
   <span class="ruby-identifier">copy_to</span> = <span class="ruby-identifier">job_data</span>[<span class="ruby-string">&quot;@copy_to&quot;</span>]
@@ -278,7 +278,7 @@ job succeeded and if it failed, the reason for it.</p>
 
           
           <div class="method-source-code" id="get_input-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 406</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 407</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_input</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">ssh_args</span>, <span class="ruby-identifier">shadow_ip</span>, <span class="ruby-identifier">controller</span>)
   <span class="ruby-identifier">result</span> = {<span class="ruby-value">:result</span> =<span class="ruby-operator">&gt;</span> <span class="ruby-value">:success</span>}
 
@@ -335,7 +335,7 @@ NeptuneManager.</p>
 
           
           <div class="method-source-code" id="get_job_data-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 313</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 314</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_job_data</span>(<span class="ruby-identifier">params</span>)
   <span class="ruby-identifier">job_data</span> = {}
   <span class="ruby-identifier">params</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">k</span>, <span class="ruby-identifier">v</span><span class="ruby-operator">|</span>
@@ -411,7 +411,7 @@ not the job completed successfully (success = no errors).</p>
 
           
           <div class="method-source-code" id="get_std_out_and_err-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 486</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 487</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">get_std_out_and_err</span>(<span class="ruby-identifier">location</span>)
   <span class="ruby-identifier">result</span> = {}
 
@@ -458,7 +458,7 @@ engines can be found by contacting an AppScale node.</p>
 
           
           <div class="method-source-code" id="preprocess_babel-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 261</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 262</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">preprocess_babel</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
   <span class="ruby-keyword">self</span>.<span class="ruby-identifier">require_param</span>(<span class="ruby-string">&quot;@code&quot;</span>, <span class="ruby-identifier">job_data</span>)
   <span class="ruby-keyword">self</span>.<span class="ruby-identifier">require_param</span>(<span class="ruby-string">&quot;@engine&quot;</span>, <span class="ruby-identifier">job_data</span>)
@@ -731,7 +731,8 @@ it does not, throws an exception.</p>
   <span class="ruby-keyword">if</span> <span class="ruby-identifier">controller</span>.<span class="ruby-identifier">does_file_exist?</span>(<span class="ruby-identifier">file</span>, <span class="ruby-identifier">job_data</span>)
     <span class="ruby-keyword">return</span>
   <span class="ruby-keyword">else</span>
-    <span class="ruby-identifier">raise</span> <span class="ruby-constant">FileNotFoundException</span>
+    <span class="ruby-identifier">raise</span> <span class="ruby-constant">FileNotFoundException</span>.<span class="ruby-identifier">new</span>(<span class="ruby-node">&quot;Expecting file #{file} to exist &quot;</span> <span class="ruby-operator">+</span>
+      <span class="ruby-string">&quot;in the remote datastore, which did not exist.&quot;</span>)
   <span class="ruby-keyword">end</span>
 <span class="ruby-keyword">end</span></pre>
           </div><!-- require_file_to_exist-source -->
@@ -762,7 +763,7 @@ raising an exception if the named file does exist.</p>
 
           
           <div class="method-source-code" id="require_file_to_not_exist-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 246</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 247</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">require_file_to_not_exist</span>(<span class="ruby-identifier">file</span>, <span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
   <span class="ruby-keyword">begin</span>
     <span class="ruby-keyword">self</span>.<span class="ruby-identifier">require_file_to_exist</span>(<span class="ruby-identifier">file</span>, <span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">controller</span>)
@@ -832,7 +833,7 @@ as well as information about the node to send the request to.</p>
 
           
           <div class="method-source-code" id="run_job-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 536</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 537</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">run_job</span>(<span class="ruby-identifier">job_data</span>, <span class="ruby-identifier">ssh_args</span>, <span class="ruby-identifier">shadow_ip</span>, <span class="ruby-identifier">secret</span>)
   <span class="ruby-identifier">controller</span> = <span class="ruby-constant">NeptuneManagerClient</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">shadow_ip</span>, <span class="ruby-identifier">secret</span>)
 
@@ -893,7 +894,7 @@ with Cicero jobs. It requires the AppScale tools to be installed.</p>
 
           
           <div class="method-source-code" id="upload_app_for_cicero-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 507</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 508</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">upload_app_for_cicero</span>(<span class="ruby-identifier">job_data</span>)
   <span class="ruby-keyword">if</span> <span class="ruby-operator">!</span><span class="ruby-identifier">job_data</span>[<span class="ruby-string">&quot;@app&quot;</span>]
     <span class="ruby-comment"># Kernel.puts &quot;No app specified, not uploading...&quot; </span>
@@ -948,7 +949,7 @@ parameter is missing.</p>
 
           
           <div class="method-source-code" id="validate_storage_params-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 366</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 367</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">validate_storage_params</span>(<span class="ruby-identifier">job_data</span>)
   <span class="ruby-identifier">job_data</span>[<span class="ruby-string">&quot;@storage&quot;</span>] <span class="ruby-operator">||=</span> <span class="ruby-string">&quot;appdb&quot;</span>
 
@@ -1012,7 +1013,7 @@ location.</p>
 
           
           <div class="method-source-code" id="wait_for_compilation_to_finish-source">
-            <pre><span class="ruby-comment"># File lib/neptune.rb, line 437</span>
+            <pre><span class="ruby-comment"># File lib/neptune.rb, line 438</span>
 <span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier">wait_for_compilation_to_finish</span>(<span class="ruby-identifier">ssh_args</span>, <span class="ruby-identifier">shadow_ip</span>, <span class="ruby-identifier">compiled_location</span>)
   <span class="ruby-identifier">loop</span> {
     <span class="ruby-identifier">ssh_command</span> = <span class="ruby-node">&quot;ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2&gt;&amp;1&quot;</span>

data/doc/Object.html

@@ -329,7 +329,7 @@ select the best cloud for their job and run it there.</p>
 
           
           <div class="method-source-code" id="exodus-source">
-            <pre><span class="ruby-comment"># File lib/exodus.rb, line 15</span>
+            <pre><span class="ruby-comment"># File lib/exodus.rb, line 20</span>
 <span class="ruby-keyword">def</span> <span class="ruby-identifier">exodus</span>(<span class="ruby-identifier">jobs</span>)
   <span class="ruby-keyword">if</span> <span class="ruby-identifier">jobs</span>.<span class="ruby-identifier">class</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Hash</span>
     <span class="ruby-identifier">job_given_as_hash</span> = <span class="ruby-keyword">true</span>

data/doc/created.rid

@@ -1,10 +1,10 @@
-Mon, 30 Jul 2012 17:12:01 -0700
-bin/neptune	Sun, 12 Feb 2012 16:12:33 -0800
-lib/exodus.rb	Mon, 28 May 2012 21:02:31 -0700
-lib/task_info.rb	Mon, 28 May 2012 16:24:14 -0700
-lib/babel.rb	Mon, 30 Jul 2012 17:10:06 -0700
-lib/neptune_manager_client.rb	Mon, 04 Jun 2012 13:53:07 -0700
-lib/common_functions.rb	Sun, 12 Feb 2012 16:18:14 -0800
-lib/neptune.rb	Mon, 30 Jul 2012 17:09:09 -0700
+Fri, 28 Sep 2012 13:17:21 -0700
+lib/babel.rb	Sat, 01 Sep 2012 13:14:23 -0700
+lib/exodus.rb	Tue, 25 Sep 2012 12:38:28 -0700
 lib/custom_exceptions.rb	Mon, 28 May 2012 16:23:52 -0700
-lib/exodus_task_info.rb	Mon, 28 May 2012 21:03:03 -0700
+lib/neptune_manager_client.rb	Sat, 01 Sep 2012 13:14:23 -0700
+lib/exodus_task_info.rb	Mon, 30 Jul 2012 17:40:12 -0700
+lib/common_functions.rb	Sat, 01 Sep 2012 13:14:23 -0700
+lib/neptune.rb	Fri, 28 Sep 2012 13:12:58 -0700
+lib/task_info.rb	Mon, 28 May 2012 16:24:14 -0700
+bin/neptune	Sat, 01 Sep 2012 13:14:23 -0700

data/doc/table_of_contents.html

@@ -119,10 +119,10 @@
   
     <li class="method"><a href="ExodusTaskInfo.html#method-c-new">::new &mdash; ExodusTaskInfo</a>
   
-    <li class="method"><a href="NeptuneManagerClient.html#method-c-new">::new &mdash; NeptuneManagerClient</a>
-  
     <li class="method"><a href="TaskInfo.html#method-c-new">::new &mdash; TaskInfo</a>
   
+    <li class="method"><a href="NeptuneManagerClient.html#method-c-new">::new &mdash; NeptuneManagerClient</a>
+  
     <li class="method"><a href="NeptuneHelper.html#method-c-preprocess_babel">::preprocess_babel &mdash; NeptuneHelper</a>
   
     <li class="method"><a href="NeptuneHelper.html#method-c-preprocess_compile">::preprocess_compile &mdash; NeptuneHelper</a>
@@ -147,12 +147,12 @@
   
     <li class="method"><a href="NeptuneHelper.html#method-c-require_param">::require_param &mdash; NeptuneHelper</a>
   
+    <li class="method"><a href="ExodusHelper.html#method-c-run_job">::run_job &mdash; ExodusHelper</a>
+  
     <li class="method"><a href="BabelHelper.html#method-c-run_job">::run_job &mdash; BabelHelper</a>
   
     <li class="method"><a href="NeptuneHelper.html#method-c-run_job">::run_job &mdash; NeptuneHelper</a>
   
-    <li class="method"><a href="ExodusHelper.html#method-c-run_job">::run_job &mdash; ExodusHelper</a>
-  
     <li class="method"><a href="CommonFunctions.html#method-c-scp_file">::scp_file &mdash; CommonFunctions</a>
   
     <li class="method"><a href="CommonFunctions.html#method-c-scp_to_shadow">::scp_to_shadow &mdash; CommonFunctions</a>
@@ -211,10 +211,10 @@
   
     <li class="method"><a href="TaskInfo.html#method-i-to_json">#to_json &mdash; TaskInfo</a>
   
-    <li class="method"><a href="ExodusTaskInfo.html#method-i-to_s">#to_s &mdash; ExodusTaskInfo</a>
-  
     <li class="method"><a href="TaskInfo.html#method-i-to_s">#to_s &mdash; TaskInfo</a>
   
+    <li class="method"><a href="ExodusTaskInfo.html#method-i-to_s">#to_s &mdash; ExodusTaskInfo</a>
+  
 </ul>
 
 

data/lib/exodus.rb

@@ -2,7 +2,12 @@
 # Programmer: Chris Bunch
 
 
+require 'rubygems'
+require 'json'
+
+
 require 'babel'
+require 'common_functions'
 require 'custom_exceptions'
 require 'exodus_task_info'
 
@@ -87,6 +92,16 @@ module ExodusHelper
   }
 
 
+  # The location on this machine where we can read and write profiling
+  # information about jobs.
+  NEPTUNE_DATA_DIR = File.expand_path("~/.neptune")
+
+
+  # The command that we can run to get information about the number and speed
+  # of CPUs on this machine.
+  GET_CPU_INFO = "cat /proc/cpuinfo"
+
+
   OPTIMIZE_FOR_CHOICES = [:performance, :cost, :auto]
 
 
@@ -213,14 +228,47 @@ module ExodusHelper
 
   def self.get_profiling_info(job)
     key = self.get_key_from_job_data(job)
-    neptune_manager = BabelHelper.get_neptune_manager_client(job)
-    return neptune_manager.get_profiling_info(key)
+
+    if !File.exists?(NEPTUNE_DATA_DIR)
+      FileUtils.mkdir(NEPTUNE_DATA_DIR)
+    end
+
+    profiling_info_file = "#{NEPTUNE_DATA_DIR}/#{key}.json"
+    if File.exists?(profiling_info_file)
+      contents = File.open(profiling_info_file) { |f| f.read() }
+      return JSON.load(contents)
+    end
+
+    # If we don't have any profiling info on this job, run it locally and
+    # gather the data ourselves.
+
+    start_time = Time.now
+    # TODO(cgb): exec the user's code
+    end_time = Time.now
+    
+    # To find out how fast this computer is, just check the file that has
+    # this info on it and take the first processor. This should be fine
+    # since we assume the user's code is not-multi-core aware and that
+    # all processors on this box are the same speed.
+    cpu_speed = Float(CommonFunctions.shell(GET_CPU_INFO).
+      scan(/cpu MHz\s*:\s*(\d+\.\d+)/).flatten[0])
+
+    json_info = {
+      "total_execution_time" => end_time - start_time,
+      "cpu_speed" => cpu_speed
+    }
+
+    File.open(profiling_info_file, "w+") { |file| 
+      file.write(JSON.dump(json_info)) 
+    }
+
+    return json_info
   end
 
 
   # TODO(cgb): what is a job's key?
   def self.get_key_from_job_data(job)
-    return job[:code]
+    return job[:code].gsub(/[\/\.]/, "")
   end
 
   

data/lib/neptune.rb

@@ -236,7 +236,8 @@ module NeptuneHelper
     if controller.does_file_exist?(file, job_data)
       return
     else
-      raise FileNotFoundException
+      raise FileNotFoundException.new("Expecting file #{file} to exist " +
+        "in the remote datastore, which did not exist.")
     end
   end
 

data/test/test_exodus.rb

@@ -36,6 +36,8 @@ class TestExodus < Test::Unit::TestCase
       :S3_URL => "http://s3.url",
       :S3_bucket_name => "bazbucket"
     }
+
+    flexmock(FileUtils).should_receive(:mkdir).and_return()
   end
 
 
@@ -223,16 +225,38 @@ class TestExodus < Test::Unit::TestCase
       :executable => "ruby"
     }
 
-    # mock out the SOAP call for get_profiling_info
-    no_job_data = {
-    }
-    flexmock(NeptuneManagerClient).new_instances { |instance|
-      instance.should_receive(:get_profiling_info).with(job[:code]).
-        and_return(no_job_data)
-    }
+    # let's say they've never run an exodus job before, so they have
+    # no profiling data stored locally
+    flexmock(File).should_receive(:exists?).
+      with(ExodusHelper::NEPTUNE_DATA_DIR).and_return(false)
+    flexmock(FileUtils).should_receive(:mkdir).
+      with(ExodusHelper::NEPTUNE_DATA_DIR).and_return()
+
+    # mock out the gathering of profiling info
+    profiling_filename = "#{ExodusHelper::NEPTUNE_DATA_DIR}/#{ExodusHelper.get_key_from_job_data(job)}.json"
+    flexmock(File).should_receive(:exists?).
+      with(profiling_filename).and_return(false)
+
+    # mock out timing the user's code
+    flexmock(Time).should_receive(:now).and_return(1.0, 2.0)
 
-    profiling_info = ExodusHelper.get_profiling_info(job)
-    assert_equal(true, profiling_info.empty?)
+    # then mock out exec'ing the user's code
+    # TODO(cgb): do this
+
+    # mock out getting cpuinfo
+    flexmock(CommonFunctions).should_receive(:shell).
+      with(ExodusHelper::GET_CPU_INFO).and_return("cpu MHz: 666.66")
+
+    # finally, mock out writing the profiling information
+    flexmock(File).should_receive(:open).with(profiling_filename, "w+", Proc).
+      and_return()
+
+    expected = {
+      "total_execution_time" => 1.0,
+      "cpu_speed" => 666.66
+    }
+    actual = ExodusHelper.get_profiling_info(job)
+    assert_equal(expected, actual)
   end
 
 
@@ -526,10 +550,6 @@ class TestExodus < Test::Unit::TestCase
 
     # mock out calls to the NeptuneManager
     flexmock(NeptuneManagerClient).new_instances { |instance|
-      # for this test, let's say there's no data on this task right now
-      instance.should_receive(:get_profiling_info).with(String).
-        and_return({})
-
       # let's say that all checks to see if temp files exist tell us that
       # the files don't exist
       instance.should_receive(:does_file_exist?).
@@ -562,6 +582,31 @@ class TestExodus < Test::Unit::TestCase
     # we'll say that our code does exist
     flexmock(File).should_receive(:exists?).with("/foo").and_return(true)
 
+    # let's say that we have a neptune profiling directory
+    flexmock(File).should_receive(:exists?).
+      with(ExodusHelper::NEPTUNE_DATA_DIR).and_return(true)
+
+    # and let's say that we've never run the job before
+    # start by mocking out its filesystem reads
+    key = ExodusHelper.get_key_from_job_data(job)
+    profiling_key = "#{ExodusHelper::NEPTUNE_DATA_DIR}/#{key}.json"
+    flexmock(File).should_receive(:exists?).
+      with(profiling_key).and_return(false)
+
+    # mock out timing the user's code
+    flexmock(Time).should_receive(:now).and_return(1.0, 2.0)
+
+    # then mock out exec'ing the user's code
+    # TODO(cgb): do this
+
+    # mock out getting cpuinfo
+    flexmock(CommonFunctions).should_receive(:shell).
+      with(ExodusHelper::GET_CPU_INFO).and_return("cpu MHz: 666.66")
+
+    # finally, mock out writing the profiling information
+    flexmock(File).should_receive(:open).with(profiling_key, "w+", Proc).
+      and_return()
+
     # mock out scp calls - assume they go through with no problems
     flexmock(CommonFunctions).should_receive(:shell).with(/\Ascp/).
       and_return()
@@ -605,12 +650,24 @@ class TestExodus < Test::Unit::TestCase
 
     job2 = job.dup
 
+    # let's say that we have a neptune profiling directory
+    flexmock(File).should_receive(:exists?).
+      with(ExodusHelper::NEPTUNE_DATA_DIR).and_return(true)
+
+    # and let's say that we've run the job before
+    key = ExodusHelper.get_key_from_job_data(job)
+    profiling_filename = "#{ExodusHelper::NEPTUNE_DATA_DIR}/#{key}.json"
+    dumped_data = JSON.dump({
+      "total_execution_time" => 60.00,
+      "cpu_speed" => 666.66
+    })
+    flexmock(File).should_receive(:exists?).
+      with(profiling_filename).and_return(true)
+    flexmock(File).should_receive(:open).
+      with(profiling_filename, Proc).and_return(dumped_data)
+
     # mock out calls to the NeptuneManager
     flexmock(NeptuneManagerClient).new_instances { |instance|
-      # for this test, let's say there's no data on this task right now
-      instance.should_receive(:get_profiling_info).with(String).
-        and_return({})
-
       # let's say that all checks to see if temp files exist tell us that
       # the files don't exist
       instance.should_receive(:does_file_exist?).

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: neptune
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 0
   - 2
-  - 2
-  version: 0.2.2
+  - 3
+  version: 0.2.3
 platform: ruby
 authors: 
 - Chris Bunch
@@ -15,7 +15,7 @@ autorequire: neptune
 bindir: bin
 cert_chain: []
 
-date: 2012-07-31 00:00:00 Z
+date: 2012-09-28 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: promise