neptune 0.0.9 → 0.1.0

data/lib/neptune.rb

@@ -33,7 +33,7 @@ ALLOWED_STORAGE_TYPES = ["appdb", "gstorage", "s3", "walrus"]
 
 # A list of jobs that require some kind of work to be done before
 # the actual computation can be performed.
-NEED_PREPROCESSING = ["compile", "erlang", "mpi"]
+NEED_PREPROCESSING = ["compile", "erlang", "mpi", "ssa"]
 
 # A set of methods and constants that we've monkey-patched to enable Neptune
 # support. In the future, it is likely that the only exposed / monkey-patched
@@ -58,6 +58,7 @@ end
 # Shadow node so that it can be compiled there. A future version
 # of this method may also copy over libraries as well.
 def preprocess_compile(job_data)
+  verbose = job_data["@verbose"]
   code = File.expand_path(job_data["@code"])
   unless File.exists?(code)
     abort("The source file #{code} does not exist.")
@@ -70,7 +71,7 @@ def preprocess_compile(job_data)
 
   ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no root@#{shadow_ip}"
   remove_dir = "ssh #{ssh_args} 'rm -rf #{dest}' 2>&1"
-  #puts remove_dir
+  puts remove_dir if verbose
   `#{remove_dir}`
 
   CommonFunctions.scp_to_shadow(code, dest, keyname, is_dir=true)
@@ -95,6 +96,7 @@ end
 # code to the master node in AppScale - this node will
 # then copy it to whoever will run the MPI job.
 def preprocess_mpi(job_data)
+  verbose = job_data["@verbose"]
   if job_data["@procs_to_use"]
     p = job_data["@procs_to_use"]
     n = job_data["@nodes_to_use"]
@@ -123,9 +125,19 @@ def preprocess_mpi(job_data)
   dest_code = "/tmp/thempicode"
 
   keyname = job_data["@keyname"]
-  puts "Copying over code..."
+  puts "Copying over code..." if verbose
   CommonFunctions.scp_to_shadow(source_code, dest_code, keyname)
-  puts "Done copying code!"
+  puts "Done copying code!" if verbose
+end
+
+def preprocess_ssa(job_data)
+  if job_data["@simulations"]
+    job_data["@trajectories"] = job_data["@simulations"]
+  end
+
+  unless job_data["@trajectories"]
+    abort(":trajectories needs to be specified when running ssa jobs")
+  end
 end
 
 # TODO: actually use me!
@@ -145,8 +157,10 @@ end
 # job can be used to set it to public later (and
 # vice-versa).
 def neptune(params)
-  puts "Received a request to run a job."
-  puts params[:type]
+  verbose = params[:verbose]
+
+  puts "Received a request to run a job." if verbose
+  puts params[:type] if verbose
 
   keyname = params[:keyname] || "appscale"
 
@@ -206,7 +220,7 @@ def neptune(params)
       ["EC2_ACCESS_KEY", "EC2_SECRET_KEY", "S3_URL"].each { |item|
         unless job_data["@#{item}"]
           if ENV[item]
-            puts "Using #{item} from environment"
+            puts "Using #{item} from environment" if verbose
             job_data["@#{item}"] = ENV[item]
           else
             msg = "When storing data to S3, #{item} must be specified or be in " + 
@@ -226,7 +240,7 @@ def neptune(params)
   #  job_data["@can_run_on"] = [job_data["@can_run_on"]]
   #end
 
-  puts "job data = #{job_data.inspect}"
+  puts "job data = #{job_data.inspect}" if verbose
 
   do_preprocessing(job_data) 
 
@@ -248,11 +262,11 @@ def neptune(params)
 
     remote = "/tmp/neptune-input-#{rand(100000)}"
     scp_cmd = "scp #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
-    puts scp_cmd
+    puts scp_cmd if verbose
     `#{scp_cmd}`
 
     job_data["@local"] = remote
-    puts "job data = #{job_data.inspect}"
+    puts "job data = #{job_data.inspect}" if verbose
     result[:input] = controller.put_input(job_data)
   elsif type == "output"
     result[:output] = controller.get_output(job_data)
@@ -269,24 +283,24 @@ def neptune(params)
 
     loop {
       ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}' 2>&1"
-      #puts ssh_command
+      puts ssh_command if verbose
       result = `#{ssh_command}`
-      #puts "result was [#{result}]"
+      puts "result was [#{result}]" if verbose
       if result =~ /No such file or directory/
-        puts "Still waiting for code to be compiled..."
+        puts "Still waiting for code to be compiled..." if verbose
       else
-        puts "compilation complete! Copying compiled code to #{copy_to}"
+        puts "compilation complete! Copying compiled code to #{copy_to}" if verbose
         break
       end
       sleep(5)
     }
 
     rm_local = "rm -rf #{copy_to}"
-    #puts rm_local
+    puts rm_local if verbose
     `#{rm_local}`
 
     scp_command = "scp -r #{ssh_args} root@#{shadow_ip}:#{compiled_location} #{copy_to} 2>&1"
-    puts scp_command
+    puts scp_command if verbose
     `#{scp_command}`
 
     code = job_data["@code"]
@@ -294,19 +308,24 @@ def neptune(params)
     remote_dir = "/tmp/" + dirs[-1] 
 
     ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{remote_dir}' 2>&1"
-    puts ssh_command
+    puts ssh_command if verbose
     `#{ssh_command}`
 
     ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'rm -rf #{compiled_location}' 2>&1"
-    puts ssh_command
+    puts ssh_command if verbose
     `#{ssh_command}`
 
     out = File.open("#{copy_to}/compile_out") { |f| f.read.chomp! }
     err = File.open("#{copy_to}/compile_err") { |f| f.read.chomp! }
+    result = {}
     result[:out] = out
     result[:err] = err
-    
-    result[:result] = :failure if result[:err].any?
+
+    if result[:err]
+      result[:result] = :failure
+    else
+      result[:result] = :success
+    end    
   else
     msg = controller.start_neptune_job(job_data)
     result[:msg] = msg

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: neptune
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 27
   prerelease: 
   segments: 
   - 0
+  - 1
   - 0
-  - 9
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors: 
 - Chris Bunch
@@ -15,7 +15,7 @@ autorequire: neptune
 bindir: bin
 cert_chain: []
 
-date: 2011-05-28 00:00:00 -07:00
+date: 2011-06-04 00:00:00 -07:00
 default_executable: neptune
 dependencies: []