neptune 0.0.5 → 0.0.6

data/lib/app_controller_client.rb

@@ -29,6 +29,7 @@ class AppControllerClient
     
     @conn = SOAP::RPC::Driver.new("https://#{@ip}:17443")
     @conn.add_method("neptune_start_job", "job_data", "secret")
+    @conn.add_method("neptune_put_input", "job_data", "secret")
     @conn.add_method("neptune_get_output", "job_data", "secret")
     @conn.add_method("neptune_get_acl", "job_data", "secret")
     @conn.add_method("neptune_set_acl", "job_data", "secret")
@@ -87,12 +88,26 @@ class AppControllerClient
     return result
   end
 
+  # Stores a file stored on the user's local file system in the underlying
+  # database. The user can specify to use either the underlying database
+  # that AppScale is using, or alternative storage mechanisms (as of writing,
+  # Google Storage, Amazon S3, and Eucalyptus Walrus are supported) via the
+  # storage parameter.
+  def put_input(job_data)
+    result = ""
+    make_call(NO_TIMEOUT, false) { 
+      result = conn.neptune_put_input(job_data, @secret)
+    }  
+    abort(result) if result =~ /Error:/
+    return result
+  end
+
   # Retrieves the output of a Neptune job, stored in an underlying
   # database. Within AppScale, a special application runs, referred to as the
   # Repository, which provides a key-value interface to Neptune job data.
   # Data is stored as though it were on a file system, therefore output
   # be of the usual form /folder/filename . Currently the contents of the
-  # file is returned as a string to the caller, but as this is inefficient
+  # file is returned as a string to the caller, but as this may be inefficient
   # for non-trivial output jobs, the next version of Neptune will add an
   # additional call to directly copy the output to a file on the local
   # filesystem. See start_neptune_job for conditions by which this method

data/lib/neptune.rb

@@ -21,7 +21,11 @@ $VERBOSE = nil
 
 # A list of Neptune jobs that do not require nodes to be spawned
 # up for computation
-NO_NODES_NEEDED = ["acl", "output", "compile"]
+NO_NODES_NEEDED = ["acl", "input", "output", "compile"]
+
+# A list of Neptune jobs that do not require the output to be
+# specified beforehand
+NO_OUTPUT_NEEDED = ["input"]
 
 # A list of storage mechanisms that we can use to store and retrieve
 # data to for Neptune jobs.
@@ -184,17 +188,20 @@ def neptune(params)
 
   job_data["@job"] = nil
   job_data["@keyname"] = keyname || "appscale"
+  type = job_data["@type"]
 
   if job_data["@nodes_to_use"].class == Hash
     job_data["@nodes_to_use"] = job_data["@nodes_to_use"].to_a.flatten
   end
 
-  if (job_data["@output"].nil? or job_data["@output"] == "")
-    abort("Job output must be specified")
-  end
+  if !NO_OUTPUT_NEEDED.include?(type)
+    if (job_data["@output"].nil? or job_data["@output"] == "")
+      abort("Job output must be specified")
+    end
 
-  if job_data["@output"][0].chr != "/"
-    abort("Job output must begin with a slash ('/')")
+    if job_data["@output"][0].chr != "/"
+      abort("Job output must begin with a slash ('/')")
+    end
   end
 
   if job_data["@storage"]
@@ -250,8 +257,26 @@ def neptune(params)
 
   do_preprocessing(job_data) 
 
-  type = job_data["@type"]
-  if type == "output"
+  ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2>&1"
+
+  if type == "input"
+    # copy file to remote
+    # set location
+    local_file = File.expand_path(job_data["@local"])
+    if !File.exists?(local_file)
+      msg = "the file you specified to copy, #{local_file}, doesn't exist." + 
+        " Please specify a file that exists and try again."
+      abort(msg)
+    end
+
+    remote = "/tmp/neptune-input-#{rand(100000)}"
+    scp_cmd = "scp #{ssh_args} #{local_file} root@#{shadow_ip}:#{remote}"
+    puts scp_cmd
+    `#{scp_cmd}`
+
+    job_data["@local"] = remote
+    return controller.put_input(job_data)
+  elsif type == "output"
     return controller.get_output(job_data)
   elsif type == "get-acl"
     job_data["@type"] = "acl"
@@ -262,11 +287,10 @@ def neptune(params)
   elsif type == "compile"
     compiled_location = controller.compile_code(job_data)
 
-    ssh_args = "-i ~/.appscale/#{keyname}.key -o StrictHostkeyChecking=no 2>&1 root@#{shadow_ip}"
     copy_to = job_data["@copy_to"]
 
     loop {
-      ssh_command = "ssh #{ssh_args} 'ls #{compiled_location}'"
+      ssh_command = "ssh #{ssh_args} root@#{shadow_ip} 'ls #{compiled_location}'"
       #puts ssh_command
       result = `#{ssh_command}`
       #puts "result was [#{result}]"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: neptune
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: 
   segments: 
   - 0
   - 0
-  - 5
-  version: 0.0.5
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - Chris Bunch
@@ -15,7 +15,7 @@ autorequire: neptune
 bindir: bin
 cert_chain: []
 
-date: 2011-03-18 00:00:00 -07:00
+date: 2011-03-28 00:00:00 -07:00
 default_executable: neptune
 dependencies: []