mobilize-ssh 1.0.95 → 1.1.0

data/README.md

@@ -84,14 +84,15 @@ over to the nodes. They will be deleted afterwards, unless the job
 fails in mid-copy. By default this is tmp/file/.
 * nodes, identified by aliases, such as `test_node`. This alias is what you should
 pass into the "node" param over in the ssh.run task.
+* default_node is where commands will be executed if no node is specified.
 
 Each node has: 
 * a host;
 * a gateway (optional); If you don't need a gateway, remove that row from the configuration file.
-* sudoers; these are user names that are allowed to pass su_user params
+* sudoers; these are user names that are allowed to pass user params
 to the run call. This requires passwordless sudo for the host user.
 * su_all_users true/false option, which ensures that commands are executed by the
-user on the Runner. It prefixes all commands with sudo su <user_name> before executing the
+user on the Runner. It prefixes all commands with sudo su <user> before executing the
 command. This is strongly recommended if possible as it ensures users do
 not overstep their permissions. This requires passwordless sudo for the
 host user and accounts on the host machine for each user.
@@ -109,6 +110,7 @@ Sample ssh.yml:
 ---
 development:
   tmp_file_dir: tmp/file/
+  default_node: dev_node
   nodes:
     dev_node:
       sudoers: 
@@ -126,6 +128,7 @@ development:
         user: gateway_user
 test:
   tmp_file_dir: tmp/file/
+  default_node: test_node
   nodes:
     test_node:
       sudoers: 
@@ -143,6 +146,7 @@ test:
         user: gateway_user
 production:
   tmp_file_dir: tmp/file/
+  default_node: prod_node
   nodes:
     prod_node:
       sudoers: 
@@ -168,12 +172,13 @@ Start
 ### Create Job
 
 * For mobilize-ssh, the following task is available:
-  * ssh.run `node: <node_alias>, cmd: <command>, su_user: su_user, sources:[*<gsheet_full_paths>]`, which reads
+  * ssh.run `node: <node_alias>, cmd: <command>, user: user, sources:[*<gsheet_full_paths>]`, which reads
 all gsheets, copies them to a temporary folder on the selected node, and
 runs the command inside that folder. 
-  * su_user and sources are optional; node and cmd are required. su_user
-will cause the command to be prefixed with sudo su <su_user> -c.
-  * The test uses `ssh.run node:"test_node", cmd:"ruby code.rb", su_user: "root", sources:["Runner_mobilize(test)/code.rb","Runner_mobilize(test)/code.sh"]`
+  * user, sources, and node are optional; cmd is required. 
+  * specifying user will cause the command to be prefixed with sudo su <user> -c.
+  * not specifying node will cause the command to be run on the default_node
+  * The test uses `ssh.run node:"test_node", cmd:"ruby code.rb", user: "root", sources:["Runner_mobilize(test)/code.rb","Runner_mobilize(test)/code.sh"]`
 
 <a name='section_Start_Run_Test'></a>
 ### Run Test

data/lib/mobilize-ssh/handlers/ssh.rb

@@ -24,6 +24,10 @@ module Mobilize
       Ssh.config['nodes'][node]['su_all_users']
     end
 
+    def Ssh.default_node
+      Ssh.config['default_node']
+    end
+
     #determine if current machine is on host domain, needs gateway if one is provided and it is not
     def Ssh.needs_gateway?(node)
       host_domain_name = Ssh.host(node)['name'].split(".")[-2..-1].join(".")
@@ -67,14 +71,13 @@ module Mobilize
       return true
     end
 
-    def Ssh.run(node,command,file_hash=nil,su_user=nil)
-      key,user = Ssh.host(node).ie{|h| ['key','user'].map{|k| h[k]}}
+    def Ssh.run(node,command,user,file_hash={})
+      key,default_user = Ssh.host(node).ie{|h| ['key','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
       Ssh.set_key_permissions(key_path)
-      su_user ||= user
       file_hash ||= {}
       #make sure the dir for this command is clear
-      comm_md5 = [su_user,node,command,file_hash.keys.to_s].join.to_md5
+      comm_md5 = [user,node,command,file_hash.keys.to_s].join.to_md5
       comm_dir = "#{Ssh.tmp_file_dir}#{comm_md5}"
       #populate comm dir with any files
       Ssh.pop_comm_dir(comm_dir,file_hash)
@@ -98,8 +101,8 @@ module Mobilize
       Ssh.write(node,command,cmd_path)
       full_cmd = "(cd #{rem_dir} && sh #{cmd_file})"
       #fire_cmd runs sh on cmd_path, optionally with sudo su
-      fire_cmd = if su_user != user
-                   %{sudo su #{su_user} -c "#{full_cmd}"}
+      fire_cmd = if user != default_user
+                   %{sudo su #{user} -c "#{full_cmd}"}
                  else
                    full_cmd
                  end
@@ -149,47 +152,29 @@ module Mobilize
       return tmp_file_path
     end
 
-    def Ssh.file_hash_by_stage_path(stage_path)
-      #this is not meant to be called directly
-      #from the Runner -- it's used by run_by_stage_path
-      s = Stage.where(:path=>stage_path).first
-      params = s.params
-      gsheet_paths = if params['sources']
-                       params['sources']
-                     elsif params['source']
-                       [params['source']]
-                     end
-      if gsheet_paths and gsheet_paths.length>0
-        gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
-        file_hash = {}
-        gsheet_paths.map do |gpath|
-          string = Gsheet.find_by_path(gpath,gdrive_slot).to_tsv
-          fname = gpath.split("/").last
-          {fname => string}
-        end.each do |f|
-          file_hash = f.merge(file_hash)
-        end
-        Gdrive.unslot_worker_by_path(stage_path)
-        return file_hash
-      end
-    end
-
     def Ssh.run_by_stage_path(stage_path)
       s = Stage.where(:path=>stage_path).first
       u = s.job.runner.user
       params = s.params
       node, command = [params['node'],params['cmd']]
-      file_hash = Ssh.file_hash_by_stage_path(stage_path)
-      su_user = s.params['su_user']
-      if su_user and !Ssh.sudoers(node).include?(u.name)
-        raise "You do not have su permissions for this node"
-      elsif su_user.nil? and Ssh.su_all_users(node)
-        su_user = u.name
+      node ||= Ssh.default_node
+      gdrive_slot = Gdrive.slot_worker_by_path(s.path)
+      file_hash = {}
+      s.source_dsts(gdrive_slot).each do |sdst|
+                                      file_name = sdst.path.split("/").last
+                                      file_hash[file_name] = sdst.read(u.name)
+                                    end
+      Gdrive.unslot_worker_by_path(s.path)
+      user = s.params['user']
+      if user and !Ssh.sudoers(node).include?(u.name)
+        raise "#{u.name} does not have su permissions for this node"
+      elsif user.nil? and Ssh.su_all_users(node)
+        user = u.name
       end
-      out_tsv = Ssh.run(node,command,file_hash,su_user)
+      out_tsv = Ssh.run(node,command,user,file_hash)
       #use Gridfs to cache result
       out_url = "gridfs://#{s.path}/out"
-      Dataset.write_to_url(out_url,out_tsv)
+      Dataset.write_by_url(out_url,out_tsv,Gdrive.owner_name)
     end
   end
 end

data/lib/mobilize-ssh/tasks.rb

@@ -1,6 +1,7 @@
 namespace :mobilize_ssh do
   desc "Set up config and log folders and files"
   task :setup do
+    require 'yaml'
     sample_dir = File.dirname(__FILE__) + '/../samples/'
     sample_files = Dir.entries(sample_dir)
     config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")

data/lib/mobilize-ssh/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Ssh
-    VERSION = "1.0.95"
+    VERSION = "1.1.0"
   end
 end

data/lib/samples/ssh.yml

@@ -1,6 +1,7 @@
 ---
 development:
   tmp_file_dir: tmp/file/
+  default_node: dev_node
   nodes:
     dev_node:
       sudoers: 
@@ -18,6 +19,7 @@ development:
         user: gateway_user
 test:
   tmp_file_dir: tmp/file/
+  default_node: test_node
   nodes:
     test_node:
       sudoers: 
@@ -35,6 +37,7 @@ test:
         user: gateway_user
 production:
   tmp_file_dir: tmp/file/
+  default_node: prod_node
   nodes:
     prod_node:
       sudoers: 

data/mobilize-ssh.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-base","1.0.94"
+  gem.add_runtime_dependency "mobilize-base","1.1.0"
   gem.add_runtime_dependency "net-ssh"
   gem.add_runtime_dependency "net-scp"
   gem.add_runtime_dependency "net-ssh-gateway"

data/test/mobilize-ssh_test.rb

@@ -14,8 +14,8 @@ describe "Mobilize" do
 
     gdrive_slot = Mobilize::Gdrive.owner_email
     puts "create user 'mobilize'"
-    user_name = gdrive_slot.split("@").first
-    u = Mobilize::User.where(:name=>user_name).first
+    user = gdrive_slot.split("@").first
+    u = Mobilize::User.where(:name=>user).first
     r = u.runner
 
     rb_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
@@ -25,11 +25,11 @@ describe "Mobilize" do
     puts "add test code"
     rb_code_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
     rb_code_tsv = File.open("#{Mobilize::Base.root}/test/code.rb").read
-    rb_code_sheet.write(rb_code_tsv)
+    rb_code_sheet.write(rb_code_tsv,Mobilize::Gdrive.owner_name)
 
     sh_code_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/code.sh",gdrive_slot)
     sh_code_tsv = File.open("#{Mobilize::Base.root}/test/code.sh").read
-    sh_code_sheet.write(sh_code_tsv)
+    sh_code_sheet.write(sh_code_tsv,Mobilize::Gdrive.owner_name)
 
     jobs_sheet = r.gsheet(gdrive_slot)
 

data/test/ssh_job_rows.yml

@@ -2,17 +2,17 @@
   active: true
   trigger: once
   status: ""
-  stage1: 'ssh.run node:"test_node", cmd:"ruby code.rb", su_user:"root", sources:["Runner_mobilize(test)/code.rb", "Runner_mobilize(test)/code.sh"]'
+  stage1: 'ssh.run node:"test_node", cmd:"ruby code.rb", user:"root", sources:["Runner_mobilize(test)/code.rb", "Runner_mobilize(test)/code.sh"]'
   stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_1.out"'
 - name: test_ssh_2
   active: true
   trigger: "after test_ssh_1"
   status: ""
-  stage1: 'ssh.run node:"test_node", cmd:"sh code.sh", su_user:"root", source:"Runner_mobilize(test)/code.sh"'
+  stage1: 'ssh.run cmd:"sh code.sh", user:"root", source:"Runner_mobilize(test)/code.sh"'
   stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_2.out"'
 - name: test_ssh_3
   active: true
   trigger: "after test_ssh_2"
   status: ""
-  stage1: 'ssh.run node:"test_node", cmd:"whoami"'
+  stage1: 'ssh.run cmd:"whoami"'
   stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_3.out"'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-ssh
 version: !ruby/object:Gem::Version
-  version: 1.0.95
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-09 00:00:00.000000000 Z
+date: 2013-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-base
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.94
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.94
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement