mobilize-ssh 1.0.7 → 1.0.9

data/README.md

@@ -83,8 +83,16 @@ fails in mid-copy. By default this is tmp/file/.
 * nodes, identified by aliases, such as `test_node`. This alias is what you should
 pass into the "node" param over in the ssh.run task.
 
-Each node has a host, and optionally has a gateway. If you don't need a
-gateway, remove that row from the configuration file.
+Each node has: 
+* a host;
+* a gateway (optional); If you don't need a gateway, remove that row from the configuration file.
+* sudoers; these are user names that are allowed to pass su_user params
+to the run call. This requires passwordless sudo for the host user.
+* su_all_users true/false option, which ensures that commands are executed by the
+user on the Runner. It prefixes all commands with sudo su <user_name> before executing the
+command. This is strongly recommended if possible as it ensures users do
+not overstep their permissions. This requires passwordless sudo for the
+host user and accounts on the host machine for each user.
 
 Each host and gateway has a series of ssh params:
 * name - the ip address or name of the host

data/lib/mobilize-ssh/handlers/ssh.rb

@@ -16,6 +16,14 @@ module Mobilize
       Ssh.config['nodes'][node]['gateway']
     end
 
+    def Ssh.sudoers(node)
+      Ssh.config['nodes'][node]['sudoers']
+    end
+
+    def Ssh.su_all_users(node)
+      Ssh.config['nodes'][node]['su_all_users']
+    end
+
     #determine if current machine is on host domain, needs gateway if one is provided and it is not
     def Ssh.needs_gateway?(node)
       host_domain_name = Ssh.host(node)['name'].split(".")[-2..-1].join(".")
@@ -34,9 +42,17 @@ module Mobilize
       return true if file_hash.keys.length>0
     end
 
+    def Ssh.set_key_permissions(key_path)
+      #makes sure permissions are set as appropriate for ssh key
+      "chmod a-rwx #{key_path}".bash
+      "chmod u+rw #{key_path}".bash
+      return true
+    end
+
     def Ssh.scp(node,from_path,to_path)
       name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
+      Ssh.set_key_permissions(key_path)
       opts = {:port=>(port || 22),:keys=>key_path}
       if Ssh.needs_gateway?(node)
         gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
@@ -52,9 +68,9 @@ module Mobilize
     end
 
     def Ssh.run(node,command,file_hash=nil,su_user=nil)
-      name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
+      key,user = Ssh.host(node).ie{|h| ['key','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
-      opts = {:port=>(port || 22),:keys=>key_path}
+      Ssh.set_key_permissions(key_path)
       su_user ||= user
       file_hash ||= {}
       #make sure the dir for this command is clear
@@ -64,48 +80,55 @@ module Mobilize
       Ssh.pop_comm_dir(comm_dir,file_hash)
       #move any files up to the node
       rem_dir = nil
+      #make sure user starts in rem_dir
+      rem_dir = "#{comm_md5}/"
+      #make sure the rem_dir is gone
+      Ssh.fire!(node,"rm -rf #{rem_dir}")
       if File.exists?(comm_dir)
-        #make sure user starts in rem_dir
-        rem_dir = "#{comm_md5}/"
-        command = ["cd #{rem_dir}",command].join(";")
-        #make sure the rem_dir is gone
-        Ssh.run(node,"rm -rf #{rem_dir}")
         Ssh.scp(node,comm_dir,rem_dir)
         "rm -rf #{comm_dir}".bash
-        if su_user
-          chown_command = "sudo chown -R #{su_user} #{rem_dir}"
-          Ssh.run(node,chown_command)
-        end
-      end
-      if su_user != user
-        #wrap the command in sudo su -c
-        command = %{sudo su #{su_user} -c "#{command}"}
+      else
+        #create folder
+        mkdir_command = "mkdir #{rem_dir}"
+        Ssh.fire!(node,mkdir_command)
       end
-      result = nil
-      #one with gateway, one without
+      #create cmd_file in rem_folder
+      cmd_file = "#{comm_md5}.sh"
+      cmd_path = "#{rem_dir}#{cmd_file}"
+      Ssh.write(node,command,cmd_path)
+      full_cmd = "(cd #{rem_dir} && sh #{cmd_file})"
+      #fire_cmd runs sh on cmd_path, optionally with sudo su
+      fire_cmd = if su_user != user
+                   %{sudo su #{su_user} -c "#{full_cmd}"}
+                 else
+                   full_cmd
+                 end
+      result = Ssh.fire!(node,fire_cmd)
+      #remove the directory after you're done
+      rm_cmd = "rm -rf #{rem_dir}"
+      Ssh.fire!(node,rm_cmd)
+      result
+    end
+
+    def Ssh.fire!(node,cmd)
+      name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
+      key_path = "#{Base.root}/#{key}"
+      Ssh.set_key_permissions(key_path)
+      opts = {:port=>(port || 22),:keys=>key_path}
       if Ssh.needs_gateway?(node)
-         gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
-         gkey_path = "#{Base.root}/#{gkey}"
-         gopts = {:port=>(gport || 22),:keys=>gkey_path}
-         result = Net::SSH::Gateway.run(gname,guser,name,user,command,gopts,opts)
+        gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
+        gkey_path = "#{Base.root}/#{gkey}"
+        gopts = {:port=>(gport || 22),:keys=>gkey_path}
+        Net::SSH::Gateway.run(gname,guser,name,user,cmd,gopts,opts)
       else
-         Net::SSH.start(name,user,opts) do |ssh|
-           result = ssh.run(command)
-         end
-      end
-      #delete remote dir if necessary
-      if rem_dir
-        del_cmd = "rm -rf #{rem_dir}"
-        if su_user
-          del_cmd = %{sudo su #{su_user} -c "#{del_cmd}"}
+        Net::SSH.start(name,user,opts) do |ssh|
+          ssh.run(cmd)
         end
-        Ssh.run(node,del_cmd)
       end
-      result
     end
 
     def Ssh.read(node,path)
-      Ssh.run(node,"cat #{path}")
+      Ssh.fire!(node,"cat #{path}")
     end
 
     def Ssh.write(node,fdata,to_path,binary=false)
@@ -153,11 +176,20 @@ module Mobilize
 
     def Ssh.run_by_stage_path(stage_path)
       s = Stage.where(:path=>stage_path).first
+      u = s.job.runner.user
       params = s.params
       node, command = [params['node'],params['cmd']]
       file_hash = Ssh.file_hash_by_stage_path(stage_path)
       su_user = s.params['su_user']
-      Ssh.run(node,command,file_hash,su_user)
+      if su_user and !Ssh.sudoers(node).include?(u.name)
+        raise "You do not have su permissions for this node"
+      elsif su_user.nil? and Ssh.su_all_users(node)
+        su_user = u.name
+      end
+      out_tsv = Ssh.run(node,command,file_hash,su_user)
+      #use Gridfs to cache result
+      out_url = "gridfs://#{s.path}/out"
+      Dataset.write_to_url(out_url,out_tsv)
     end
   end
 end

data/lib/mobilize-ssh/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Ssh
-    VERSION = "1.0.7"
+    VERSION = "1.0.9"
   end
 end

data/lib/samples/ssh.yml

@@ -2,17 +2,23 @@ development:
   tmp_file_dir: "tmp/file/"
   nodes:
     dev_node:
+      sudoers: [sudo_user]
+      su_all_users: true
       host: {name: dev-host.com, key: "config/mobilize/ssh_private.key", port: 22, user: host_user}
       gateway: {name: dev-gateway.com, key: "config/mobilize/ssh_private.key", port: 22, user: gateway_user}
 test:
   tmp_file_dir: "tmp/file/"
   nodes:
     test_node:
+      sudoers: [sudo_user]
+      su_all_users: true
       host: {name: test-host.com, key: "config/mobilize/ssh_private.key", port: 22, user: host_user}
       gateway: {name: test-gateway.com, key: "config/mobilize/ssh_private.key", port: 22, user: gateway_user}
 production:
   tmp_file_dir: "tmp/file/"
   nodes:
     prod_node:
+      sudoers: [sudo_user]
+      su_all_users: true
       host: {name: prod-host.com, key: "config/mobilize/ssh_private.key", port: 22, user: host_user}
       gateway: {name: prod-gateway.com, key: "config/mobilize/ssh_private.key", port: 22, user: gateway_user}

data/mobilize-ssh.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-base","1.0.8"
+  gem.add_runtime_dependency "mobilize-base","1.0.9"
   gem.add_runtime_dependency "net-ssh"
   gem.add_runtime_dependency "net-scp"
   gem.add_runtime_dependency "net-ssh-gateway"

data/test/mobilize-ssh_test.rb

@@ -43,9 +43,9 @@ describe "Mobilize" do
     ssh_job_rows.map{|j| r.jobs(j['name'])}.each{|j| j.delete if j}
     jobs_sheet.add_or_update_rows(ssh_job_rows)
 
-    puts "job row added, force enqueued runner, wait 120s"
+    puts "job row added, force enqueue runner, wait 150s"
     r.enqueue!
-    sleep 120
+    sleep 150
 
     puts "update job status and activity"
     r.update_gsheet(gdrive_slot)

data/test/ssh_job_rows.yml

@@ -6,13 +6,13 @@
   stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_1.out"'
 - name: test_ssh_2
   active: true
-  trigger: once
+  trigger: "after test_ssh_1"
   status: ""
   stage1: 'ssh.run node:"test_node", cmd:"sh code.sh", su_user:"root", source:"Runner_mobilize(test)/code.sh"'
   stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_2.out"'
 - name: test_ssh_3
   active: true
-  trigger: once
+  trigger: "after test_ssh_2"
   status: ""
   stage1: 'ssh.run node:"test_node", cmd:"whoami"'
   stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_3.out"'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-ssh
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-16 00:00:00.000000000 Z
+date: 2012-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-base
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.8
+        version: 1.0.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.8
+        version: 1.0.9
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement