mobilize-ssh 1.0.9 → 1.0.21

data/README.md

@@ -81,18 +81,10 @@ The Ssh configuration consists of:
 over to the nodes. They will be deleted afterwards, unless the job
 fails in mid-copy. By default this is tmp/file/.
 * nodes, identified by aliases, such as `test_node`. This alias is what you should
-pass into the "node" param over in the ssh.run task.
-
-Each node has: 
-* a host;
-* a gateway (optional); If you don't need a gateway, remove that row from the configuration file.
-* sudoers; these are user names that are allowed to pass su_user params
-to the run call. This requires passwordless sudo for the host user.
-* su_all_users true/false option, which ensures that commands are executed by the
-user on the Runner. It prefixes all commands with sudo su <user_name> before executing the
-command. This is strongly recommended if possible as it ensures users do
-not overstep their permissions. This requires passwordless sudo for the
-host user and accounts on the host machine for each user.
+pass into the "node_alias" param over in the ssh.run task.
+
+Each node has a host, and optionally has a gateway. If you don't need a
+gateway, remove that row from the configuration file.
 
 Each host and gateway has a series of ssh params:
 * name - the ip address or name of the host
@@ -133,12 +125,10 @@ Start
 ### Create Job
 
 * For mobilize-ssh, the following task is available:
-  * ssh.run `node: <node_alias>, cmd: <command>, su_user: su_user, sources:[*<gsheet_full_paths>]`, which reads
+  * ssh.run `<node_alias>,<command>,*<gsheet_full_paths>`, which reads
 all gsheets, copies them to a temporary folder on the selected node, and
 runs the command inside that folder. 
-  * su_user and sources are optional; node and cmd are required. su_user
-will cause the command to be prefixed with sudo su <su_user> -c.
-  * The test uses `ssh.run node:"test_node", cmd:"ruby code.rb", su_user: "root", sources:["Runner_mobilize(test)/code.rb","Runner_mobilize(test)/code.sh"]`
+  * The test uses `ssh.run "test_node", "ruby code.rb", "Runner_mobilize(test)/code.rb", "Runner_mobilize(test)/code.sh"`
 
 <a name='section_Start_Run_Test'></a>
 ### Run Test

data/Rakefile

@@ -1,7 +1,7 @@
 require "bundler/gem_tasks"
 
-require 'mobilize-base/tasks'
-require 'mobilize-ssh/tasks'
+require 'mobilize-base/rakes'
+require 'mobilize-ssh/rakes'
 
 #
 # Tests

data/lib/mobilize-ssh/handlers/ssh.rb

@@ -16,14 +16,6 @@ module Mobilize
       Ssh.config['nodes'][node]['gateway']
     end
 
-    def Ssh.sudoers(node)
-      Ssh.config['nodes'][node]['sudoers']
-    end
-
-    def Ssh.su_all_users(node)
-      Ssh.config['nodes'][node]['su_all_users']
-    end
-
     #determine if current machine is on host domain, needs gateway if one is provided and it is not
     def Ssh.needs_gateway?(node)
       host_domain_name = Ssh.host(node)['name'].split(".")[-2..-1].join(".")
@@ -42,17 +34,9 @@ module Mobilize
       return true if file_hash.keys.length>0
     end
 
-    def Ssh.set_key_permissions(key_path)
-      #makes sure permissions are set as appropriate for ssh key
-      "chmod a-rwx #{key_path}".bash
-      "chmod u+rw #{key_path}".bash
-      return true
-    end
-
     def Ssh.scp(node,from_path,to_path)
       name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
-      Ssh.set_key_permissions(key_path)
       opts = {:port=>(port || 22),:keys=>key_path}
       if Ssh.needs_gateway?(node)
         gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
@@ -68,9 +52,9 @@ module Mobilize
     end
 
     def Ssh.run(node,command,file_hash=nil,su_user=nil)
-      key,user = Ssh.host(node).ie{|h| ['key','user'].map{|k| h[k]}}
+      name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
-      Ssh.set_key_permissions(key_path)
+      opts = {:port=>(port || 22),:keys=>key_path}
       su_user ||= user
       file_hash ||= {}
       #make sure the dir for this command is clear
@@ -80,55 +64,46 @@ module Mobilize
       Ssh.pop_comm_dir(comm_dir,file_hash)
       #move any files up to the node
       rem_dir = nil
-      #make sure user starts in rem_dir
-      rem_dir = "#{comm_md5}/"
-      #make sure the rem_dir is gone
-      Ssh.fire!(node,"rm -rf #{rem_dir}")
       if File.exists?(comm_dir)
+        #make sure user starts in rem_dir
+        rem_dir = "#{comm_md5}/"
+        command = ["cd #{rem_dir}",command].join(";")
         Ssh.scp(node,comm_dir,rem_dir)
         "rm -rf #{comm_dir}".bash
-      else
-        #create folder
-        mkdir_command = "mkdir #{rem_dir}"
-        Ssh.fire!(node,mkdir_command)
+        if su_user
+          chown_command = "sudo chown -R #{su_user} #{rem_dir}"
+          Ssh.run(node,chown_command)
+        end
       end
-      #create cmd_file in rem_folder
-      cmd_file = "#{comm_md5}.sh"
-      cmd_path = "#{rem_dir}#{cmd_file}"
-      Ssh.write(node,command,cmd_path)
-      full_cmd = "(cd #{rem_dir} && sh #{cmd_file})"
-      #fire_cmd runs sh on cmd_path, optionally with sudo su
-      fire_cmd = if su_user != user
-                   %{sudo su #{su_user} -c "#{full_cmd}"}
-                 else
-                   full_cmd
-                 end
-      result = Ssh.fire!(node,fire_cmd)
-      #remove the directory after you're done
-      rm_cmd = "rm -rf #{rem_dir}"
-      Ssh.fire!(node,rm_cmd)
-      result
-    end
-
-    def Ssh.fire!(node,cmd)
-      name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
-      key_path = "#{Base.root}/#{key}"
-      Ssh.set_key_permissions(key_path)
-      opts = {:port=>(port || 22),:keys=>key_path}
+      if su_user != user
+        #wrap the command in sudo su -c
+        command = %{sudo su #{su_user} -c "#{command}"}
+      end
+      result = nil
+      #one with gateway, one without
       if Ssh.needs_gateway?(node)
-        gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
-        gkey_path = "#{Base.root}/#{gkey}"
-        gopts = {:port=>(gport || 22),:keys=>gkey_path}
-        Net::SSH::Gateway.run(gname,guser,name,user,cmd,gopts,opts)
+         gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
+         gkey_path = "#{Base.root}/#{gkey}"
+         gopts = {:port=>(gport || 22),:keys=>gkey_path}
+         result = Net::SSH::Gateway.run(gname,guser,name,user,command,gopts,opts)
       else
-        Net::SSH.start(name,user,opts) do |ssh|
-          ssh.run(cmd)
+         Net::SSH.start(name,user,opts) do |ssh|
+           result = ssh.run(command)
+         end
+      end
+      #delete remote dir if necessary
+      if rem_dir
+        del_cmd = "rm -rf #{rem_dir}"
+        if su_user
+          del_cmd = %{sudo su #{su_user} -c "#{del_cmd}"}
         end
+        Ssh.run(node,del_cmd)
       end
+      result
     end
 
     def Ssh.read(node,path)
-      Ssh.fire!(node,"cat #{path}")
+      Ssh.run(node,"cat #{path}")
     end
 
     def Ssh.write(node,fdata,to_path,binary=false)
@@ -149,47 +124,28 @@ module Mobilize
       return tmp_file_path
     end
 
-    def Ssh.file_hash_by_stage_path(stage_path)
-      #this is not meant to be called directly
-      #from the Runner -- it's used by run_by_stage_path
-      s = Stage.where(:path=>stage_path).first
-      params = s.params
-      gsheet_paths = if params['sources']
-                       params['sources']
-                     elsif params['source']
-                       [params['source']]
-                     end
-      if gsheet_paths and gsheet_paths.length>0
-        gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
-        file_hash = {}
-        gsheet_paths.map do |gpath|
-          string = Gsheet.find_by_path(gpath,gdrive_slot).to_tsv
-          fname = gpath.split("/").last
-          {fname => string}
-        end.each do |f|
-          file_hash = f.merge(file_hash)
-        end
-        Gdrive.unslot_worker_by_path(stage_path)
-        return file_hash
-      end
-    end
-
-    def Ssh.run_by_stage_path(stage_path)
-      s = Stage.where(:path=>stage_path).first
-      u = s.job.runner.user
-      params = s.params
-      node, command = [params['node'],params['cmd']]
-      file_hash = Ssh.file_hash_by_stage_path(stage_path)
-      su_user = s.params['su_user']
-      if su_user and !Ssh.sudoers(node).include?(u.name)
-        raise "You do not have su permissions for this node"
-      elsif su_user.nil? and Ssh.su_all_users(node)
-        su_user = u.name
+    def Ssh.get_file_hash(gsheet_paths,gdrive_slot)
+      file_hash = {}
+      gsheet_paths.map do |gpath|
+        string = Gsheet.find_by_path(gpath,gdrive_slot).to_tsv
+        fname = gpath.split("/").last
+        {fname => string}
+      end.each do |f|
+        file_hash = f.merge(file_hash)
       end
-      out_tsv = Ssh.run(node,command,file_hash,su_user)
-      #use Gridfs to cache result
-      out_url = "gridfs://#{s.path}/out"
-      Dataset.write_to_url(out_url,out_tsv)
+      file_hash
+    end
+
+    def Ssh.run_by_task_path(task_path)
+      t = Task.where(:path=>task_path).first
+      params = t.params
+      node, command = [params[0],params[1]]
+      file_hash = if params[2]
+                    gsheet_paths = params[2..-1] if params[2]
+                    gdrive_slot = Gdrive.slot_worker_by_path(task_path)
+                    Ssh.get_file_hash(gsheet_paths,gdrive_slot)
+                  end
+      Ssh.run(node,command,file_hash)
     end
   end
 end

data/lib/mobilize-ssh/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Ssh
-    VERSION = "1.0.9"
+    VERSION = "1.0.21"
   end
 end

data/lib/samples/ssh.yml

@@ -2,23 +2,17 @@ development:
   tmp_file_dir: "tmp/file/"
   nodes:
     dev_node:
-      sudoers: [sudo_user]
-      su_all_users: true
       host: {name: dev-host.com, key: "config/mobilize/ssh_private.key", port: 22, user: host_user}
       gateway: {name: dev-gateway.com, key: "config/mobilize/ssh_private.key", port: 22, user: gateway_user}
 test:
   tmp_file_dir: "tmp/file/"
   nodes:
     test_node:
-      sudoers: [sudo_user]
-      su_all_users: true
       host: {name: test-host.com, key: "config/mobilize/ssh_private.key", port: 22, user: host_user}
       gateway: {name: test-gateway.com, key: "config/mobilize/ssh_private.key", port: 22, user: gateway_user}
 production:
   tmp_file_dir: "tmp/file/"
   nodes:
     prod_node:
-      sudoers: [sudo_user]
-      su_all_users: true
       host: {name: prod-host.com, key: "config/mobilize/ssh_private.key", port: 22, user: host_user}
       gateway: {name: prod-gateway.com, key: "config/mobilize/ssh_private.key", port: 22, user: gateway_user}

data/mobilize-ssh.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-base","1.0.9"
+  gem.add_runtime_dependency "mobilize-base","1.0.51"
   gem.add_runtime_dependency "net-ssh"
   gem.add_runtime_dependency "net-scp"
   gem.add_runtime_dependency "net-ssh-gateway"

data/test/mobilize-ssh_test.rb

@@ -18,10 +18,6 @@ describe "Mobilize" do
     u = Mobilize::User.where(:name=>user_name).first
     r = u.runner
 
-    rb_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
-    sh_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.sh",gdrive_slot)
-    [rb_code_sheet,sh_code_sheet].each {|s| s.delete if s}
-
     puts "add test code"
     rb_code_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
     rb_code_tsv = File.open("#{Mobilize::Base.root}/test/code.rb").read
@@ -33,32 +29,20 @@ describe "Mobilize" do
 
     jobs_sheet = r.gsheet(gdrive_slot)
 
-    #delete target sheets if they exist
-    ssh_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_1.out",gdrive_slot)
-    ssh_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_2.out",gdrive_slot)
-    ssh_target_sheet_3 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_3.out",gdrive_slot)
-    [ssh_target_sheet_1,ssh_target_sheet_2,ssh_target_sheet_3].each {|s| s.delete if s}
-
     ssh_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/ssh_job_rows.yml")
-    ssh_job_rows.map{|j| r.jobs(j['name'])}.each{|j| j.delete if j}
     jobs_sheet.add_or_update_rows(ssh_job_rows)
 
-    puts "job row added, force enqueue runner, wait 150s"
+    puts "job row added, force enqueued runner, wait 90s"
     r.enqueue!
-    sleep 150
+    sleep 90
 
     puts "update job status and activity"
     r.update_gsheet(gdrive_slot)
 
     puts "jobtracker posted data to test sheet"
-    ssh_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_1.out",gdrive_slot)
-    ssh_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_2.out",gdrive_slot)
-    ssh_target_sheet_3 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_3.out",gdrive_slot)
-
-    assert ssh_target_sheet_1.to_tsv.length > 100
-    assert ssh_target_sheet_2.to_tsv.length > 100
-    assert ssh_target_sheet_3.to_tsv.length > 3
+    ssh_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh.out",gdrive_slot)
 
+    assert ssh_target_sheet.to_tsv.length > 100
   end
 
 end

data/test/ssh_job_rows.yml

@@ -1,18 +1,6 @@
-- name: test_ssh_1
+- name: test_ssh
   active: true
   trigger: once
   status: ""
-  stage1: 'ssh.run node:"test_node", cmd:"ruby code.rb", su_user:"root", sources:["Runner_mobilize(test)/code.rb", "Runner_mobilize(test)/code.sh"]'
-  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_1.out"'
-- name: test_ssh_2
-  active: true
-  trigger: "after test_ssh_1"
-  status: ""
-  stage1: 'ssh.run node:"test_node", cmd:"sh code.sh", su_user:"root", source:"Runner_mobilize(test)/code.sh"'
-  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_2.out"'
-- name: test_ssh_3
-  active: true
-  trigger: "after test_ssh_2"
-  status: ""
-  stage1: 'ssh.run node:"test_node", cmd:"whoami"'
-  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_3.out"'
+  task1: 'ssh.run "test_node", "ruby code.rb", "Runner_mobilize(test)/code.rb", "Runner_mobilize(test)/code.sh"'
+  task2: 'gsheet.write "task1", "Runner_mobilize(test)/test_ssh.out"'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-ssh
 version: !ruby/object:Gem::Version
-  version: 1.0.9
+  version: 1.0.21
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-22 00:00:00.000000000 Z
+date: 2012-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-base
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.9
+        version: 1.0.51
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.9
+        version: 1.0.51
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -93,7 +93,7 @@ files:
 - lib/mobilize-ssh/extensions/socket.rb
 - lib/mobilize-ssh/extensions/string.rb
 - lib/mobilize-ssh/handlers/ssh.rb
-- lib/mobilize-ssh/tasks.rb
+- lib/mobilize-ssh/rakes.rb
 - lib/mobilize-ssh/version.rb
 - lib/samples/ssh.yml
 - mobilize-ssh.gemspec
@@ -115,12 +115,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 871259298100733038
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 871259298100733038
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -134,4 +140,3 @@ test_files:
 - test/redis-test.conf
 - test/ssh_job_rows.yml
 - test/test_helper.rb
-has_rdoc: