mobilize-ssh 1.361 → 1.363

data/lib/mobilize-ssh/handlers/ssh.rb

@@ -2,15 +2,44 @@ module Mobilize
   module Ssh
     #adds convenience methods
     require "#{File.dirname(__FILE__)}/../helpers/ssh_helper"
-    def Ssh.pop_comm_dir(comm_dir,file_hash)
+    def Ssh.pop_loc_dir(unique_name,file_hash)
+      loc_dir = "/tmp/#{unique_name}"
+      `rm -rf #{loc_dir} && mkdir -p #{loc_dir}`
       file_hash.each do |fname,fdata|
-        fpath = "#{comm_dir}/#{fname}"
+        fpath = "#{loc_dir}/#{fname}"
         #for now, only gz is binary
-        binary = fname.ends_with?(".gz") ? true : false
-        #read data from cache, put it in a tmp_file
-        Ssh.tmp_file(fdata,binary,fpath)
+        mode = fname.ends_with?(".gz") ? "wb" : "w"
+        File.open(fpath,mode) {|f| f.print(fdata)}
       end
-      return true if file_hash.keys.length>0
+      return loc_dir if file_hash.keys.length>0
+    end
+
+    def Ssh.deploy(node,user_name,unique_name,command,file_hash)
+      loc_dir = Ssh.pop_loc_dir(unique_name,file_hash)
+      Ssh.fire!(node,"rm -rf #{unique_name} && mkdir -p #{unique_name} && chown -R #{Ssh.node_owner(node)} #{unique_name}")
+      if loc_dir
+        Ssh.scp(node,loc_dir,".")
+        #make sure loc_dir is removed
+        FileUtils.rm_r(loc_dir,:force=>true)
+      end
+      #create cmd_file in unique_name
+      cmd_path = "#{unique_name}/cmd.sh"
+      Ssh.write(node,command,cmd_path)
+      #move folder to user's home, change ownership
+      user_dir = "/home/#{user_name}/"
+      mobilize_dir = "#{user_dir}mobilize/"
+      deploy_dir = "#{mobilize_dir}#{unique_name}/"
+      deploy_cmd_path = "#{deploy_dir}cmd.sh"
+      deploy_cmd = "sudo mkdir -p #{mobilize_dir} && " +
+                   "sudo rm -rf  #{mobilize_dir}#{unique_name} && " +
+                   "sudo mv #{unique_name} #{mobilize_dir} && " +
+                   "sudo chown -R #{user_name} #{mobilize_dir}"
+      Ssh.fire!(node,deploy_cmd)
+      #need to use bash or we get no tee
+      full_cmd = "/bin/bash -l -c '(cd #{deploy_dir} && sh #{deploy_cmd_path} > >(tee stdout) 2> >(tee stderr >&2))'"
+      #fire_cmd runs sh on cmd_path, optionally with sudo su
+      fire_cmd = %{sudo su #{user_name} -c "#{full_cmd}"}
+      return fire_cmd
     end
 
     # converts a source path or target path to a dst in the context of handler and stage
@@ -67,13 +96,9 @@ module Mobilize
       return true
     end
 
-    def Ssh.run(node,command,user_name,file_hash={},run_params=nil)
-      default_user_name = Ssh.host(node)['user']
+    def Ssh.run(node,command,user_name,stage_path=nil,file_hash={},run_params=nil)
       file_hash ||= {}
       run_params ||={}
-      #make sure the dir for this command is clear
-      comm_md5 = [user_name,node,command,file_hash.keys.to_s,Time.now.to_f.to_s].join.to_md5
-      comm_dir = Dir.mktmpdir
       #replace any params in the file_hash and command
       run_params.each do |k,v|
         command.gsub!("@#{k}",v)
@@ -81,41 +106,25 @@ module Mobilize
           data.gsub!("@#{k}",v)
         end
       end
-     #populate comm dir with any files
-      Ssh.pop_comm_dir(comm_dir,file_hash)
-      #make sure user starts in rem_dir
-      rem_dir = "#{comm_md5}/"
-      #make sure the rem_dir is gone
-      Ssh.fire!(node,"sudo rm -rf #{rem_dir}")
-      if File.exists?(comm_dir)
-        Ssh.scp(node,comm_dir,rem_dir)
-        #make sure comm_dir is removed
-        FileUtils.rm_r(comm_dir,:force=>true)
-      else
-        #create folder
-        mkdir_command = "mkdir #{rem_dir}"
-        Ssh.fire!(node,mkdir_command)
-      end
-      #create cmd_file in rem_folder
-      cmd_file = "#{comm_md5}.sh"
-      cmd_path = "#{rem_dir}#{cmd_file}"
-      Ssh.write(node,command,cmd_path)
-      full_cmd = "(cd #{rem_dir} && sh #{cmd_file})"
-      #fire_cmd runs sh on cmd_path, optionally with sudo su
-      if user_name != default_user_name
-        #make sure user owns the folder and all files
-        fire_cmd = %{sudo chown -R #{user_name} #{rem_dir}; sudo su #{user_name} -c "#{full_cmd}"}
-        rm_cmd = %{sudo rm -rf #{rem_dir}}
-      else
-        fire_cmd = full_cmd
-        rm_cmd = "rm -rf #{rem_dir}"
-      end
+      #make sure the dir for this command is unique
+      unique_name = if stage_path
+                     stage_path.downcase.alphanunderscore
+                   else
+                     [user_name,node,command,file_hash.keys.to_s,Time.now.to_f.to_s].join.to_md5
+                   end
+      fire_cmd = Ssh.deploy(node, user_name, unique_name, command, file_hash)
       result = Ssh.fire!(node,fire_cmd)
-      Ssh.fire!(node,rm_cmd)
-      result
+      #clear out the md5 folders and those not requested to keep
+      s = Stage.find_by_path(stage_path) if stage_path
+      unless s and s.params['keep_logs']
+        rm_cmd = "sudo rm -rf /home/#{user_name}/mobilize/#{unique_name}"
+        Ssh.fire!(node,rm_cmd)
+      end
+      return result
     end
 
     def Ssh.fire!(node,cmd)
+      puts "#{Time.now.utc}--Ssh on #{node}: #{cmd}"
       name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
       opts = {:port=>(port || 22),:keys=>key_path}
@@ -169,7 +178,7 @@ module Mobilize
       node = Ssh.default_node unless Ssh.nodes.include?(node)
       if user_name and !Ssh.sudoers(node).include?(u.name)
         raise "#{u.name} does not have su permissions for this node"
-      elsif user_name.nil? and Ssh.su_all_users(node)
+      elsif user_name.nil?
         user_name = u.name
       end
       return user_name
@@ -213,10 +222,14 @@ module Mobilize
       node, command = [params['node'],params['cmd']]
       node ||= Ssh.default_node
       user_name = Ssh.user_name_by_stage_path(stage_path)
+      #do not allow server commands from non-sudoers for the special server node
+      if node=='server' and !Ssh.sudoers(node).include?(user_name)
+        raise "You do not have permission to run commands on the mobilize server"
+      end
       file_hash = Ssh.file_hash_by_stage_path(stage_path,gdrive_slot)
       Gdrive.unslot_worker_by_path(stage_path)
       run_params = params['params']
-      result = Ssh.run(node,command,user_name,file_hash,run_params)
+      result = Ssh.run(node,command,user_name,stage_path,file_hash,run_params)
       #use Gridfs to cache result
       response = {}
       response['out_url'] = Dataset.write_by_url("gridfs://#{s.path}/out",result['stdout'].to_s,Gdrive.owner_name)

data/lib/mobilize-ssh/helpers/ssh_helper.rb

@@ -16,10 +16,6 @@ module Mobilize
       self.config['nodes'][node]['sudoers']
     end
 
-    def self.su_all_users(node)
-      self.config['nodes'][node]['su_all_users']
-    end
-
     def self.nodes
       self.config['nodes'].keys
     end
@@ -28,8 +24,17 @@ module Mobilize
       self.nodes.first
     end
 
+    def self.node_owner(node)
+      self.host(node)['user']
+    end
+
+    def self.skip_gateway?(node)
+      self.config['nodes'][node]['skip_gateway']
+    end
+
     #determine if current machine is on host domain, needs gateway if one is provided and it is not
     def self.needs_gateway?(node)
+      return false if self.skip_gateway?(node)
       host_domain_name = self.host(node)['name'].split(".")[-2..-1].join(".")
       return true if self.gateway(node) and Socket.domain_name != host_domain_name
     end

data/lib/mobilize-ssh/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Ssh
-    VERSION = "1.361"
+    VERSION = "1.363"
   end
 end

data/lib/samples/ssh.yml

@@ -23,11 +23,13 @@ test:
       su_all_users: true
       host: 
         name: test-host.com 
+        skip_gateway: true
         key: config/mobilize/ssh_private.key 
         port: 22 
         user: host_user
       gateway: 
         name: test-gateway.com 
+        skip_gateway: true
         key: config/mobilize/ssh_private.key 
         port: 22 
         user: gateway_user

data/mobilize-ssh.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-base","1.361"
+  gem.add_runtime_dependency "mobilize-base","1.363"
   gem.add_runtime_dependency "net-ssh"
   gem.add_runtime_dependency "net-scp"
   gem.add_runtime_dependency "net-ssh-gateway"

data/test/fixtures/integration_jobs.yml

@@ -3,23 +3,23 @@
   active: true
   trigger: once
   status: ""
-  stage1: ssh.run node:"test_node", cmd:"ruby code.rb", user:"root", sources:["code.rb", "code.sh"]
+  stage1: ssh.run keep_logs:true, node:"test_node", cmd:"ruby code.rb", user:"root", sources:["code.rb", "code.sh"]
   stage2: gsheet.write source:"stage1", target:"ssh1.out"
 - name: ssh2
   active: true
   trigger: "after ssh1"
   status: ""
-  stage1: ssh.run cmd:"sh code2.sh", user:"root", sources:["code2.sh","test_node/var/log/syslog"], params:{file:"syslog"}
+  stage1: ssh.run keep_logs:true, cmd:"sh code2.sh", user:"root", sources:["code2.sh","test_node/var/log/syslog"], params:{file:"syslog"}
   stage2: gsheet.write source:"stage1", target:"ssh2.out"
 - name: ssh3
   active: true
   trigger: "after ssh2"
   status: ""
-  stage1: ssh.run cmd:"echo '@test_param'", params:{test_param:"test param successful"}
+  stage1: ssh.run keep_logs:true, cmd:"echo '@test_param'", params:{test_param:"test param successful"}
   stage2: gsheet.write source:"stage1", target:"ssh3.out"
 - name: ssh4
   active: true
   trigger: "after ssh3"
   status: ""
-  stage1: ssh.run node:"test_node", user:root, sources:["git://DeNA/mobilize-ssh/test/fixtures/code.rb","git://DeNA/mobilize-ssh/test/fixtures/code.sh"], cmd:"ruby code.rb"
+  stage1: ssh.run keep_logs:true, node:"test_node", user:root, sources:["git://DeNA/mobilize-ssh/test/fixtures/code.rb","git://DeNA/mobilize-ssh/test/fixtures/code.sh"], cmd:"ruby code.rb"
   stage2: gsheet.write source:stage1, target:"ssh4.out"

data/test/integration/mobilize-ssh_test.rb

@@ -18,7 +18,7 @@ describe "Mobilize" do
     end
 
     puts "add/update jobs"
-    u.jobs.each{|j| j.delete}
+    u.jobs.each{|j| j.stages.each{|s| s.delete}; j.delete}
     jobs_fixture_name = "integration_jobs"
     jobs_target_url = "gsheet://#{r.title}/jobs"
     TestHelper.write_fixture(jobs_fixture_name, jobs_target_url, 'update')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-ssh
 version: !ruby/object:Gem::Version
-  version: '1.361'
+  version: '1.363'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-31 00:00:00.000000000 Z
+date: 2013-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-base
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.361'
+        version: '1.363'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.361'
+        version: '1.363'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -121,7 +121,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3139478933470599366
+      hash: 4500700013252989723
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -130,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3139478933470599366
+      hash: 4500700013252989723
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25