mobilize-ssh 1.1.10 → 1.2

data/README.md

@@ -84,7 +84,7 @@ over to the nodes. They will be deleted afterwards, unless the job
 fails in mid-copy. By default this is tmp/file/.
 * nodes, identified by aliases, such as `test_node`. This alias is what you should
 pass into the "node" param over in the ssh.run task.
-* default_node is where commands will be executed if no node is specified.
+  * if no node is specified, commands will default to the first node listed.
 
 Each node has: 
 * a host;
@@ -110,7 +110,6 @@ Sample ssh.yml:
 ---
 development:
   tmp_file_dir: tmp/file/
-  default_node: dev_node
   nodes:
     dev_node:
       sudoers: 
@@ -128,7 +127,6 @@ development:
         user: gateway_user
 test:
   tmp_file_dir: tmp/file/
-  default_node: test_node
   nodes:
     test_node:
       sudoers: 
@@ -146,7 +144,6 @@ test:
         user: gateway_user
 production:
   tmp_file_dir: tmp/file/
-  default_node: prod_node
   nodes:
     prod_node:
       sudoers: 
@@ -172,13 +169,15 @@ Start
 ### Create Job
 
 * For mobilize-ssh, the following task is available:
-  * ssh.run `node: <node_alias>, cmd: <command>, user: user, sources:[*<gsheet_full_paths>]`, which reads
-all gsheets, copies them to a temporary folder on the selected node, and
-runs the command inside that folder. 
+  * ssh.run `node: <node_alias>, cmd: <command>, user: user, sources:[*<source_paths>]`, which reads sources, copies them to a temporary folder on the selected node, and runs the command inside that folder. 
   * user, sources, and node are optional; cmd is required. 
-  * specifying user will cause the command to be prefixed with sudo su <user> -c.
-  * not specifying node will cause the command to be run on the default_node
-  * The test uses `ssh.run node:"test_node", cmd:"ruby code.rb", user: "root", sources:["Runner_mobilize(test)/code.rb","Runner_mobilize(test)/code.sh"]`
+  * specifying user will cause the command to be prefixed with sudo su <user> -c. 
+    * non-google sources will also be read as the specified user.
+  * not specifying node will cause the command to be run on the default node.
+  * ssh sources can be specified with syntax
+`ssh://<node><file_full_path>`. If node is omitted, default node will be used.
+  * `<node><file_full_path>` and `<file_full_path>` can be used in the context of ssh.run, but if the path has only 1 slash, or none, it will try to find a google sheet or file instead.
+  * The test uses `ssh.run node:"test_node", cmd:"ruby code.rb", user: "root", sources:["code.rb","code.sh"]`
 
 <a name='section_Start_Run_Test'></a>
 ### Run Test

data/lib/mobilize-ssh/extensions/net-ssh-connection-session.rb

@@ -1,14 +1,35 @@
 class Net::SSH::Connection::Session
+  #from http://stackoverflow.com/questions/3386233/how-to-get-exit-status-with-rubys-netssh-library
   def run(command)
-    stdout,stderr = ["",""]
-    self.exec!(command) do |ch, stream, data|
-      if stream == :stderr
-        stderr += data
-      else
-        stdout += data
+    ssh = self
+    stdout_data = ""
+    stderr_data = ""
+    exit_code = nil
+    exit_signal = nil
+    ssh.open_channel do |channel|
+      channel.exec(command) do |chan, success|
+        unless success
+          abort "FAILED: couldn't execute command (ssh.channel.exec)"
+        end
+        channel.on_data do |ch,data|
+          stdout_data+=data
+        end
+
+        channel.on_extended_data do |ch,type,data|
+          stderr_data+=data
+        end
+
+        channel.on_request("exit-status") do |ch,data|
+          exit_code = data.read_long
+        end
+
+        channel.on_request("exit-signal") do |ch, data|
+          exit_signal = data.read_long
+        end
       end
     end
-    raise stderr if stderr.length>0
-    return stdout
+    ssh.loop
+    response = {'stdout'=>stdout_data, 'stderr'=>stderr_data, 'exit_code'=>exit_code, 'exit_signal'=>exit_signal}
+    response
   end
 end

data/lib/mobilize-ssh/extensions/net-ssh-gateway.rb

@@ -1,18 +1,40 @@
 class Net::SSH::Gateway
   def self.run(gname,guser,name,user,command,gopts={},opts={})
-    gateway = self.new(gname,guser,gopts)
+    gate = self
+    gateway = gate.new(gname,guser,gopts)
+    response = nil
     gateway.ssh(name,user,opts) do |ssh|
-      stderr,stdout = ["",""]
-      ssh.exec!(command) do |ch, stream, data|
-        if stream == :stderr
-          stderr += data
-        else
-          stdout += data
+      #from http://stackoverflow.com/questions/3386233/how-to-get-exit-status-with-rubys-netssh-library
+      stdout_data = ""
+      stderr_data = ""
+      exit_code = nil
+      exit_signal = nil
+      ssh.open_channel do |channel|
+        channel.exec(command) do |chan, success|
+          unless success
+            abort "FAILED: couldn't execute command (ssh.channel.exec)"
+          end
+          channel.on_data do |ch,data|
+            stdout_data+=data
+          end
+
+          channel.on_extended_data do |ch,type,data|
+            stderr_data+=data
+          end
+
+          channel.on_request("exit-status") do |ch,data|
+            exit_code = data.read_long
+          end
+
+          channel.on_request("exit-signal") do |ch, data|
+            exit_signal = data.read_long
+          end
         end
       end
-      raise stderr if stderr.length>0
-      return stdout
+      ssh.loop
+      response = {'stdout'=>stdout_data, 'stderr'=>stderr_data, 'exit_code'=>exit_code, 'exit_signal'=>exit_signal}
     end
+    response
   end
   def self.sync(gname,guser,name,user,from_path,to_path,gopts={},opts={})
     gateway = self.new(gname,guser,gopts)

data/lib/mobilize-ssh/handlers/ssh.rb

@@ -24,8 +24,12 @@ module Mobilize
       Ssh.config['nodes'][node]['su_all_users']
     end
 
+    def Ssh.nodes
+      Ssh.config['nodes'].keys
+    end
+
     def Ssh.default_node
-      Ssh.config['default_node']
+      Ssh.nodes.first
     end
 
     #determine if current machine is on host domain, needs gateway if one is provided and it is not
@@ -53,6 +57,43 @@ module Mobilize
       return true
     end
 
+    # converts a source path or target path to a dst in the context of handler and stage
+    def Ssh.path_to_dst(path,stage_path)
+      has_handler = true if path.index("://")
+      red_path = path.split("://").last
+      #is user has a handler, their first path node is a node name,
+      #or there are more than 2 path nodes, try to find Ssh file
+      if has_handler or Ssh.nodes.include?(red_path.split("/").first) or red_path.split("/").length > 2
+        user_name = Ssh.user_name_by_stage_path(stage_path)
+        ssh_url = Ssh.url_by_path(red_path,user_name)
+        return Dataset.find_or_create_by_url(ssh_url)
+      end
+      #otherwise, use Gsheet
+      return Gsheet.path_to_dst(red_path,stage_path)
+    end
+
+    def Ssh.url_by_path(path,user_name)
+      node = path.split("/").first.to_s
+      if Ssh.nodes.include?(node)
+        #cut node out of path
+        path = "/" + path.split("/")[1..-1].join("/")
+      else
+        node = Ssh.default_node
+        path = path.starts_with?("/") ? path : "/#{path}"
+      end
+      url = "ssh://#{node}#{path}"
+      begin
+        response = Ssh.run(node, "head -1 #{path}", user_name)
+        if response['exit_code'] != 0
+          raise "Unable to find #{url} with error: #{response['stderr']}"
+        else
+          return "ssh://#{node}#{path}"
+        end
+      rescue => exc
+        raise Exception, "Unable to find #{url} with error: #{exc.to_s}", exc.backtrace
+      end
+    end
+
     def Ssh.scp(node,from_path,to_path)
       name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
@@ -119,20 +160,29 @@ module Mobilize
       key_path = "#{Base.root}/#{key}"
       Ssh.set_key_permissions(key_path)
       opts = {:port=>(port || 22),:keys=>key_path}
-      if Ssh.needs_gateway?(node)
-        gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
-        gkey_path = "#{Base.root}/#{gkey}"
-        gopts = {:port=>(gport || 22),:keys=>gkey_path}
-        Net::SSH::Gateway.run(gname,guser,name,user,cmd,gopts,opts)
-      else
-        Net::SSH.start(name,user,opts) do |ssh|
-          ssh.run(cmd)
-        end
-      end
+      response = if Ssh.needs_gateway?(node)
+                   gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
+                   gkey_path = "#{Base.root}/#{gkey}"
+                   gopts = {:port=>(gport || 22),:keys=>gkey_path}
+                   Net::SSH::Gateway.run(gname,guser,name,user,cmd,gopts,opts)
+                 else
+                   Net::SSH.start(name,user,opts) do |ssh|
+                     ssh.run(cmd)
+                   end
+                 end
+      response
     end
 
-    def Ssh.read(node,path)
-      Ssh.fire!(node,"cat #{path}")
+    def Ssh.read_by_dataset_path(dst_path,user_name,*args)
+      #expects node as first part of path
+      node,path = dst_path.split("/").ie{|pa| [pa.first,pa[1..-1].join("/")]}
+      #slash in front of path
+      response = Ssh.run(node,"cat /#{path}",user_name)
+      if response['exit_code'] == 0
+        return response['stdout']
+      else
+        raise "Unable to read ssh://#{dst_path} with error: #{response['stderr']}"
+      end
     end
 
     def Ssh.write(node,fdata,to_path,binary=false)
@@ -154,37 +204,59 @@ module Mobilize
       return tmp_file_path
     end
 
-    def Ssh.run_by_stage_path(stage_path)
+    def Ssh.user_name_by_stage_path(stage_path,node=nil)
       s = Stage.where(:path=>stage_path).first
       u = s.job.runner.user
+      user_name = s.params['user']
+      node = s.params['node'] 
+      node = Ssh.default_node unless Ssh.nodes.include?(node)
+      if user_name and !Ssh.sudoers(node).include?(u.name)
+        raise "#{u.name} does not have su permissions for this node"
+      elsif user_name.nil? and Ssh.su_all_users(node)
+        user_name = u.name
+      end
+      return user_name
+    end
+
+    def Ssh.file_hash_by_stage_path(stage_path)
+      file_hash = {}
+      s = Stage.where(:path=>stage_path).first
+      u = s.job.runner.user
+      user_name = Ssh.user_name_by_stage_path(stage_path)
+      s.sources.each do |sdst|
+                       split_path = sdst.path.split("/")
+                       #if path is to stage output, name with stage name
+                       file_name = if split_path.last == "out" and
+                                     (1..5).to_a.map{|n| "stage#{n.to_s}"}.include?(split_path[-2].to_s)
+                                     "#{split_path[-2]}.out"
+                                   else
+                                     split_path.last
+                                   end
+                       if ["gsheet","gfile"].include?(sdst.handler)
+                         #google drive sources are always read as the user
+                         file_hash[file_name] = sdst.read(u.name)
+                       else
+                         #other sources should be read by su-user
+                         file_hash[file_name] = sdst.read(user_name)
+                       end
+                     end
+      return file_hash
+    end
+
+    def Ssh.run_by_stage_path(stage_path)
+      s = Stage.where(:path=>stage_path).first
       params = s.params
       node, command = [params['node'],params['cmd']]
       node ||= Ssh.default_node
-      gdrive_slot = Gdrive.slot_worker_by_path(s.path)
-      return false unless gdrive_slot
-      file_hash = {}
-      s.source_dsts(gdrive_slot).each do |sdst|
-                                        split_path = sdst.path.split("/")
-                                        #if path is to stage output, name with stage name
-                                        file_name = if split_path.last == "out" and
-                                                      (1..5).to_a.map{|n| "stage#{n.to_s}"}.include?(split_path[-2].to_s)
-                                                      "#{split_path[-2]}.out"
-                                                    else
-                                                      split_path.last
-                                                    end
-                                        file_hash[file_name] = sdst.read(u.name)
-                                      end
-      Gdrive.unslot_worker_by_path(s.path)
-      user = s.params['user']
-      if user and !Ssh.sudoers(node).include?(u.name)
-        raise "#{u.name} does not have su permissions for this node"
-      elsif user.nil? and Ssh.su_all_users(node)
-        user = u.name
-      end
-      out_tsv = Ssh.run(node,command,user,file_hash)
+      user_name = Ssh.user_name_by_stage_path(stage_path)
+      file_hash = Ssh.file_hash_by_stage_path(stage_path)
+      result = Ssh.run(node,command,user_name,file_hash)
       #use Gridfs to cache result
-      out_url = "gridfs://#{s.path}/out"
-      Dataset.write_by_url(out_url,out_tsv,Gdrive.owner_name)
+      response = {}
+      response['out_url'] = Dataset.write_by_url("gridfs://#{s.path}/out",result['stdout'].to_s,Gdrive.owner_name)
+      response['err_url'] = Dataset.write_by_url("gridfs://#{s.path}/err",result['stderr'].to_s,Gdrive.owner_name) if result['stderr'].to_s.length>0
+      response['signal'] = result['exit_code']
+      response
     end
   end
 end

data/lib/mobilize-ssh/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Ssh
-    VERSION = "1.1.10"
+    VERSION = "1.2"
   end
 end

data/lib/samples/ssh.yml

@@ -1,7 +1,6 @@
 ---
 development:
   tmp_file_dir: tmp/file/
-  default_node: dev_node
   nodes:
     dev_node:
       sudoers: 
@@ -19,7 +18,6 @@ development:
         user: gateway_user
 test:
   tmp_file_dir: tmp/file/
-  default_node: test_node
   nodes:
     test_node:
       sudoers: 
@@ -37,7 +35,6 @@ test:
         user: gateway_user
 production:
   tmp_file_dir: tmp/file/
-  default_node: prod_node
   nodes:
     prod_node:
       sudoers: 

data/mobilize-ssh.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.name          = "mobilize-ssh"
   gem.version       = Mobilize::Ssh::VERSION
   gem.authors       = ["Cassio Paes-Leme"]
-  gem.email         = ["cpaesleme@ngmoco.com"]
+  gem.email         = ["cpaesleme@dena.com"]
   gem.description   = %q{mobilize-ssh allows you to automate ssh commands and files across hosts}
   gem.summary       = %q{extend mobilize-base with the ability to run files across hosts}
   gem.homepage      = "http://github.com/dena/mobilize-ssh"
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-base","1.1.10"
+  gem.add_runtime_dependency "mobilize-base","1.2"
   gem.add_runtime_dependency "net-ssh"
   gem.add_runtime_dependency "net-scp"
   gem.add_runtime_dependency "net-ssh-gateway"

data/test/code2.sh

@@ -0,0 +1 @@
+tail syslog

data/test/mobilize-ssh_test.rb

@@ -20,6 +20,7 @@ describe "Mobilize" do
 
     rb_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
     sh_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.sh",gdrive_slot)
+    sh_code_sheet2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code2.sh",gdrive_slot)
     [rb_code_sheet,sh_code_sheet].each {|s| s.delete if s}
 
     puts "add test code"
@@ -31,6 +32,10 @@ describe "Mobilize" do
     sh_code_tsv = File.open("#{Mobilize::Base.root}/test/code.sh").read
     sh_code_sheet.write(sh_code_tsv,Mobilize::Gdrive.owner_name)
 
+    sh_code_sheet2 = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/code2.sh",gdrive_slot)
+    sh_code_tsv2 = File.open("#{Mobilize::Base.root}/test/code2.sh").read
+    sh_code_sheet2.write(sh_code_tsv2,Mobilize::Gdrive.owner_name)
+
     jobs_sheet = r.gsheet(gdrive_slot)
 
     #delete target sheets if they exist
@@ -43,9 +48,9 @@ describe "Mobilize" do
     ssh_job_rows.map{|j| r.jobs(j['name'])}.each{|j| j.delete if j}
     jobs_sheet.add_or_update_rows(ssh_job_rows)
 
-    puts "job row added, force enqueue runner, wait 300s"
+    puts "job row added, force enqueue runner, wait for stages"
     r.enqueue!
-    sleep 300
+    wait_for_stages(900)
 
     puts "update job status and activity"
     r.update_gsheet(gdrive_slot)
@@ -61,4 +66,27 @@ describe "Mobilize" do
 
   end
 
+  def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
+    time = 0
+    time_since_stage = 0
+    #check for 10 min
+    while time < time_limit and time_since_stage < stage_limit
+      sleep wait_length
+      job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
+      if job_classes.include?("Mobilize::Stage")
+        time_since_stage = 0
+        puts "saw stage at #{time.to_s} seconds"
+      else
+        time_since_stage += wait_length
+        puts "#{time_since_stage.to_s} seconds since stage seen"
+      end
+      time += wait_length
+      puts "total wait time #{time.to_s} seconds"
+    end
+
+    if time >= time_limit
+      raise "Timed out before stage completion"
+    end
+  end
+
 end

data/test/ssh_job_rows.yml

@@ -2,17 +2,17 @@
   active: true
   trigger: once
   status: ""
-  stage1: 'ssh.run node:"test_node", cmd:"ruby code.rb", user:"root", sources:["Runner_mobilize(test)/code.rb", "Runner_mobilize(test)/code.sh"]'
-  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_1.out"'
+  stage1: 'ssh.run node:"test_node", cmd:"ruby code.rb", user:"root", sources:["code.rb", "code.sh"]'
+  stage2: 'gsheet.write source:"stage1", target:"test_ssh_1.out"'
 - name: test_ssh_2
   active: true
   trigger: "after test_ssh_1"
   status: ""
-  stage1: 'ssh.run cmd:"sh code.sh", user:"root", source:"Runner_mobilize(test)/code.sh"'
-  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_2.out"'
+  stage1: 'ssh.run cmd:"sh code2.sh", user:"root", sources:["code2.sh","test_node/var/log/syslog"]'
+  stage2: 'gsheet.write source:"stage1", target:"test_ssh_2.out"'
 - name: test_ssh_3
   active: true
   trigger: "after test_ssh_2"
   status: ""
   stage1: 'ssh.run cmd:"whoami"'
-  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/test_ssh_3.out"'
+  stage2: 'gsheet.write source:"stage1", target:"test_ssh_3.out"'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-ssh
 version: !ruby/object:Gem::Version
-  version: 1.1.10
+  version: '1.2'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-05 00:00:00.000000000 Z
+date: 2013-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-base
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.10
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.10
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ dependencies:
         version: '0'
 description: mobilize-ssh allows you to automate ssh commands and files across hosts
 email:
-- cpaesleme@ngmoco.com
+- cpaesleme@dena.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -99,6 +99,7 @@ files:
 - mobilize-ssh.gemspec
 - test/code.rb
 - test/code.sh
+- test/code2.sh
 - test/mobilize-ssh_test.rb
 - test/redis-test.conf
 - test/ssh_job_rows.yml
@@ -130,6 +131,7 @@ summary: extend mobilize-base with the ability to run files across hosts
 test_files:
 - test/code.rb
 - test/code.sh
+- test/code2.sh
 - test/mobilize-ssh_test.rb
 - test/redis-test.conf
 - test/ssh_job_rows.yml