mobilize-ssh 1.297 → 1.298

data/README.md

@@ -13,6 +13,7 @@ Table Of Contents
   * [Install Dirs and Files](#section_Install_Dirs_and_Files)
 * [Configure](#section_Configure)
   * [Ssh](#section_Configure_Ssh)
+  * [Git](#section_Configure_Git)
 * [Start](#section_Start)
   * [Create Job](#section_Start_Create_Job)
   * [Run Test](#section_Start_Run_Test)
@@ -79,9 +80,6 @@ Configure
 ### Configure Ssh
 
 The Ssh configuration consists of:
-* tmp_file_dir, which is where files will be stored before being scp'd
-over to the nodes. They will be deleted afterwards, unless the job
-fails in mid-copy. By default this is tmp/file/.
 * nodes, identified by aliases, such as `test_node`. This alias is what you should
 pass into the "node" param over in the ssh.run task.
   * if no node is specified, commands will default to the first node listed.
@@ -109,7 +107,6 @@ Sample ssh.yml:
 ``` yml
 ---
 development:
-  tmp_file_dir: tmp/file/
   nodes:
     dev_node:
       sudoers: 
@@ -126,7 +123,6 @@ development:
         port: 22 
         user: gateway_user
 test:
-  tmp_file_dir: tmp/file/
   nodes:
     test_node:
       sudoers: 
@@ -143,7 +139,6 @@ test:
         port: 22 
         user: gateway_user
 production:
-  tmp_file_dir: tmp/file/
   nodes:
     prod_node:
       sudoers: 
@@ -161,6 +156,57 @@ production:
         user: gateway_user
 ```
 
+<a name='section_Configure_Git'></a>
+### Configure Git
+
+Git configuration is not required but recommended, as it allows you to
+pull files directly from public or private Git repositories.
+
+The Git configuration consists of:
+* domains, identified by aliases, such as `private` and `public`.
+  * domains are passed into the source parameters in the git.run task.
+  * if no domain is specified, commands will default to the first domain listed.
+
+Each domain has: 
+* a host;
+* a key (optional); If you don't need an ssh key to access the repo, remove that row from the configuration file.
+  * this is the relative path of the ssh key required to access the repository.
+* a user, which is the user used for the git clone command.
+
+Sample git.yml:
+
+``` yml
+---
+development:
+  domains:
+    private:
+      host: github.<domain>.com
+      key: config/mobilize/ssh_private.key
+      user: git
+    public:
+      host: github.com
+      user: git
+test:
+  domains:
+    private:
+      host: github.<domain>.com
+      key: config/mobilize/ssh_private.key
+      user: git
+    public:
+      host: github.com
+      user: git
+production:
+  domains:
+    private:
+      host: github.<domain>.com
+      key: config/mobilize/ssh_private.key
+      user: git
+    public:
+      host: github.com
+      user: git
+```
+
+
 <a name='section_Start'></a>
 Start
 -----
@@ -173,6 +219,10 @@ Start
   * user, sources, and node are optional; cmd is required. 
   * specifying user will cause the command to be prefixed with sudo su <user> -c. 
     * non-google sources will also be read as the specified user.
+  * git sources can be specified with syntax `git://<domain>/<repo_owner>/<repo_name>/<file_path>`. 
+    * Accessing private repos requires that you add the Mobilize public key to the repository as a deploy key.
+      * there is no user-level access control for git repositories at this time.
+    * domain defaults to the first one listed, if not included.
   * not specifying node will cause the command to be run on the default node.
   * ssh sources can be specified with syntax
 `ssh://<node><file_full_path>`. If node is omitted, default node will be used.

data/lib/mobilize-ssh/handlers/git.rb

@@ -0,0 +1,111 @@
+module Mobilize
+  module Git
+    def Git.config
+      Base.config('git')
+    end
+
+    def Git.host(domain)
+      Git.config['domains'][domain]['host']
+    end
+
+    def Git.domains
+      Git.config['domains'].keys
+    end
+
+    def Git.default_domain
+      Git.domains.first
+    end
+
+    # converts a source path or target path to a dst in the context of handler and stage
+    def Git.path_to_dst(path,stage_path,gdrive_slot)
+      red_path = path.split("://").last
+      git_url = Git.url_by_path(red_path)
+      return Dataset.find_or_create_by_url(git_url)
+    end
+
+    def Git.url_by_path(path)
+      path_nodes = path.split("/")
+      domain = path_nodes.first.to_s
+      revision = "HEAD"
+      if Git.domains.include?(domain)
+        repo = path_nodes[1..2].join("/")
+        file_path = path_nodes[3..-1].join("/")
+      else
+        domain = Git.default_domain
+        repo = path_nodes[0..1].join("/")
+        file_path = path_nodes[2..-1].join("/")
+      end
+      url = "git://#{domain}/#{repo}/#{revision}/#{file_path}"
+      return url
+    end
+
+    #return path to tar.gz of git repo
+    def Git.pack(domain,repo,revision="HEAD")
+      repo_dir = Git.pull(domain,repo,revision)
+      repo_name = repo.split("/").last
+      tar_gz_path = "#{repo_dir}/../#{repo_name}.tar.gz"
+      pack_cmd = "cd #{repo_dir} && git archive #{revision} --format=tar.gz > #{tar_gz_path}"
+      pack_cmd.bash(true)
+      FileUtils.rm_r(repo_dir,:force=>true)
+      return tar_gz_path
+    end
+
+    #confirm that git file exists
+    def Git.exists?(url)
+      domain,repo,revision,file_path=[]
+      url.split("/").ie do |url_nodes|
+        domain    = url_nodes[2]
+        repo      = url_nodes[3..4].join("/")
+        revision  = url_nodes[5]
+        file_path = url_nodes[6..-1].join("/")
+      end
+      repo_dir = Git.pull(domain,repo,revision)
+      full_path = "#{repo_dir}/#{file_path}"
+      exists = File.exists?(full_path)
+      if exists
+        FileUtils.rm_r(repo_dir,:force=>true)
+        return exists
+      else
+        raise "Unable to find #{full_path}"
+      end
+    end
+
+    #pulls a git repo and sets it to the specified revision in the
+    #specified folder
+    def Git.pull(domain,repo,revision,run_dir=Dir.mktmpdir)
+      domain_properties = Git.config['domains'][domain]
+      user,host,key = ['user','host','key'].map{|k| domain_properties[k]}
+      #create folder for repo and command
+      run_file_path = run_dir + "/cmd.sh"
+      #put together command
+      git_prefix = key ? "ssh-add #{Base.root}/#{key};" : ""
+      git_suffix = (revision=="HEAD" ? " --depth=1" : "; git checkout -q #{revision}")
+      #add keys, clone repo, go to specific revision, execute command
+      full_cmd = "cd #{run_dir};#{git_prefix}git clone -q #{user}@#{host}:#{repo}.git#{git_suffix}"
+      #put command in file, run ssh-agent bash on it
+      File.open(run_file_path,"w") {|f| f.print(full_cmd)}
+      run_cmd = "ssh-agent bash #{run_file_path}"
+      #run the command, it will return an exception if there are issues
+      run_cmd.bash(true)
+      repo_name = repo.split("/").last
+      repo_dir = "#{run_dir}/#{repo_name}"
+      return repo_dir
+    end
+
+    def Git.read_by_dataset_path(dst_path,user_name,*args)
+      domain,repo,revision,file_path = []
+      dst_path.split("/").ie do |path_nodes|
+        domain    = path_nodes[0]
+        repo      = path_nodes[1..2].join("/")
+        revision  = path_nodes[3]
+        file_path = path_nodes[4..-1].join("/")
+      end
+      #slash in front of path
+      repo_dir = Git.pull(domain,repo,revision)
+      full_path = "#{repo_dir}/#{file_path}"
+      result = "cat #{full_path}".bash(true)
+      FileUtils.rm_r(repo_dir,:force=>true)
+      return result
+    end
+  end
+end

data/lib/mobilize-ssh/handlers/ssh.rb

@@ -1,45 +1,8 @@
 module Mobilize
   module Ssh
-    def Ssh.config
-      Base.config('ssh')
-    end
-
-    def Ssh.tmp_file_dir
-      Ssh.config['tmp_file_dir']
-    end
-
-    def Ssh.host(node)
-      Ssh.config['nodes'][node]['host']
-    end
-
-    def Ssh.gateway(node)
-      Ssh.config['nodes'][node]['gateway']
-    end
-
-    def Ssh.sudoers(node)
-      Ssh.config['nodes'][node]['sudoers']
-    end
-
-    def Ssh.su_all_users(node)
-      Ssh.config['nodes'][node]['su_all_users']
-    end
-
-    def Ssh.nodes
-      Ssh.config['nodes'].keys
-    end
-
-    def Ssh.default_node
-      Ssh.nodes.first
-    end
-
-    #determine if current machine is on host domain, needs gateway if one is provided and it is not
-    def Ssh.needs_gateway?(node)
-      host_domain_name = Ssh.host(node)['name'].split(".")[-2..-1].join(".")
-      return true if Ssh.gateway(node) and Socket.domain_name != host_domain_name
-    end
-
+    #adds convenience methods
+    require "#{File.dirname(__FILE__)}/../helpers/ssh_helper"
     def Ssh.pop_comm_dir(comm_dir,file_hash)
-      FileUtils.rm_r comm_dir, :force=>true
       file_hash.each do |fname,fdata|
         fpath = "#{comm_dir}/#{fname}"
         #for now, only gz is binary
@@ -50,18 +13,6 @@ module Mobilize
       return true if file_hash.keys.length>0
     end
 
-    def Ssh.set_key_permissions(key_path)
-      #makes sure permissions are set as appropriate for ssh key
-      raise "could not find ssh key at #{key_path}" unless File.exists?(key_path)
-      #keys named with .pem are 
-      if key_path.ends_with?(".pem")
-        File.chmod(0400,key_path) unless File.stat(key_path).mode.to_s(8)[3..5] == "400"
-      else
-        File.chmod(0600,key_path) unless File.stat(key_path).mode.to_s(8)[3..5] == "600"
-      end
-      return true
-    end
-
     # converts a source path or target path to a dst in the context of handler and stage
     def Ssh.path_to_dst(path,stage_path,gdrive_slot)
       has_handler = true if path.index("://")
@@ -102,7 +53,6 @@ module Mobilize
     def Ssh.scp(node,from_path,to_path)
       name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
-      Ssh.set_key_permissions(key_path)
       opts = {:port=>(port || 22),:keys=>key_path}
       if Ssh.needs_gateway?(node)
         gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
@@ -117,25 +67,31 @@ module Mobilize
       return true
     end
 
-    def Ssh.run(node,command,user,file_hash={})
-      key,default_user = Ssh.host(node).ie{|h| ['key','user'].map{|k| h[k]}}
-      key_path = "#{Base.root}/#{key}"
-      Ssh.set_key_permissions(key_path)
+    def Ssh.run(node,command,user,file_hash={},params={})
+      default_user = Ssh.host(node)['user']
       file_hash ||= {}
       #make sure the dir for this command is clear
       comm_md5 = [user,node,command,file_hash.keys.to_s,Time.now.to_f.to_s].join.to_md5
-      comm_dir = "#{Ssh.tmp_file_dir}#{comm_md5}"
+      comm_dir = Dir.mktmpdir
+      #add in default methods to the params
+      params.merge(Ssh.default_params)
+      #replace any params in the file_hash and command
+      params.each do |k,v|
+        command.gsub!(k,v)
+        file_hash.each do |name,data|
+          data.gsub!(k,v)
+        end
+      end
       #populate comm dir with any files
       Ssh.pop_comm_dir(comm_dir,file_hash)
-      #move any files up to the node
-      rem_dir = nil
       #make sure user starts in rem_dir
       rem_dir = "#{comm_md5}/"
       #make sure the rem_dir is gone
       Ssh.fire!(node,"sudo rm -rf #{rem_dir}")
       if File.exists?(comm_dir)
         Ssh.scp(node,comm_dir,rem_dir)
-        FileUtils.rm_r comm_dir, :force=>true
+        #make sure comm_dir is removed
+        FileUtils.rm_r(comm_dir,:force=>true)
       else
         #create folder
         mkdir_command = "mkdir #{rem_dir}"
@@ -163,7 +119,6 @@ module Mobilize
     def Ssh.fire!(node,cmd)
       name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
       key_path = "#{Base.root}/#{key}"
-      Ssh.set_key_permissions(key_path)
       opts = {:port=>(port || 22),:keys=>key_path}
       response = if Ssh.needs_gateway?(node)
                    gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
@@ -193,17 +148,15 @@ module Mobilize
     def Ssh.write(node,fdata,to_path,binary=false)
       from_path = Ssh.tmp_file(fdata,binary)
       Ssh.scp(node,from_path,to_path)
-      FileUtils.rm from_path
+      #make sure local is removed
+      FileUtils.rm_r(from_path,:force=>true)
       return true
     end
 
     def Ssh.tmp_file(fdata,binary=false,fpath=nil)
       #creates a file under tmp/files with an md5 from the data
-      tmp_file_path = fpath || "#{Ssh.tmp_file_dir}#{(fdata + Time.now.utc.to_f.to_s).to_md5}"
+      tmp_file_path = fpath || "#{Dir.mktmpdir}/#{(fdata + Time.now.utc.to_f.to_s).to_md5}"
       write_mode = binary ? "wb" : "w"
-      #make sure folder is created
-      tmp_file_dir = tmp_file_path.split("/")[0..-2].join("/")
-      FileUtils.mkdir_p(tmp_file_dir)
       #write data to path
       File.open(tmp_file_path,write_mode) {|f| f.print(fdata)}
       return tmp_file_path
@@ -268,6 +221,8 @@ module Mobilize
       response = {}
       response['out_url'] = Dataset.write_by_url("gridfs://#{s.path}/out",result['stdout'].to_s,Gdrive.owner_name)
       response['err_url'] = Dataset.write_by_url("gridfs://#{s.path}/err",result['stderr'].to_s,Gdrive.owner_name) if result['stderr'].to_s.length>0
+      #is an error if there is no out and there is an err, regardless of signal
+      result['exit_code'] = 500 if result['stdout'].to_s.strip.length==0 and result['stderr'].to_s.strip.length>0
       response['signal'] = result['exit_code']
       response
     end

data/lib/mobilize-ssh/helpers/ssh_helper.rb

@@ -0,0 +1,45 @@
+module Mobilize
+  module Ssh
+    def self.config
+      Base.config('ssh')
+    end
+
+    def self.host(node)
+      self.config['nodes'][node]['host']
+    end
+
+    def self.gateway(node)
+      self.config['nodes'][node]['gateway']
+    end
+
+    def self.sudoers(node)
+      self.config['nodes'][node]['sudoers']
+    end
+
+    def self.su_all_users(node)
+      self.config['nodes'][node]['su_all_users']
+    end
+
+    def self.nodes
+      self.config['nodes'].keys
+    end
+
+    def self.default_node
+      self.nodes.first
+    end
+
+    #determine if current machine is on host domain, needs gateway if one is provided and it is not
+    def self.needs_gateway?(node)
+      host_domain_name = self.host(node)['name'].split(".")[-2..-1].join(".")
+      return true if self.gateway(node) and Socket.domain_name != host_domain_name
+    end
+
+    def self.default_params
+      time = Time.now.utc
+      {
+       '$utc_date'=>time.strftime("%Y-%m-%d"),
+       '$utc_time'=>time.strftime("%H:%M"),
+      }
+    end
+  end
+end

data/lib/mobilize-ssh/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Ssh
-    VERSION = "1.297"
+    VERSION = "1.298"
   end
 end

data/lib/mobilize-ssh.rb

@@ -12,3 +12,4 @@ module Mobilize
   end
 end
 require "mobilize-ssh/handlers/ssh"
+require "mobilize-ssh/handlers/git"

data/mobilize-ssh.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-base","1.297"
+  gem.add_runtime_dependency "mobilize-base","1.298"
   gem.add_runtime_dependency "net-ssh"
   gem.add_runtime_dependency "net-scp"
   gem.add_runtime_dependency "net-ssh-gateway"

data/test/mobilize-ssh_test.rb

@@ -14,8 +14,8 @@ describe "Mobilize" do
 
     gdrive_slot = Mobilize::Gdrive.owner_email
     puts "create user 'mobilize'"
-    user = gdrive_slot.split("@").first
-    u = Mobilize::User.where(:name=>user).first
+    user_name = gdrive_slot.split("@").first
+    u = Mobilize::User.where(:name=>user_name).first
     r = u.runner
 
     rb_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
@@ -42,6 +42,7 @@ describe "Mobilize" do
     ssh_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_1.out",gdrive_slot)
     ssh_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_2.out",gdrive_slot)
     ssh_target_sheet_3 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_3.out",gdrive_slot)
+    ssh_target_sheet_4 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_4.out",gdrive_slot)
     [ssh_target_sheet_1,ssh_target_sheet_2,ssh_target_sheet_3].each {|s| s.delete if s}
 
     ssh_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/ssh_job_rows.yml")
@@ -59,10 +60,12 @@ describe "Mobilize" do
     ssh_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_1.out",gdrive_slot)
     ssh_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_2.out",gdrive_slot)
     ssh_target_sheet_3 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_3.out",gdrive_slot)
+    ssh_target_sheet_4 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_4.out",gdrive_slot)
 
     assert ssh_target_sheet_1.to_tsv.length > 100
     assert ssh_target_sheet_2.to_tsv.length > 100
     assert ssh_target_sheet_3.to_tsv.length > 3
+    assert ssh_target_sheet_4.to_tsv.length > 100
 
   end
 

data/test/ssh_job_rows.yml

@@ -1,3 +1,4 @@
+---
 - name: test_ssh_1
   active: true
   trigger: once
@@ -16,3 +17,9 @@
   status: ""
   stage1: 'ssh.run cmd:"whoami"'
   stage2: 'gsheet.write source:"stage1", target:"test_ssh_3.out"'
+- name: test_ssh_4
+  active: true
+  trigger: "after test_ssh_3"
+  status: ""
+  stage1: 'ssh.run node:"test_node", user:root, sources:["git://DeNA/mobilize-ssh/test/code.rb","git://DeNA/mobilize-ssh/test/code.sh"], cmd:"ruby code.rb"'
+  stage2: 'gsheet.write source:stage1, target:"test_ssh_4.out"'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-ssh
 version: !ruby/object:Gem::Version
-  version: '1.297'
+  version: '1.298'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-06 00:00:00.000000000 Z
+date: 2013-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-base
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.297'
+        version: '1.298'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '1.297'
+        version: '1.298'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -91,7 +91,9 @@ files:
 - lib/mobilize-ssh/extensions/net-ssh-connection-session.rb
 - lib/mobilize-ssh/extensions/net-ssh-gateway.rb
 - lib/mobilize-ssh/extensions/socket.rb
+- lib/mobilize-ssh/handlers/git.rb
 - lib/mobilize-ssh/handlers/ssh.rb
+- lib/mobilize-ssh/helpers/ssh_helper.rb
 - lib/mobilize-ssh/tasks.rb
 - lib/mobilize-ssh/version.rb
 - lib/samples/ssh.yml
@@ -117,7 +119,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1145721245595334784
+      hash: 2943604812785338750
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -126,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1145721245595334784
+      hash: 2943604812785338750
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25