mobilize-ssh 1.36 → 1.291
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +223 -3
- data/lib/mobilize-ssh/extensions/string.rb +5 -0
- data/lib/mobilize-ssh/handlers/ssh.rb +69 -32
- data/lib/mobilize-ssh/tasks.rb +3 -3
- data/lib/mobilize-ssh/version.rb +1 -1
- data/lib/mobilize-ssh.rb +1 -4
- data/lib/samples/ssh.yml +3 -0
- data/mobilize-ssh.gemspec +1 -1
- data/test/{fixtures/code.rb → code.rb} +0 -0
- data/test/{fixtures/code.sh → code.sh} +0 -0
- data/test/code2.sh +1 -0
- data/test/mobilize-ssh_test.rb +92 -0
- data/test/ssh_job_rows.yml +18 -0
- data/test/test_helper.rb +0 -1
- metadata +26 -46
- data/lib/mobilize-ssh/handlers/git.rb +0 -120
- data/lib/mobilize-ssh/helpers/ssh_helper.rb +0 -37
- data/lib/samples/git.yml +0 -34
- data/test/fixtures/code2.sh +0 -1
- data/test/fixtures/integration_expected.yml +0 -37
- data/test/fixtures/integration_jobs.yml +0 -25
- data/test/integration/mobilize-ssh_test.rb +0 -46
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7ed1a7197b5d22a82360be98d54de38cc907f491
|
4
|
+
data.tar.gz: 9f13be4303ff61696a5e44087f0a88954585b4ae
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: eb9b1f01968105c9d79886008e653cc04ba50788437b571373a20e2588c9c1025ab91cea4cccf2224517b479b4dc4aa3e9c0924c860294574f03db19d90350e4
|
7
|
+
data.tar.gz: 39035b250da3e23cb6a8330ca51433c59628bd8d8724c259d3701f57953b835927625d8b007fca23654d45de8b1802278f9d8c3a9ff2a35f968757e2ea4b26be
|
data/README.md
CHANGED
@@ -1,4 +1,224 @@
|
|
1
|
-
Mobilize
|
2
|
-
|
1
|
+
Mobilize-Ssh
|
2
|
+
============
|
3
3
|
|
4
|
-
|
4
|
+
Mobilize-Ssh adds the power of ssh to [mobilize-base][mobilize-base].
|
5
|
+
* move files, execute scripts, and output logs and datasets, all through
|
6
|
+
Google Spreadsheets.
|
7
|
+
|
8
|
+
Table Of Contents
|
9
|
+
-----------------
|
10
|
+
* [Overview](#section_Overview)
|
11
|
+
* [Install](#section_Install)
|
12
|
+
* [Mobilize-Ssh](#section_Install_Mobilize-Ssh)
|
13
|
+
* [Install Dirs and Files](#section_Install_Dirs_and_Files)
|
14
|
+
* [Configure](#section_Configure)
|
15
|
+
* [Ssh](#section_Configure_Ssh)
|
16
|
+
* [Start](#section_Start)
|
17
|
+
* [Create Job](#section_Start_Create_Job)
|
18
|
+
* [Run Test](#section_Start_Run_Test)
|
19
|
+
* [Meta](#section_Meta)
|
20
|
+
* [Author](#section_Author)
|
21
|
+
|
22
|
+
<a name='section_Overview'></a>
|
23
|
+
Overview
|
24
|
+
-----------
|
25
|
+
|
26
|
+
* Mobilize-ssh adds script deployment to mobilize-base.
|
27
|
+
|
28
|
+
<a name='section_Install'></a>
|
29
|
+
Install
|
30
|
+
------------
|
31
|
+
|
32
|
+
Make sure you go through all the steps in the [mobilize-base][mobilize-base]
|
33
|
+
install section first.
|
34
|
+
|
35
|
+
<a name='section_Install_Mobilize-Ssh'></a>
|
36
|
+
### Mobilize-Ssh
|
37
|
+
|
38
|
+
add this to your Gemfile:
|
39
|
+
|
40
|
+
``` ruby
|
41
|
+
gem "mobilize-ssh"
|
42
|
+
```
|
43
|
+
|
44
|
+
or do
|
45
|
+
|
46
|
+
$ gem install mobilize-ssh
|
47
|
+
|
48
|
+
for a ruby-wide install.
|
49
|
+
|
50
|
+
<a name='section_Install_Dirs_and_Files'></a>
|
51
|
+
### Dirs and Files
|
52
|
+
|
53
|
+
### Rakefile
|
54
|
+
|
55
|
+
Inside the Rakefile in your project's root dir, make sure you have:
|
56
|
+
|
57
|
+
``` ruby
|
58
|
+
require 'mobilize-base/tasks'
|
59
|
+
require 'mobilize-ssh/tasks'
|
60
|
+
```
|
61
|
+
|
62
|
+
This defines rake tasks essential to run the environment.
|
63
|
+
|
64
|
+
### Config Dir
|
65
|
+
|
66
|
+
run
|
67
|
+
|
68
|
+
$ rake mobilize_ssh:setup
|
69
|
+
|
70
|
+
This will copy over a sample ssh.yml to your config dir.
|
71
|
+
|
72
|
+
It will also add mobilize-ssh to the extensions in jobtracker.yml.
|
73
|
+
|
74
|
+
<a name='section_Configure'></a>
|
75
|
+
Configure
|
76
|
+
------------
|
77
|
+
|
78
|
+
<a name='section_Configure_Ssh'></a>
|
79
|
+
### Configure Ssh
|
80
|
+
|
81
|
+
The Ssh configuration consists of:
|
82
|
+
* tmp_file_dir, which is where files will be stored before being scp'd
|
83
|
+
over to the nodes. They will be deleted afterwards, unless the job
|
84
|
+
fails in mid-copy. By default this is tmp/file/.
|
85
|
+
* nodes, identified by aliases, such as `test_node`. This alias is what you should
|
86
|
+
pass into the "node" param over in the ssh.run task.
|
87
|
+
* if no node is specified, commands will default to the first node listed.
|
88
|
+
|
89
|
+
Each node has:
|
90
|
+
* a host;
|
91
|
+
* a gateway (optional); If you don't need a gateway, remove that row from the configuration file.
|
92
|
+
* sudoers; these are user names that are allowed to pass user params
|
93
|
+
to the run call. This requires passwordless sudo for the host user.
|
94
|
+
* su_all_users true/false option, which ensures that commands are executed by the
|
95
|
+
user on the Runner. It prefixes all commands with sudo su <user> before executing the
|
96
|
+
command. This is strongly recommended if possible as it ensures users do
|
97
|
+
not overstep their permissions. This requires passwordless sudo for the
|
98
|
+
host user and accounts on the host machine for each user.
|
99
|
+
|
100
|
+
Each host and gateway has a series of ssh params:
|
101
|
+
* name - the ip address or name of the host
|
102
|
+
* key - the relative path of the ssh key file. Default is
|
103
|
+
"config/mobilize/ssh_private.key"
|
104
|
+
* port - the port to connect on
|
105
|
+
* user - the user you are connecting as
|
106
|
+
|
107
|
+
Sample ssh.yml:
|
108
|
+
|
109
|
+
``` yml
|
110
|
+
---
|
111
|
+
development:
|
112
|
+
tmp_file_dir: tmp/file/
|
113
|
+
nodes:
|
114
|
+
dev_node:
|
115
|
+
sudoers:
|
116
|
+
- sudo_user
|
117
|
+
su_all_users: true
|
118
|
+
host:
|
119
|
+
name: dev-host.com
|
120
|
+
key: config/mobilize/ssh_private.key
|
121
|
+
port: 22
|
122
|
+
user: host_user
|
123
|
+
gateway:
|
124
|
+
name: dev-gateway.com
|
125
|
+
key: config/mobilize/ssh_private.key
|
126
|
+
port: 22
|
127
|
+
user: gateway_user
|
128
|
+
test:
|
129
|
+
tmp_file_dir: tmp/file/
|
130
|
+
nodes:
|
131
|
+
test_node:
|
132
|
+
sudoers:
|
133
|
+
- sudo_user
|
134
|
+
su_all_users: true
|
135
|
+
host:
|
136
|
+
name: test-host.com
|
137
|
+
key: config/mobilize/ssh_private.key
|
138
|
+
port: 22
|
139
|
+
user: host_user
|
140
|
+
gateway:
|
141
|
+
name: test-gateway.com
|
142
|
+
key: config/mobilize/ssh_private.key
|
143
|
+
port: 22
|
144
|
+
user: gateway_user
|
145
|
+
production:
|
146
|
+
tmp_file_dir: tmp/file/
|
147
|
+
nodes:
|
148
|
+
prod_node:
|
149
|
+
sudoers:
|
150
|
+
- sudo_user
|
151
|
+
su_all_users: true
|
152
|
+
host:
|
153
|
+
name: prod-host.com
|
154
|
+
key: config/mobilize/ssh_private.key
|
155
|
+
port: 22
|
156
|
+
user: host_user
|
157
|
+
gateway:
|
158
|
+
name: prod-gateway.com
|
159
|
+
key: config/mobilize/ssh_private.key
|
160
|
+
port: 22
|
161
|
+
user: gateway_user
|
162
|
+
```
|
163
|
+
|
164
|
+
<a name='section_Start'></a>
|
165
|
+
Start
|
166
|
+
-----
|
167
|
+
|
168
|
+
<a name='section_Start_Create_Job'></a>
|
169
|
+
### Create Job
|
170
|
+
|
171
|
+
* For mobilize-ssh, the following task is available:
|
172
|
+
* ssh.run `node: <node_alias>, cmd: <command>, user: user, sources:[*<source_paths>]`, which reads sources, copies them to a temporary folder on the selected node, and runs the command inside that folder.
|
173
|
+
* user, sources, and node are optional; cmd is required.
|
174
|
+
* specifying user will cause the command to be prefixed with sudo su <user> -c.
|
175
|
+
* non-google sources will also be read as the specified user.
|
176
|
+
* not specifying node will cause the command to be run on the default node.
|
177
|
+
* ssh sources can be specified with syntax
|
178
|
+
`ssh://<node><file_full_path>`. If node is omitted, default node will be used.
|
179
|
+
* `<node><file_full_path>` and `<file_full_path>` can be used in the context of ssh.run, but if the path has only 1 slash, or none, it will try to find a google sheet or file instead.
|
180
|
+
* The test uses `ssh.run node:"test_node", cmd:"ruby code.rb", user: "root", sources:["code.rb","code.sh"]`
|
181
|
+
|
182
|
+
<a name='section_Start_Run_Test'></a>
|
183
|
+
### Run Test
|
184
|
+
|
185
|
+
To run tests, you will need to
|
186
|
+
|
187
|
+
1) go through the [mobilize-base][mobilize-base] test first
|
188
|
+
|
189
|
+
2) clone the mobilize-ssh repository
|
190
|
+
|
191
|
+
From the project folder, run
|
192
|
+
|
193
|
+
3) $ rake mobilize_ssh:setup
|
194
|
+
|
195
|
+
Copy over the config files from the mobilize-base project into the
|
196
|
+
config dir, and populate the values in the ssh.yml file, esp. the
|
197
|
+
test_node item.
|
198
|
+
|
199
|
+
You should also copy the ssh private key you wish to use into your
|
200
|
+
desired path (by default: config/mobilize/ssh_private.key), and make sure it is referenced in ssh.yml
|
201
|
+
|
202
|
+
3) $ rake test
|
203
|
+
|
204
|
+
This will populate your test Runner from mobilize-base with a sample ssh job.
|
205
|
+
|
206
|
+
The purpose of the test will be to deploy two code files, have the first
|
207
|
+
execute the second, which is a "tail /var/log/syslog" command, and write the resulting output to a gsheet.
|
208
|
+
|
209
|
+
<a name='section_Meta'></a>
|
210
|
+
Meta
|
211
|
+
----
|
212
|
+
|
213
|
+
* Code: `git clone git://github.com/dena/mobilize-ssh.git`
|
214
|
+
* Home: <https://github.com/dena/mobilize-ssh>
|
215
|
+
* Bugs: <https://github.com/dena/mobilize-ssh/issues>
|
216
|
+
* Gems: <http://rubygems.org/gems/mobilize-ssh>
|
217
|
+
|
218
|
+
<a name='section_Author'></a>
|
219
|
+
Author
|
220
|
+
------
|
221
|
+
|
222
|
+
Cassio Paes-Leme :: cpaesleme@dena.com :: @cpaesleme
|
223
|
+
|
224
|
+
[mobilize-base]: https://github.com/dena/mobilize-base
|
@@ -1,8 +1,45 @@
|
|
1
1
|
module Mobilize
|
2
2
|
module Ssh
|
3
|
-
|
4
|
-
|
3
|
+
def Ssh.config
|
4
|
+
Base.config('ssh')
|
5
|
+
end
|
6
|
+
|
7
|
+
def Ssh.tmp_file_dir
|
8
|
+
Ssh.config['tmp_file_dir']
|
9
|
+
end
|
10
|
+
|
11
|
+
def Ssh.host(node)
|
12
|
+
Ssh.config['nodes'][node]['host']
|
13
|
+
end
|
14
|
+
|
15
|
+
def Ssh.gateway(node)
|
16
|
+
Ssh.config['nodes'][node]['gateway']
|
17
|
+
end
|
18
|
+
|
19
|
+
def Ssh.sudoers(node)
|
20
|
+
Ssh.config['nodes'][node]['sudoers']
|
21
|
+
end
|
22
|
+
|
23
|
+
def Ssh.su_all_users(node)
|
24
|
+
Ssh.config['nodes'][node]['su_all_users']
|
25
|
+
end
|
26
|
+
|
27
|
+
def Ssh.nodes
|
28
|
+
Ssh.config['nodes'].keys
|
29
|
+
end
|
30
|
+
|
31
|
+
def Ssh.default_node
|
32
|
+
Ssh.nodes.first
|
33
|
+
end
|
34
|
+
|
35
|
+
#determine if current machine is on host domain, needs gateway if one is provided and it is not
|
36
|
+
def Ssh.needs_gateway?(node)
|
37
|
+
host_domain_name = Ssh.host(node)['name'].split(".")[-2..-1].join(".")
|
38
|
+
return true if Ssh.gateway(node) and Socket.domain_name != host_domain_name
|
39
|
+
end
|
40
|
+
|
5
41
|
def Ssh.pop_comm_dir(comm_dir,file_hash)
|
42
|
+
FileUtils.rm_r comm_dir, :force=>true
|
6
43
|
file_hash.each do |fname,fdata|
|
7
44
|
fpath = "#{comm_dir}/#{fname}"
|
8
45
|
#for now, only gz is binary
|
@@ -13,6 +50,13 @@ module Mobilize
|
|
13
50
|
return true if file_hash.keys.length>0
|
14
51
|
end
|
15
52
|
|
53
|
+
def Ssh.set_key_permissions(key_path)
|
54
|
+
#makes sure permissions are set as appropriate for ssh key
|
55
|
+
raise "could not find ssh key at #{key_path}" unless File.exists?(key_path)
|
56
|
+
File.chmod(0600,key_path) unless File.stat(key_path).mode.to_s(8)[3..5] == "600"
|
57
|
+
return true
|
58
|
+
end
|
59
|
+
|
16
60
|
# converts a source path or target path to a dst in the context of handler and stage
|
17
61
|
def Ssh.path_to_dst(path,stage_path,gdrive_slot)
|
18
62
|
has_handler = true if path.index("://")
|
@@ -53,6 +97,7 @@ module Mobilize
|
|
53
97
|
def Ssh.scp(node,from_path,to_path)
|
54
98
|
name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
|
55
99
|
key_path = "#{Base.root}/#{key}"
|
100
|
+
Ssh.set_key_permissions(key_path)
|
56
101
|
opts = {:port=>(port || 22),:keys=>key_path}
|
57
102
|
if Ssh.needs_gateway?(node)
|
58
103
|
gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
|
@@ -67,30 +112,25 @@ module Mobilize
|
|
67
112
|
return true
|
68
113
|
end
|
69
114
|
|
70
|
-
def Ssh.run(node,command,
|
71
|
-
|
115
|
+
def Ssh.run(node,command,user,file_hash={})
|
116
|
+
key,default_user = Ssh.host(node).ie{|h| ['key','user'].map{|k| h[k]}}
|
117
|
+
key_path = "#{Base.root}/#{key}"
|
118
|
+
Ssh.set_key_permissions(key_path)
|
72
119
|
file_hash ||= {}
|
73
|
-
run_params ||={}
|
74
120
|
#make sure the dir for this command is clear
|
75
|
-
comm_md5 = [
|
76
|
-
comm_dir =
|
77
|
-
#
|
78
|
-
run_params.each do |k,v|
|
79
|
-
command.gsub!("@#{k}",v)
|
80
|
-
file_hash.each do |name,data|
|
81
|
-
data.gsub!("@#{k}",v)
|
82
|
-
end
|
83
|
-
end
|
84
|
-
#populate comm dir with any files
|
121
|
+
comm_md5 = [user,node,command,file_hash.keys.to_s,Time.now.to_f.to_s].join.to_md5
|
122
|
+
comm_dir = "#{Ssh.tmp_file_dir}#{comm_md5}"
|
123
|
+
#populate comm dir with any files
|
85
124
|
Ssh.pop_comm_dir(comm_dir,file_hash)
|
125
|
+
#move any files up to the node
|
126
|
+
rem_dir = nil
|
86
127
|
#make sure user starts in rem_dir
|
87
128
|
rem_dir = "#{comm_md5}/"
|
88
129
|
#make sure the rem_dir is gone
|
89
130
|
Ssh.fire!(node,"sudo rm -rf #{rem_dir}")
|
90
131
|
if File.exists?(comm_dir)
|
91
132
|
Ssh.scp(node,comm_dir,rem_dir)
|
92
|
-
|
93
|
-
FileUtils.rm_r(comm_dir,:force=>true)
|
133
|
+
FileUtils.rm_r comm_dir, :force=>true
|
94
134
|
else
|
95
135
|
#create folder
|
96
136
|
mkdir_command = "mkdir #{rem_dir}"
|
@@ -102,9 +142,9 @@ module Mobilize
|
|
102
142
|
Ssh.write(node,command,cmd_path)
|
103
143
|
full_cmd = "(cd #{rem_dir} && sh #{cmd_file})"
|
104
144
|
#fire_cmd runs sh on cmd_path, optionally with sudo su
|
105
|
-
if
|
145
|
+
if user != default_user
|
106
146
|
#make sure user owns the folder and all files
|
107
|
-
fire_cmd = %{sudo chown -R #{
|
147
|
+
fire_cmd = %{sudo chown -R #{user} #{rem_dir}; sudo su #{user} -c "#{full_cmd}"}
|
108
148
|
rm_cmd = %{sudo rm -rf #{rem_dir}}
|
109
149
|
else
|
110
150
|
fire_cmd = full_cmd
|
@@ -118,6 +158,7 @@ module Mobilize
|
|
118
158
|
def Ssh.fire!(node,cmd)
|
119
159
|
name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
|
120
160
|
key_path = "#{Base.root}/#{key}"
|
161
|
+
Ssh.set_key_permissions(key_path)
|
121
162
|
opts = {:port=>(port || 22),:keys=>key_path}
|
122
163
|
response = if Ssh.needs_gateway?(node)
|
123
164
|
gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}}
|
@@ -147,15 +188,17 @@ module Mobilize
|
|
147
188
|
def Ssh.write(node,fdata,to_path,binary=false)
|
148
189
|
from_path = Ssh.tmp_file(fdata,binary)
|
149
190
|
Ssh.scp(node,from_path,to_path)
|
150
|
-
|
151
|
-
FileUtils.rm_r(from_path,:force=>true)
|
191
|
+
FileUtils.rm from_path
|
152
192
|
return true
|
153
193
|
end
|
154
194
|
|
155
195
|
def Ssh.tmp_file(fdata,binary=false,fpath=nil)
|
156
196
|
#creates a file under tmp/files with an md5 from the data
|
157
|
-
tmp_file_path = fpath || "#{
|
197
|
+
tmp_file_path = fpath || "#{Ssh.tmp_file_dir}#{(fdata + Time.now.utc.to_f.to_s).to_md5}"
|
158
198
|
write_mode = binary ? "wb" : "w"
|
199
|
+
#make sure folder is created
|
200
|
+
tmp_file_dir = tmp_file_path.split("/")[0..-2].join("/")
|
201
|
+
FileUtils.mkdir_p(tmp_file_dir)
|
159
202
|
#write data to path
|
160
203
|
File.open(tmp_file_path,write_mode) {|f| f.print(fdata)}
|
161
204
|
return tmp_file_path
|
@@ -165,7 +208,7 @@ module Mobilize
|
|
165
208
|
s = Stage.where(:path=>stage_path).first
|
166
209
|
u = s.job.runner.user
|
167
210
|
user_name = s.params['user']
|
168
|
-
node = s.params['node']
|
211
|
+
node = s.params['node']
|
169
212
|
node = Ssh.default_node unless Ssh.nodes.include?(node)
|
170
213
|
if user_name and !Ssh.sudoers(node).include?(u.name)
|
171
214
|
raise "#{u.name} does not have su permissions for this node"
|
@@ -183,12 +226,9 @@ module Mobilize
|
|
183
226
|
s.sources(gdrive_slot).each do |sdst|
|
184
227
|
split_path = sdst.path.split("/")
|
185
228
|
#if path is to stage output, name with stage name
|
186
|
-
file_name = if
|
187
|
-
|
229
|
+
file_name = if split_path.last == "out" and
|
230
|
+
(1..5).to_a.map{|n| "stage#{n.to_s}"}.include?(split_path[-2].to_s)
|
188
231
|
"#{split_path[-2]}.out"
|
189
|
-
elsif (1..5).to_a.map{|n| "stage#{n.to_s}"}.include?(split_path.last[-6..-1])
|
190
|
-
#runner<jobname>stage1
|
191
|
-
"#{split_path.last[-6..-1]}.out"
|
192
232
|
else
|
193
233
|
split_path.last
|
194
234
|
end
|
@@ -215,14 +255,11 @@ module Mobilize
|
|
215
255
|
user_name = Ssh.user_name_by_stage_path(stage_path)
|
216
256
|
file_hash = Ssh.file_hash_by_stage_path(stage_path,gdrive_slot)
|
217
257
|
Gdrive.unslot_worker_by_path(stage_path)
|
218
|
-
|
219
|
-
result = Ssh.run(node,command,user_name,file_hash,run_params)
|
258
|
+
result = Ssh.run(node,command,user_name,file_hash)
|
220
259
|
#use Gridfs to cache result
|
221
260
|
response = {}
|
222
261
|
response['out_url'] = Dataset.write_by_url("gridfs://#{s.path}/out",result['stdout'].to_s,Gdrive.owner_name)
|
223
262
|
response['err_url'] = Dataset.write_by_url("gridfs://#{s.path}/err",result['stderr'].to_s,Gdrive.owner_name) if result['stderr'].to_s.length>0
|
224
|
-
#is an error if there is no out and there is an err, regardless of signal
|
225
|
-
result['exit_code'] = 500 if result['stdout'].to_s.strip.length==0 and result['stderr'].to_s.strip.length>0
|
226
263
|
response['signal'] = result['exit_code']
|
227
264
|
response
|
228
265
|
end
|
data/lib/mobilize-ssh/tasks.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
-
|
2
|
-
namespace :mobilize do
|
1
|
+
namespace :mobilize_ssh do
|
3
2
|
desc "Set up config and log folders and files"
|
4
|
-
task :
|
3
|
+
task :setup do
|
4
|
+
require 'yaml'
|
5
5
|
sample_dir = File.dirname(__FILE__) + '/../samples/'
|
6
6
|
sample_files = Dir.entries(sample_dir)
|
7
7
|
config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
|
data/lib/mobilize-ssh/version.rb
CHANGED
data/lib/mobilize-ssh.rb
CHANGED
@@ -6,13 +6,10 @@ require "net/scp"
|
|
6
6
|
require "mobilize-ssh/extensions/net-ssh-connection-session"
|
7
7
|
require "mobilize-ssh/extensions/net-ssh-gateway"
|
8
8
|
require "mobilize-ssh/extensions/socket"
|
9
|
+
require "mobilize-ssh/extensions/string"
|
9
10
|
|
10
11
|
module Mobilize
|
11
12
|
module Ssh
|
12
|
-
def Ssh.home_dir
|
13
|
-
File.expand_path('..',File.dirname(__FILE__))
|
14
|
-
end
|
15
13
|
end
|
16
14
|
end
|
17
15
|
require "mobilize-ssh/handlers/ssh"
|
18
|
-
require "mobilize-ssh/handlers/git"
|
data/lib/samples/ssh.yml
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
+
tmp_file_dir: tmp/file/
|
3
4
|
nodes:
|
4
5
|
dev_node:
|
5
6
|
sudoers:
|
@@ -16,6 +17,7 @@ development:
|
|
16
17
|
port: 22
|
17
18
|
user: gateway_user
|
18
19
|
test:
|
20
|
+
tmp_file_dir: tmp/file/
|
19
21
|
nodes:
|
20
22
|
test_node:
|
21
23
|
sudoers:
|
@@ -32,6 +34,7 @@ test:
|
|
32
34
|
port: 22
|
33
35
|
user: gateway_user
|
34
36
|
production:
|
37
|
+
tmp_file_dir: tmp/file/
|
35
38
|
nodes:
|
36
39
|
prod_node:
|
37
40
|
sudoers:
|
data/mobilize-ssh.gemspec
CHANGED
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
17
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
18
|
gem.require_paths = ["lib"]
|
19
|
-
gem.add_runtime_dependency "mobilize-base","1.
|
19
|
+
gem.add_runtime_dependency "mobilize-base","1.29"
|
20
20
|
gem.add_runtime_dependency "net-ssh"
|
21
21
|
gem.add_runtime_dependency "net-scp"
|
22
22
|
gem.add_runtime_dependency "net-ssh-gateway"
|
File without changes
|
File without changes
|
data/test/code2.sh
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
tail syslog
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
describe "Mobilize" do
|
4
|
+
|
5
|
+
def before
|
6
|
+
puts 'nothing before'
|
7
|
+
end
|
8
|
+
|
9
|
+
# enqueues 4 workers on Resque
|
10
|
+
it "runs integration test" do
|
11
|
+
|
12
|
+
puts "restart workers"
|
13
|
+
Mobilize::Jobtracker.restart_workers!
|
14
|
+
|
15
|
+
gdrive_slot = Mobilize::Gdrive.owner_email
|
16
|
+
puts "create user 'mobilize'"
|
17
|
+
user = gdrive_slot.split("@").first
|
18
|
+
u = Mobilize::User.where(:name=>user).first
|
19
|
+
r = u.runner
|
20
|
+
|
21
|
+
rb_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
|
22
|
+
sh_code_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code.sh",gdrive_slot)
|
23
|
+
sh_code_sheet2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/code2.sh",gdrive_slot)
|
24
|
+
[rb_code_sheet,sh_code_sheet].each {|s| s.delete if s}
|
25
|
+
|
26
|
+
puts "add test code"
|
27
|
+
rb_code_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/code.rb",gdrive_slot)
|
28
|
+
rb_code_tsv = File.open("#{Mobilize::Base.root}/test/code.rb").read
|
29
|
+
rb_code_sheet.write(rb_code_tsv,Mobilize::Gdrive.owner_name)
|
30
|
+
|
31
|
+
sh_code_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/code.sh",gdrive_slot)
|
32
|
+
sh_code_tsv = File.open("#{Mobilize::Base.root}/test/code.sh").read
|
33
|
+
sh_code_sheet.write(sh_code_tsv,Mobilize::Gdrive.owner_name)
|
34
|
+
|
35
|
+
sh_code_sheet2 = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/code2.sh",gdrive_slot)
|
36
|
+
sh_code_tsv2 = File.open("#{Mobilize::Base.root}/test/code2.sh").read
|
37
|
+
sh_code_sheet2.write(sh_code_tsv2,Mobilize::Gdrive.owner_name)
|
38
|
+
|
39
|
+
jobs_sheet = r.gsheet(gdrive_slot)
|
40
|
+
|
41
|
+
#delete target sheets if they exist
|
42
|
+
ssh_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_1.out",gdrive_slot)
|
43
|
+
ssh_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_2.out",gdrive_slot)
|
44
|
+
ssh_target_sheet_3 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_3.out",gdrive_slot)
|
45
|
+
[ssh_target_sheet_1,ssh_target_sheet_2,ssh_target_sheet_3].each {|s| s.delete if s}
|
46
|
+
|
47
|
+
ssh_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/ssh_job_rows.yml")
|
48
|
+
ssh_job_rows.map{|j| r.jobs(j['name'])}.each{|j| j.delete if j}
|
49
|
+
jobs_sheet.add_or_update_rows(ssh_job_rows)
|
50
|
+
|
51
|
+
puts "job row added, force enqueue runner, wait for stages"
|
52
|
+
r.enqueue!
|
53
|
+
wait_for_stages(900)
|
54
|
+
|
55
|
+
puts "update job status and activity"
|
56
|
+
r.update_gsheet(gdrive_slot)
|
57
|
+
|
58
|
+
puts "jobtracker posted data to test sheet"
|
59
|
+
ssh_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_1.out",gdrive_slot)
|
60
|
+
ssh_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_2.out",gdrive_slot)
|
61
|
+
ssh_target_sheet_3 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_ssh_3.out",gdrive_slot)
|
62
|
+
|
63
|
+
assert ssh_target_sheet_1.to_tsv.length > 100
|
64
|
+
assert ssh_target_sheet_2.to_tsv.length > 100
|
65
|
+
assert ssh_target_sheet_3.to_tsv.length > 3
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
|
70
|
+
time = 0
|
71
|
+
time_since_stage = 0
|
72
|
+
#check for 10 min
|
73
|
+
while time < time_limit and time_since_stage < stage_limit
|
74
|
+
sleep wait_length
|
75
|
+
job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
|
76
|
+
if job_classes.include?("Mobilize::Stage")
|
77
|
+
time_since_stage = 0
|
78
|
+
puts "saw stage at #{time.to_s} seconds"
|
79
|
+
else
|
80
|
+
time_since_stage += wait_length
|
81
|
+
puts "#{time_since_stage.to_s} seconds since stage seen"
|
82
|
+
end
|
83
|
+
time += wait_length
|
84
|
+
puts "total wait time #{time.to_s} seconds"
|
85
|
+
end
|
86
|
+
|
87
|
+
if time >= time_limit
|
88
|
+
raise "Timed out before stage completion"
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
- name: test_ssh_1
|
2
|
+
active: true
|
3
|
+
trigger: once
|
4
|
+
status: ""
|
5
|
+
stage1: 'ssh.run node:"test_node", cmd:"ruby code.rb", user:"root", sources:["code.rb", "code.sh"]'
|
6
|
+
stage2: 'gsheet.write source:"stage1", target:"test_ssh_1.out"'
|
7
|
+
- name: test_ssh_2
|
8
|
+
active: true
|
9
|
+
trigger: "after test_ssh_1"
|
10
|
+
status: ""
|
11
|
+
stage1: 'ssh.run cmd:"sh code2.sh", user:"root", sources:["code2.sh","test_node/var/log/syslog"]'
|
12
|
+
stage2: 'gsheet.write source:"stage1", target:"test_ssh_2.out"'
|
13
|
+
- name: test_ssh_3
|
14
|
+
active: true
|
15
|
+
trigger: "after test_ssh_2"
|
16
|
+
status: ""
|
17
|
+
stage1: 'ssh.run cmd:"whoami"'
|
18
|
+
stage2: 'gsheet.write source:"stage1", target:"test_ssh_3.out"'
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,78 +1,69 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-ssh
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
5
|
-
prerelease:
|
4
|
+
version: '1.291'
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Cassio Paes-Leme
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2013-
|
11
|
+
date: 2013-03-27 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: mobilize-base
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
17
|
- - '='
|
20
18
|
- !ruby/object:Gem::Version
|
21
|
-
version: '1.
|
19
|
+
version: '1.29'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
24
|
- - '='
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
version: '1.
|
26
|
+
version: '1.29'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: net-ssh
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - '>='
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - '>='
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: net-scp
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - '>='
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '0'
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - '>='
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: '0'
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: net-ssh-gateway
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - '>='
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: '0'
|
70
62
|
type: :runtime
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - '>='
|
76
67
|
- !ruby/object:Gem::Version
|
77
68
|
version: '0'
|
78
69
|
description: mobilize-ssh allows you to automate ssh commands and files across hosts
|
@@ -91,58 +82,47 @@ files:
|
|
91
82
|
- lib/mobilize-ssh/extensions/net-ssh-connection-session.rb
|
92
83
|
- lib/mobilize-ssh/extensions/net-ssh-gateway.rb
|
93
84
|
- lib/mobilize-ssh/extensions/socket.rb
|
94
|
-
- lib/mobilize-ssh/
|
85
|
+
- lib/mobilize-ssh/extensions/string.rb
|
95
86
|
- lib/mobilize-ssh/handlers/ssh.rb
|
96
|
-
- lib/mobilize-ssh/helpers/ssh_helper.rb
|
97
87
|
- lib/mobilize-ssh/tasks.rb
|
98
88
|
- lib/mobilize-ssh/version.rb
|
99
|
-
- lib/samples/git.yml
|
100
89
|
- lib/samples/ssh.yml
|
101
90
|
- mobilize-ssh.gemspec
|
102
|
-
- test/
|
103
|
-
- test/
|
104
|
-
- test/
|
105
|
-
- test/
|
106
|
-
- test/fixtures/integration_jobs.yml
|
107
|
-
- test/integration/mobilize-ssh_test.rb
|
91
|
+
- test/code.rb
|
92
|
+
- test/code.sh
|
93
|
+
- test/code2.sh
|
94
|
+
- test/mobilize-ssh_test.rb
|
108
95
|
- test/redis-test.conf
|
96
|
+
- test/ssh_job_rows.yml
|
109
97
|
- test/test_helper.rb
|
110
98
|
homepage: http://github.com/dena/mobilize-ssh
|
111
99
|
licenses: []
|
100
|
+
metadata: {}
|
112
101
|
post_install_message:
|
113
102
|
rdoc_options: []
|
114
103
|
require_paths:
|
115
104
|
- lib
|
116
105
|
required_ruby_version: !ruby/object:Gem::Requirement
|
117
|
-
none: false
|
118
106
|
requirements:
|
119
|
-
- -
|
107
|
+
- - '>='
|
120
108
|
- !ruby/object:Gem::Version
|
121
109
|
version: '0'
|
122
|
-
segments:
|
123
|
-
- 0
|
124
|
-
hash: 3534560230256585865
|
125
110
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
126
|
-
none: false
|
127
111
|
requirements:
|
128
|
-
- -
|
112
|
+
- - '>='
|
129
113
|
- !ruby/object:Gem::Version
|
130
114
|
version: '0'
|
131
|
-
segments:
|
132
|
-
- 0
|
133
|
-
hash: 3534560230256585865
|
134
115
|
requirements: []
|
135
116
|
rubyforge_project:
|
136
|
-
rubygems_version:
|
117
|
+
rubygems_version: 2.0.3
|
137
118
|
signing_key:
|
138
|
-
specification_version:
|
119
|
+
specification_version: 4
|
139
120
|
summary: extend mobilize-base with the ability to run files across hosts
|
140
121
|
test_files:
|
141
|
-
- test/
|
142
|
-
- test/
|
143
|
-
- test/
|
144
|
-
- test/
|
145
|
-
- test/fixtures/integration_jobs.yml
|
146
|
-
- test/integration/mobilize-ssh_test.rb
|
122
|
+
- test/code.rb
|
123
|
+
- test/code.sh
|
124
|
+
- test/code2.sh
|
125
|
+
- test/mobilize-ssh_test.rb
|
147
126
|
- test/redis-test.conf
|
127
|
+
- test/ssh_job_rows.yml
|
148
128
|
- test/test_helper.rb
|
@@ -1,120 +0,0 @@
|
|
1
|
-
module Mobilize
|
2
|
-
module Git
|
3
|
-
def Git.config
|
4
|
-
Base.config('git')
|
5
|
-
end
|
6
|
-
|
7
|
-
def Git.host(domain)
|
8
|
-
Git.config['domains'][domain]['host']
|
9
|
-
end
|
10
|
-
|
11
|
-
def Git.domains
|
12
|
-
Git.config['domains'].keys
|
13
|
-
end
|
14
|
-
|
15
|
-
def Git.default_domain
|
16
|
-
Git.domains.first
|
17
|
-
end
|
18
|
-
|
19
|
-
def Git.repo_key(domain,repo)
|
20
|
-
begin
|
21
|
-
Git.config['domains'][domain]['repo_keys'][repo]
|
22
|
-
rescue
|
23
|
-
nil #no key for public repos
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
# converts a source path or target path to a dst in the context of handler and stage
|
28
|
-
def Git.path_to_dst(path,stage_path,gdrive_slot)
|
29
|
-
red_path = path.split("://").last
|
30
|
-
git_url = Git.url_by_path(red_path)
|
31
|
-
return Dataset.find_or_create_by_url(git_url)
|
32
|
-
end
|
33
|
-
|
34
|
-
def Git.url_by_path(path)
|
35
|
-
path_nodes = path.split("/")
|
36
|
-
domain = path_nodes.first.to_s
|
37
|
-
revision = "HEAD"
|
38
|
-
if Git.domains.include?(domain)
|
39
|
-
repo = path_nodes[1..2].join("/")
|
40
|
-
file_path = path_nodes[3..-1].join("/")
|
41
|
-
else
|
42
|
-
domain = Git.default_domain
|
43
|
-
repo = path_nodes[0..1].join("/")
|
44
|
-
file_path = path_nodes[2..-1].join("/")
|
45
|
-
end
|
46
|
-
url = "git://#{domain}/#{repo}/#{revision}/#{file_path}"
|
47
|
-
return url
|
48
|
-
end
|
49
|
-
|
50
|
-
#return path to tar.gz of git repo
|
51
|
-
def Git.pack(domain,repo,revision="HEAD")
|
52
|
-
repo_dir = Git.pull(domain,repo,revision)
|
53
|
-
repo_name = repo.split("/").last
|
54
|
-
tar_gz_path = "#{repo_dir}/../#{repo_name}.tar.gz"
|
55
|
-
pack_cmd = "cd #{repo_dir} && git archive #{revision} --format=tar.gz > #{tar_gz_path}"
|
56
|
-
pack_cmd.bash(true)
|
57
|
-
FileUtils.rm_r(repo_dir,:force=>true)
|
58
|
-
return tar_gz_path
|
59
|
-
end
|
60
|
-
|
61
|
-
#confirm that git file exists
|
62
|
-
def Git.exists?(url)
|
63
|
-
domain,repo,revision,file_path=[]
|
64
|
-
url.split("/").ie do |url_nodes|
|
65
|
-
domain = url_nodes[2]
|
66
|
-
repo = url_nodes[3..4].join("/")
|
67
|
-
revision = url_nodes[5]
|
68
|
-
file_path = url_nodes[6..-1].join("/")
|
69
|
-
end
|
70
|
-
repo_dir = Git.pull(domain,repo,revision)
|
71
|
-
full_path = "#{repo_dir}/#{file_path}"
|
72
|
-
exists = File.exists?(full_path)
|
73
|
-
if exists
|
74
|
-
FileUtils.rm_r(repo_dir,:force=>true)
|
75
|
-
return exists
|
76
|
-
else
|
77
|
-
raise "Unable to find #{full_path}"
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
#pulls a git repo and sets it to the specified revision in the
|
82
|
-
#specified folder
|
83
|
-
def Git.pull(domain,repo,revision,run_dir=Dir.mktmpdir)
|
84
|
-
domain_properties = Git.config['domains'][domain]
|
85
|
-
user,host= ['user','host'].map{|k| domain_properties[k]}
|
86
|
-
key = Git.repo_key(domain,repo)
|
87
|
-
#create folder for repo and command
|
88
|
-
run_file_path = run_dir + "/cmd.sh"
|
89
|
-
#put together command
|
90
|
-
git_prefix = key ? "ssh-add #{Base.root}/#{key};" : ""
|
91
|
-
git_suffix = (revision=="HEAD" ? " --depth=1" : "; git checkout -q #{revision}")
|
92
|
-
#add keys, clone repo, go to specific revision, execute command
|
93
|
-
full_cmd = "cd #{run_dir};#{git_prefix}git clone -q #{user}@#{host}:#{repo}.git#{git_suffix}"
|
94
|
-
#put command in file, run ssh-agent bash on it
|
95
|
-
File.open(run_file_path,"w") {|f| f.print(full_cmd)}
|
96
|
-
run_cmd = "ssh-agent bash #{run_file_path}"
|
97
|
-
#run the command, it will return an exception if there are issues
|
98
|
-
run_cmd.bash(true)
|
99
|
-
repo_name = repo.split("/").last
|
100
|
-
repo_dir = "#{run_dir}/#{repo_name}"
|
101
|
-
return repo_dir
|
102
|
-
end
|
103
|
-
|
104
|
-
def Git.read_by_dataset_path(dst_path,user_name,*args)
|
105
|
-
domain,repo,revision,file_path = []
|
106
|
-
dst_path.split("/").ie do |path_nodes|
|
107
|
-
domain = path_nodes[0]
|
108
|
-
repo = path_nodes[1..2].join("/")
|
109
|
-
revision = path_nodes[3]
|
110
|
-
file_path = path_nodes[4..-1].join("/")
|
111
|
-
end
|
112
|
-
#slash in front of path
|
113
|
-
repo_dir = Git.pull(domain,repo,revision)
|
114
|
-
full_path = "#{repo_dir}/#{file_path}"
|
115
|
-
result = "cat #{full_path}".bash(true)
|
116
|
-
FileUtils.rm_r(repo_dir,:force=>true)
|
117
|
-
return result
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
module Mobilize
|
2
|
-
module Ssh
|
3
|
-
def self.config
|
4
|
-
Base.config('ssh')
|
5
|
-
end
|
6
|
-
|
7
|
-
def self.host(node)
|
8
|
-
self.config['nodes'][node]['host']
|
9
|
-
end
|
10
|
-
|
11
|
-
def self.gateway(node)
|
12
|
-
self.config['nodes'][node]['gateway']
|
13
|
-
end
|
14
|
-
|
15
|
-
def self.sudoers(node)
|
16
|
-
self.config['nodes'][node]['sudoers']
|
17
|
-
end
|
18
|
-
|
19
|
-
def self.su_all_users(node)
|
20
|
-
self.config['nodes'][node]['su_all_users']
|
21
|
-
end
|
22
|
-
|
23
|
-
def self.nodes
|
24
|
-
self.config['nodes'].keys
|
25
|
-
end
|
26
|
-
|
27
|
-
def self.default_node
|
28
|
-
self.nodes.first
|
29
|
-
end
|
30
|
-
|
31
|
-
#determine if current machine is on host domain, needs gateway if one is provided and it is not
|
32
|
-
def self.needs_gateway?(node)
|
33
|
-
host_domain_name = self.host(node)['name'].split(".")[-2..-1].join(".")
|
34
|
-
return true if self.gateway(node) and Socket.domain_name != host_domain_name
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
data/lib/samples/git.yml
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
---
|
2
|
-
development:
|
3
|
-
domains:
|
4
|
-
private:
|
5
|
-
host: github.private.com
|
6
|
-
user: git
|
7
|
-
repo_keys:
|
8
|
-
"repo_path_1": 'local/path/to/key1'
|
9
|
-
"repo_path_2": 'local/path/to/key1'
|
10
|
-
public:
|
11
|
-
host: github.com
|
12
|
-
user: git
|
13
|
-
test:
|
14
|
-
domains:
|
15
|
-
public:
|
16
|
-
host: github.com
|
17
|
-
user: git
|
18
|
-
private:
|
19
|
-
host: github.private.com
|
20
|
-
user: git
|
21
|
-
repo_keys:
|
22
|
-
"repo_path_1": 'local/path/to/key1'
|
23
|
-
"repo_path_2": 'local/path/to/key1'
|
24
|
-
production:
|
25
|
-
domains:
|
26
|
-
private:
|
27
|
-
host: github.private.com
|
28
|
-
user: git
|
29
|
-
repo_keys:
|
30
|
-
"repo_path_1": 'local/path/to/key1'
|
31
|
-
"repo_path_2": 'local/path/to/key1'
|
32
|
-
public:
|
33
|
-
host: github.com
|
34
|
-
user: git
|
data/test/fixtures/code2.sh
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
tail @file
|
@@ -1,37 +0,0 @@
|
|
1
|
-
---
|
2
|
-
- path: "Runner_mobilize(test)/jobs"
|
3
|
-
state: working
|
4
|
-
count: 1
|
5
|
-
confirmed_ats: []
|
6
|
-
- path: "Runner_mobilize(test)/jobs/ssh1/stage1"
|
7
|
-
state: working
|
8
|
-
count: 1
|
9
|
-
confirmed_ats: []
|
10
|
-
- path: "Runner_mobilize(test)/jobs/ssh1/stage2"
|
11
|
-
state: working
|
12
|
-
count: 1
|
13
|
-
confirmed_ats: []
|
14
|
-
- path: "Runner_mobilize(test)/jobs/ssh2/stage1"
|
15
|
-
state: working
|
16
|
-
count: 1
|
17
|
-
confirmed_ats: []
|
18
|
-
- path: "Runner_mobilize(test)/jobs/ssh2/stage2"
|
19
|
-
state: working
|
20
|
-
count: 1
|
21
|
-
confirmed_ats: []
|
22
|
-
- path: "Runner_mobilize(test)/jobs/ssh3/stage1"
|
23
|
-
state: working
|
24
|
-
count: 1
|
25
|
-
confirmed_ats: []
|
26
|
-
- path: "Runner_mobilize(test)/jobs/ssh3/stage2"
|
27
|
-
state: working
|
28
|
-
count: 1
|
29
|
-
confirmed_ats: []
|
30
|
-
- path: "Runner_mobilize(test)/jobs/ssh4/stage1"
|
31
|
-
state: working
|
32
|
-
count: 1
|
33
|
-
confirmed_ats: []
|
34
|
-
- path: "Runner_mobilize(test)/jobs/ssh4/stage2"
|
35
|
-
state: working
|
36
|
-
count: 1
|
37
|
-
confirmed_ats: []
|
@@ -1,25 +0,0 @@
|
|
1
|
-
---
|
2
|
-
- name: ssh1
|
3
|
-
active: true
|
4
|
-
trigger: once
|
5
|
-
status: ""
|
6
|
-
stage1: ssh.run node:"test_node", cmd:"ruby code.rb", user:"root", sources:["code.rb", "code.sh"]
|
7
|
-
stage2: gsheet.write source:"stage1", target:"ssh1.out"
|
8
|
-
- name: ssh2
|
9
|
-
active: true
|
10
|
-
trigger: "after ssh1"
|
11
|
-
status: ""
|
12
|
-
stage1: ssh.run cmd:"sh code2.sh", user:"root", sources:["code2.sh","test_node/var/log/syslog"], params:{file:"syslog"}
|
13
|
-
stage2: gsheet.write source:"stage1", target:"ssh2.out"
|
14
|
-
- name: ssh3
|
15
|
-
active: true
|
16
|
-
trigger: "after ssh2"
|
17
|
-
status: ""
|
18
|
-
stage1: ssh.run cmd:"echo '@test_param'", params:{test_param:"test param successful"}
|
19
|
-
stage2: gsheet.write source:"stage1", target:"ssh3.out"
|
20
|
-
- name: ssh4
|
21
|
-
active: true
|
22
|
-
trigger: "after ssh3"
|
23
|
-
status: ""
|
24
|
-
stage1: ssh.run node:"test_node", user:root, sources:["git://DeNA/mobilize-ssh/test/code.rb","git://DeNA/mobilize-ssh/test/code.sh"], cmd:"ruby code.rb"
|
25
|
-
stage2: gsheet.write source:stage1, target:"ssh4.out"
|
@@ -1,46 +0,0 @@
|
|
1
|
-
require 'test_helper'
|
2
|
-
describe "Mobilize" do
|
3
|
-
# enqueues 4 workers on Resque
|
4
|
-
it "runs integration test" do
|
5
|
-
|
6
|
-
puts "restart workers"
|
7
|
-
Mobilize::Jobtracker.restart_workers!
|
8
|
-
|
9
|
-
u = TestHelper.owner_user
|
10
|
-
r = u.runner
|
11
|
-
user_name = u.name
|
12
|
-
gdrive_slot = u.email
|
13
|
-
|
14
|
-
puts "add test code"
|
15
|
-
["code.rb","code.sh","code2.sh"].each do |fixture_name|
|
16
|
-
target_url = "gsheet://#{r.title}/#{fixture_name}"
|
17
|
-
TestHelper.write_fixture(fixture_name, target_url, 'replace')
|
18
|
-
end
|
19
|
-
|
20
|
-
puts "add/update jobs"
|
21
|
-
u.jobs.each{|j| j.delete}
|
22
|
-
jobs_fixture_name = "integration_jobs"
|
23
|
-
jobs_target_url = "gsheet://#{r.title}/jobs"
|
24
|
-
TestHelper.write_fixture(jobs_fixture_name, jobs_target_url, 'update')
|
25
|
-
|
26
|
-
puts "job rows added, force enqueue runner, wait for stages"
|
27
|
-
#wait for stages to complete
|
28
|
-
expected_fixture_name = "integration_expected"
|
29
|
-
Mobilize::Jobtracker.stop!
|
30
|
-
r.enqueue!
|
31
|
-
TestHelper.confirm_expected_jobs(expected_fixture_name)
|
32
|
-
|
33
|
-
puts "update job status and activity"
|
34
|
-
r.update_gsheet(gdrive_slot)
|
35
|
-
|
36
|
-
puts "jobtracker posted data to test sheets"
|
37
|
-
['ssh1.out','ssh2.out','ssh4.out'].each do |out_name|
|
38
|
-
url = "gsheet://#{r.title}/#{out_name}"
|
39
|
-
assert TestHelper.check_output(url, 'min_length' => 100) == true
|
40
|
-
end
|
41
|
-
|
42
|
-
#shorter
|
43
|
-
url = "gsheet://#{r.title}/ssh3.out"
|
44
|
-
assert TestHelper.check_output(url, 'min_length' => 3) == true
|
45
|
-
end
|
46
|
-
end
|