xp5k 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/xp5k/version.rb +1 -1
- data/lib/xp5k/xp.rb +191 -64
- metadata +15 -26
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 29af43dd9e56a5562a4f327a6f54be5591ef9f98
|
4
|
+
data.tar.gz: b2003bfefbaad6a7779dd396a9cc03ce85a4bc79
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d8b9f40bf89d6739869ca6633f813b8cb5ac0fc6f64dfd80a903de6cc0c91abb7111c5a7b0f3a7721f254270a25f59e59a888a540a52fe40bf8c9cc27804b1db
|
7
|
+
data.tar.gz: 521647dc1f924275374e140371ce75fa8e93dcfc4d5ece3f9141076ffa531da6d436aa99f3b9425d40685fb9e4b98ebea83b46b4fe1d56504a876e4a1a80028c
|
data/lib/xp5k/version.rb
CHANGED
data/lib/xp5k/xp.rb
CHANGED
@@ -19,6 +19,7 @@ module XP5K
|
|
19
19
|
@roles = []
|
20
20
|
@links_deployments = {"jobs" => {}, "roles" => {}}
|
21
21
|
@deployed_nodes = {"jobs" => {}, "roles" => {}}
|
22
|
+
@retries = options[:retries] || 3
|
22
23
|
@starttime = Time.now
|
23
24
|
@logger = options[:logger] || Logger.new(STDOUT)
|
24
25
|
|
@@ -44,41 +45,33 @@ module XP5K
|
|
44
45
|
self.todeploy << deployment_hash
|
45
46
|
end
|
46
47
|
|
47
|
-
def deploy
|
48
|
+
def deploy()
|
49
|
+
# prepare assigned_nodes, goals and retries value
|
48
50
|
self.todeploy.each do |x|
|
49
|
-
x[:
|
50
|
-
# Get assigned resources to deploy
|
51
|
+
x[:assigned_nodes] = []
|
51
52
|
x[:jobs].each do |jobname|
|
52
53
|
job = self.job_with_name(jobname)
|
53
|
-
|
54
|
+
self.deployed_nodes["jobs"][jobname] = []
|
55
|
+
x[:assigned_nodes] += job["assigned_nodes"]
|
54
56
|
end
|
55
57
|
x[:roles].each do |rolename|
|
56
|
-
|
58
|
+
role = role_with_name(rolename)
|
59
|
+
self.deployed_nodes["roles"][rolename] = []
|
60
|
+
x[:assigned_nodes] += role.servers
|
57
61
|
end
|
58
|
-
|
59
|
-
x
|
60
|
-
|
61
|
-
x
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
update_links_deployments(deployment["uid"], x)
|
66
|
-
end
|
67
|
-
logger.info "Waiting for all the deployments to be terminated..."
|
68
|
-
finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
|
69
|
-
while (!finished)
|
70
|
-
sleep 10
|
71
|
-
print "."
|
72
|
-
self.deployments.each do |deployment|
|
73
|
-
deployment.reload
|
74
|
-
end
|
75
|
-
finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
|
62
|
+
# initially all nodes have to be deployed
|
63
|
+
x[:nodes] = x[:assigned_nodes]
|
64
|
+
# set goal
|
65
|
+
x[:goal] = set_goal(x[:goal], x[:assigned_nodes].length)
|
66
|
+
# set retries
|
67
|
+
x[:retry] ||= false
|
68
|
+
x[:retries] ||= x[:retry]?@retries:1
|
76
69
|
end
|
77
|
-
|
78
|
-
|
79
|
-
print(" [#{green("OK")}]\n")
|
70
|
+
internal_deploy(@retries)
|
71
|
+
print_deploy_summary
|
80
72
|
end
|
81
73
|
|
74
|
+
|
82
75
|
def define_job(job_hash)
|
83
76
|
self.jobs2submit << job_hash
|
84
77
|
|
@@ -86,7 +79,7 @@ module XP5K
|
|
86
79
|
datas = JSON.parse(File.read(".xp_cache"))
|
87
80
|
uid = datas["jobs"].select { |x| x["name"] == job_hash[:name] }.first["uid"]
|
88
81
|
unless uid.nil?
|
89
|
-
job = @connection.root.sites[job_hash[:site].to_sym].jobs["#{uid}".to_sym]
|
82
|
+
job = @connection.root.sites[job_hash[:site].to_sym].jobs(:query => { :user => @connection.config.options[:username] })["#{uid}".to_sym]
|
90
83
|
if (not job.nil? or job["state"] == "running")
|
91
84
|
j = job.reload
|
92
85
|
self.jobs << j
|
@@ -94,7 +87,7 @@ module XP5K
|
|
94
87
|
end
|
95
88
|
end
|
96
89
|
# reload last deployed nodes
|
97
|
-
self.deployed_nodes = datas["deployed_nodes"]
|
90
|
+
self.deployed_nodes = datas["deployed_nodes"]
|
98
91
|
end
|
99
92
|
|
100
93
|
end
|
@@ -104,19 +97,35 @@ module XP5K
|
|
104
97
|
job = self.job_with_name(job2submit[:name])
|
105
98
|
if job.nil?
|
106
99
|
job = @connection.root.sites[job2submit[:site].to_sym].jobs.submit(job2submit)
|
107
|
-
#self.jobs << { :uid => job.properties['uid'], :name => job.properties['name'] }
|
108
100
|
update_cache
|
109
|
-
logger.info "
|
110
|
-
|
111
|
-
print(".")
|
112
|
-
sleep 3
|
113
|
-
end
|
114
|
-
self.jobs << job
|
115
|
-
create_roles(job, job2submit) unless job2submit[:roles].nil?
|
116
|
-
print(" [#{green("OK")}]\n")
|
101
|
+
logger.info "Job \"#{job['name']}\" submitted with id ##{job['uid']} at #{job2submit[:site]}"
|
102
|
+
self.jobs << job
|
117
103
|
else
|
118
|
-
logger.info "Job #{job["name"]} already submitted ##{job["uid"]}"
|
104
|
+
logger.info "Job \"#{job["name"]}\" already submitted ##{job["uid"]}"
|
105
|
+
end
|
106
|
+
end
|
107
|
+
update_cache()
|
108
|
+
end
|
109
|
+
|
110
|
+
def wait_for_jobs
|
111
|
+
logger.info "Waiting for running state"
|
112
|
+
ready = false
|
113
|
+
jobs_status = []
|
114
|
+
until ready
|
115
|
+
self.jobs.each.with_index do |job, id|
|
116
|
+
jobs_status[id] = job.reload["state"]
|
117
|
+
case jobs_status[id]
|
118
|
+
when "running"
|
119
|
+
create_roles(job, jobs2submit[id]) unless jobs2submit[id][:roles].nil?
|
120
|
+
logger.info "Job #{job['uid']} is running"
|
121
|
+
when /terminated|error/
|
122
|
+
logger.info "Job #{job['uid']} is terminated"
|
123
|
+
else
|
124
|
+
logger.info "Job #{job['uid']} will be scheduled at #{Time.at(job['scheduled_at'].to_i).to_datetime}"
|
125
|
+
end
|
119
126
|
end
|
127
|
+
ready = true if jobs_status.uniq == ["running"]
|
128
|
+
sleep 3
|
120
129
|
end
|
121
130
|
update_cache()
|
122
131
|
end
|
@@ -133,6 +142,7 @@ module XP5K
|
|
133
142
|
role.servers = available_nodes[0..(role.size - 1)]
|
134
143
|
available_nodes -= role.servers
|
135
144
|
role.jobid = job['uid']
|
145
|
+
next if not self.roles.select { |x| x.name == role.name }.empty?
|
136
146
|
self.roles << role
|
137
147
|
end
|
138
148
|
end
|
@@ -146,26 +156,24 @@ module XP5K
|
|
146
156
|
end
|
147
157
|
|
148
158
|
def get_deployed_nodes(job_or_role_name)
|
149
|
-
if deployed_nodes["jobs"].has_key?(job_or_role_name)
|
150
|
-
deployed_nodes["jobs"][job_or_role_name]
|
159
|
+
if self.deployed_nodes["jobs"].has_key?(job_or_role_name)
|
160
|
+
return self.deployed_nodes["jobs"][job_or_role_name]
|
151
161
|
end
|
152
|
-
if deployed_nodes["roles"].has_key?(job_or_role_name)
|
153
|
-
deployed_nodes["roles"][job_or_role_name]
|
162
|
+
if self.deployed_nodes["roles"].has_key?(job_or_role_name)
|
163
|
+
return self.deployed_nodes["roles"][job_or_role_name]
|
154
164
|
end
|
155
165
|
end
|
156
166
|
|
157
167
|
def status
|
158
168
|
self.jobs.each do |job|
|
159
|
-
logger.info "Job #{job["name"]} ##{job["uid"]} status : #{job["state"]}"
|
169
|
+
logger.info "Job \"#{job["name"]}\" ##{job["uid"]} status : #{job["state"]}"
|
160
170
|
end
|
161
171
|
end
|
162
172
|
|
163
173
|
def clean
|
164
174
|
self.jobs.each do |job|
|
165
|
-
if job
|
166
|
-
|
167
|
-
logger.info "Job ##{job["uid"]} deleted !"
|
168
|
-
end
|
175
|
+
job.delete if (job['state'] =~ /running|waiting/)
|
176
|
+
logger.info "Job ##{job["uid"]} deleted !"
|
169
177
|
end
|
170
178
|
FileUtils.rm(".xp_cache")
|
171
179
|
end
|
@@ -175,48 +183,56 @@ module XP5K
|
|
175
183
|
def logger
|
176
184
|
@logger
|
177
185
|
end
|
178
|
-
|
186
|
+
|
179
187
|
def update_links_deployments (duid, todeploy)
|
180
188
|
unless todeploy[:jobs].nil?
|
181
189
|
todeploy[:jobs].each do |job|
|
182
|
-
@links_deployments["jobs"][job]
|
190
|
+
@links_deployments["jobs"][job] ||= []
|
191
|
+
@links_deployments["jobs"][job] << duid
|
183
192
|
end
|
184
193
|
end
|
185
194
|
|
186
195
|
unless todeploy[:roles].nil?
|
187
196
|
todeploy[:roles].each do |role|
|
188
|
-
@links_deployments["roles"][role]
|
197
|
+
@links_deployments["roles"][role] ||= []
|
198
|
+
@links_deployments["roles"][role] << duid
|
189
199
|
end
|
190
200
|
end
|
191
201
|
end
|
192
202
|
|
193
203
|
def update_deployed_nodes
|
194
|
-
self.
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
self.links_deployments["roles"].select{|k,v| v == duid }.keys.each do |role_name|
|
201
|
-
role = role_with_name(role_name)
|
202
|
-
deployed_nodes["roles"][role.name] = intersect_nodes_role(role, deployment)
|
204
|
+
self.links_deployments["jobs"].each do |jobname,v|
|
205
|
+
job = job_with_name(jobname)
|
206
|
+
deployed_nodes["jobs"][jobname]=[]
|
207
|
+
v.each do |duid|
|
208
|
+
deployment = self.deployments.select{ |d| d["uid"] == duid}.first
|
209
|
+
deployed_nodes["jobs"][jobname] += intersect_nodes_job(job, deployment)
|
203
210
|
end
|
211
|
+
end
|
204
212
|
|
213
|
+
self.links_deployments["roles"].each do |rolename,v|
|
214
|
+
role = role_with_name(rolename)
|
215
|
+
deployed_nodes["roles"][rolename]=[]
|
216
|
+
v.each do |duid|
|
217
|
+
deployment = self.deployments.select{ |d| d["uid"] == duid}.first
|
218
|
+
deployed_nodes["roles"][rolename] += intersect_nodes_role(role, deployment)
|
219
|
+
end
|
205
220
|
end
|
221
|
+
|
206
222
|
end
|
207
|
-
|
223
|
+
|
208
224
|
def intersect_nodes_job (job, deployment)
|
209
225
|
nodes_deployed = deployment["result"].select{ |k,v| v["state"]=='OK'}.keys
|
210
|
-
job["assigned_nodes"] & nodes_deployed
|
226
|
+
return job["assigned_nodes"] & nodes_deployed
|
211
227
|
end
|
212
228
|
|
213
229
|
def intersect_nodes_role (role, deployment)
|
214
230
|
nodes_deployed = deployment["result"].select{ |k,v| v["state"]=='OK'}.keys
|
215
|
-
role.servers & nodes_deployed
|
231
|
+
return role.servers & nodes_deployed
|
216
232
|
end
|
217
|
-
|
233
|
+
|
218
234
|
def update_cache
|
219
|
-
cache = {
|
235
|
+
cache = {
|
220
236
|
:jobs => self.jobs.collect { |x| x.properties },
|
221
237
|
:roles => self.roles.map{ |x| { :name => x.name, :size => x.size, :servers => x.servers }},
|
222
238
|
:deployed_nodes => self.deployed_nodes,
|
@@ -227,5 +243,116 @@ module XP5K
|
|
227
243
|
end
|
228
244
|
end
|
229
245
|
|
246
|
+
def internal_deploy(n)
|
247
|
+
|
248
|
+
if (n<=0)
|
249
|
+
return
|
250
|
+
end
|
251
|
+
|
252
|
+
# Fill with nodes to deployed
|
253
|
+
self.todeploy.each do |x|
|
254
|
+
x[:nodes] = x[:assigned_nodes]
|
255
|
+
x[:jobs].each do |jobname|
|
256
|
+
x[:nodes] = x[:nodes] - self.deployed_nodes["jobs"][jobname]
|
257
|
+
end
|
258
|
+
x[:roles].each do |rolename|
|
259
|
+
x[:nodes] = x[:nodes] - self.deployed_nodes["roles"][rolename]
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
# Clean todeploy
|
264
|
+
self.todeploy.delete_if{ |x|
|
265
|
+
x[:nodes].empty? ||
|
266
|
+
(x[:goal] >= 0) && ( x[:nodes].length < ((1-x[:goal])*(x[:assigned_nodes].length ))) ||
|
267
|
+
x[:retries] <= 0
|
268
|
+
}
|
269
|
+
|
270
|
+
if self.todeploy.empty?
|
271
|
+
return
|
272
|
+
end
|
273
|
+
|
274
|
+
if (n<@retries)
|
275
|
+
logger.info "Redeployment of undeployed nodes"
|
276
|
+
end
|
277
|
+
|
278
|
+
# Launch deployments
|
279
|
+
self.todeploy.each do |y|
|
280
|
+
x = y.clone
|
281
|
+
site = x[:site]
|
282
|
+
x.delete(:site)
|
283
|
+
x.delete(:roles)
|
284
|
+
x.delete(:jobs)
|
285
|
+
x.delete(:assigned_nodes)
|
286
|
+
x.delete(:goal)
|
287
|
+
x.delete(:retry)
|
288
|
+
x.delete(:retries)
|
289
|
+
|
290
|
+
deployment = @connection.root.sites[site.to_sym].deployments.submit(x)
|
291
|
+
self.deployments << deployment
|
292
|
+
# Update links_deployments
|
293
|
+
update_links_deployments(deployment["uid"], y)
|
294
|
+
y[:retries] = y[:retries] - 1
|
295
|
+
end
|
296
|
+
|
297
|
+
logger.info "Waiting for all the deployments to be terminated..."
|
298
|
+
finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
|
299
|
+
while (!finished)
|
300
|
+
sleep 10
|
301
|
+
print "."
|
302
|
+
self.deployments.each do |deployment|
|
303
|
+
deployment.reload
|
304
|
+
end
|
305
|
+
finished = self.deployments.reduce(true){ |acc, d| acc && d["status"]!='processing'}
|
306
|
+
end
|
307
|
+
print(" [#{green("OK")}]\n")
|
308
|
+
|
309
|
+
# Update deployed nodes
|
310
|
+
update_deployed_nodes()
|
311
|
+
update_cache()
|
312
|
+
|
313
|
+
internal_deploy(n - 1)
|
314
|
+
|
315
|
+
end
|
316
|
+
|
317
|
+
def set_goal(goal, total)
|
318
|
+
if goal.nil?
|
319
|
+
return -1.0
|
320
|
+
end
|
321
|
+
|
322
|
+
if goal.to_s.include? "%"
|
323
|
+
return goal.to_f/100
|
324
|
+
elsif goal.to_f < 1
|
325
|
+
return goal.to_f
|
326
|
+
elsif goal.to_f == 1.0
|
327
|
+
return goal.to_f/total
|
328
|
+
else
|
329
|
+
return goal.to_f/total
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
def print_deploy_summary
|
334
|
+
puts "Summary of the deployment"
|
335
|
+
puts "-" * 60
|
336
|
+
printf "%+20s", "Name"
|
337
|
+
printf "%+20s", "Deployed"
|
338
|
+
printf "%+20s", "Undeployed"
|
339
|
+
puts "\n"
|
340
|
+
puts "-" * 60
|
341
|
+
|
342
|
+
self.deployed_nodes["jobs"].each do |jobname, deployed_nodes|
|
343
|
+
puts "\n"
|
344
|
+
printf "%+20s",jobname
|
345
|
+
printf "%20d", deployed_nodes.length
|
346
|
+
printf "%20d", job_with_name(jobname)["assigned_nodes"].length - deployed_nodes.length
|
347
|
+
puts "\n"
|
348
|
+
end
|
349
|
+
|
350
|
+
self.deployed_nodes["roles"].each do |rolename, deployed_nodes|
|
351
|
+
printf "%+20s",rolename
|
352
|
+
printf "%20d", deployed_nodes.length
|
353
|
+
printf "%20d", role_with_name(rolename).servers.length - deployed_nodes.length
|
354
|
+
puts "\n"
|
355
|
+
end
|
356
|
+
end
|
230
357
|
end
|
231
358
|
end
|
metadata
CHANGED
@@ -1,78 +1,69 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: xp5k
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.6
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Pascal Morillon
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2014-06-30 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: restfully
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - '>'
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: 1.0.0
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - '>'
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: 1.0.0
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: term-ansicolor
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - '>='
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: 1.0.7
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - '>='
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: 1.0.7
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: json
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - '>='
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: 1.5.1
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - '>='
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: 1.5.1
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: bundler
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - '>='
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: 1.0.0
|
70
62
|
type: :development
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - '>='
|
76
67
|
- !ruby/object:Gem::Version
|
77
68
|
version: 1.0.0
|
78
69
|
description: A small Grid'5000 helper to submit jobs and deploy environments via REST
|
@@ -98,28 +89,26 @@ files:
|
|
98
89
|
homepage: https://github.com/pmorillon/xp5k
|
99
90
|
licenses:
|
100
91
|
- MIT
|
92
|
+
metadata: {}
|
101
93
|
post_install_message:
|
102
94
|
rdoc_options: []
|
103
95
|
require_paths:
|
104
96
|
- lib
|
105
97
|
required_ruby_version: !ruby/object:Gem::Requirement
|
106
|
-
none: false
|
107
98
|
requirements:
|
108
|
-
- -
|
99
|
+
- - '>='
|
109
100
|
- !ruby/object:Gem::Version
|
110
101
|
version: '0'
|
111
102
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
112
|
-
none: false
|
113
103
|
requirements:
|
114
|
-
- -
|
104
|
+
- - '>='
|
115
105
|
- !ruby/object:Gem::Version
|
116
106
|
version: 1.3.6
|
117
107
|
requirements: []
|
118
108
|
rubyforge_project:
|
119
|
-
rubygems_version:
|
109
|
+
rubygems_version: 2.0.14
|
120
110
|
signing_key:
|
121
|
-
specification_version:
|
111
|
+
specification_version: 4
|
122
112
|
summary: A small Grid'5000 helper to submit jobs and deploy environments via REST
|
123
113
|
API
|
124
114
|
test_files: []
|
125
|
-
has_rdoc:
|