ruby-cute 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,38 @@
1
+ # Extends the class string for supporting timespan formats
2
+ class String
3
+
4
+ def to_secs
5
+
6
+ return Infinity if [ 'always', 'forever', 'infinitely' ].include?(self.to_s)
7
+ parts = self.split(':').map { |x| x.to_i rescue nil }
8
+ if parts.all? && [ 2, 3 ].include?(parts.length)
9
+ secs = parts.zip([ 3600, 60, 1 ]).map { |x, y| x * y }.reduce(:+)
10
+ return secs
11
+ end
12
+ m = /^(\d+|\d+\.\d*)\s*(\w*)?$/.match(self)
13
+ num, unit = m.captures
14
+ mul = case unit
15
+ when '' then 1
16
+ when 's' then 1
17
+ when 'm' then 60
18
+ when 'h' then 60 * 60
19
+ when 'd' then 24 * 60 * 60
20
+ else nil
21
+ end
22
+ raise "Unknown timespan unit: '#{unit}' in #{self}" if mul.nil?
23
+ return num.to_f * mul
24
+ end
25
+
26
+ def to_time
27
+ secs = self.to_secs.to_i
28
+ minutes = secs / 60; secs %= 60
29
+ hours = minutes / 60; minutes %= 60
30
+ minutes += 1 if secs > 0
31
+ return '%.02d:%.02d' % [ hours, minutes ]
32
+ end
33
+
34
+ def is_i?
35
+ /\A[-+]?\d+\z/ === self
36
+ end
37
+
38
+ end
@@ -0,0 +1,1190 @@
1
+ require 'restclient'
2
+ require 'yaml'
3
+ require 'json'
4
+ require 'ipaddress'
5
+ require 'uri'
6
+
7
+ module Cute
8
+ module G5K
9
+
10
+ # = {Cute::G5K} exceptions
11
+ #
12
+ # The generated exceptions are divided in 5 groups:
13
+ #
14
+ # - {Cute::G5K::BadRequest BadRequest} it means that the syntax you passed to some {Cute::G5K::API G5K::API} method is not correct from
15
+ # the Grid'5000 services point of view.
16
+ # - {Cute::G5K::RequestFailed RequestFailed} it means that there is a server problem or there is nothing the user can do to solve the problem.
17
+ # - {Cute::G5K::NotFound} it means that the requested resources do not exist.
18
+ # - {Cute::G5K::Unauthorized} it means that there is an authentication problem.
19
+ # - {Cute::G5K::EventTimeout} this exception is triggered by the methods that wait for events such as:
20
+ # job submission and environment deployment.
21
+ class Error < Exception
22
+ attr_accessor :orig # Original exception
23
+
24
+ def initialize(message = nil, object = nil)
25
+ super(message)
26
+ self.orig = object
27
+ end
28
+
29
+ def method_missing(method)
30
+ return orig.send(method)
31
+ end
32
+ end
33
+
34
+ # It wraps the http response 400 that corresponds to a bad request.
35
+ # When using the {Cute::G5K::API#reserve reserve} or {Cute::G5K::API#reserve deploy} methods this could mean:
36
+ # a bad syntax in the request, not valid properties in the request,
37
+ # not enough resources to supply the request, non existing environment, etc.
38
+ #
39
+ # = Example
40
+ #
41
+ # You can handle this exception and decide what to do with your experiment.
42
+ # In the example below, we iterate over all sites until a site has resources with the property 'ib20g' set to 'YES'.
43
+ #
44
+ # require 'cute'
45
+ #
46
+ # g5k = Cute::G5K::API.new()
47
+ #
48
+ # sites = g5k.site_uids
49
+ #
50
+ # sites.each do |site|
51
+ #
52
+ # begin
53
+ # job = g5k.reserve(:site => site, :resources => "{ib20g='YES'}/nodes=2/core=1",:walltime => '00:30:00', :keys => "~/my_ssh_jobkey" )
54
+ # rescue Cute::G5K::BadRequest
55
+ # puts "Resource not available in this site, trying with another one"
56
+ # end
57
+ #
58
+ # end
59
+ class BadRequest < Error
60
+ end
61
+
62
+ # It wraps all Restclient exceptions with http codes: 403, 405,406, 412, 415, 500, 502, 503 and 504.
63
+ class RequestFailed < Error
64
+ end
65
+
66
+ # It wraps the exceptions generated by Timeout::Error
67
+ class EventTimeout < Error
68
+ end
69
+
70
+ # It wraps the Restclient exception 404
71
+ class NotFound < Error
72
+ end
73
+
74
+ # It wraps the Restclient exception RestClient::Unauthorized
75
+ class Unauthorized < Error
76
+ end
77
+
78
+
79
+ # @api private
80
+ class G5KArray < Array
81
+
82
+ def uids
83
+ return self.map { |it| it['uid'] }
84
+ end
85
+
86
+ def rel_self
87
+ return rel('self')
88
+ end
89
+
90
+ def rel(r)
91
+ return self['links'].detect { |x| x['rel'] == r }['href']
92
+ end
93
+
94
+ end
95
+
96
+ # Provides an abstraction for handling G5K responses.
97
+ # @api private
98
+ # @see https://api.grid5000.fr/doc/3.0/reference/grid5000-media-types.html
99
+ # When this structure is used to describe jobs, it is expected to have the
100
+ # following fields which depend on the version of the API.
101
+ # {"uid"=>604692,
102
+ # "user_uid"=>"name",
103
+ # "user"=>"name",
104
+ # "walltime"=>3600,
105
+ # "queue"=>"default",
106
+ # "state"=>"running",
107
+ # "project"=>"default",
108
+ # "name"=>"rubyCute job",
109
+ # "types"=>["deploy"],
110
+ # "mode"=>"PASSIVE",
111
+ # "command"=>"./oarapi.subscript.ZzvnM",
112
+ # "submitted_at"=>1423575384,
113
+ # "scheduled_at"=>1423575386,
114
+ # "started_at"=>1423575386,
115
+ # "message"=>"FIFO scheduling OK",
116
+ # "properties"=>"(deploy = 'YES') AND maintenance = 'NO'",
117
+ # "directory"=>"/home/name",
118
+ # "events"=>[],
119
+ # "links"=>[{"rel"=>"self", "href"=>"/sid/sites/nancy/jobs/604692", "type"=>"application/vnd.grid5000.item+json"},
120
+ # {"rel"=>"parent", "href"=>"/sid/sites/nancy", "type"=>"application/vnd.grid5000.item+json"}],
121
+ # "resources_by_type"=>
122
+ # {"cores"=>
123
+ # ["griffon-8.nancy.grid5000.fr",
124
+ # "griffon-8.nancy.grid5000.fr",
125
+ # "griffon-8.nancy.grid5000.fr",
126
+ # "griffon-8.nancy.grid5000.fr",
127
+ # "griffon-9.nancy.grid5000.fr",
128
+ # "griffon-9.nancy.grid5000.fr",
129
+ # "griffon-9.nancy.grid5000.fr",
130
+ # "griffon-9.nancy.grid5000.fr",
131
+ # "griffon-77.nancy.grid5000.fr",
132
+ # "griffon-77.nancy.grid5000.fr",
133
+ # "griffon-77.nancy.grid5000.fr",
134
+ # "griffon-77.nancy.grid5000.fr",
135
+ # "vlans"=>["5"]},
136
+ # "assigned_nodes"=>["griffon-8.nancy.grid5000.fr", "griffon-9.nancy.grid5000.fr", "griffon-77.nancy.grid5000.fr"],
137
+ # "deploy"=>
138
+ # {"created_at"=>1423575401,
139
+ # "environment"=>"http://public.sophia.grid5000.fr/~nniclausse/openmx.dsc",
140
+ # "key"=>"https://api.grid5000.fr/sid/sites/nancy/files/cruizsanabria-key-84f3f1dbb1279bc1bddcd618e26c960307d653c5",
141
+ # "nodes"=>["griffon-8.nancy.grid5000.fr", "griffon-9.nancy.grid5000.fr", "griffon-77.nancy.grid5000.fr"],
142
+ # "site_uid"=>"nancy",
143
+ # "status"=>"processing",
144
+ # "uid"=>"D-751096de-0c33-461a-9d27-56be1b2dd980",
145
+ # "updated_at"=>1423575401,
146
+ # "user_uid"=>"cruizsanabria",
147
+ # "vlan"=>5,
148
+ # "links"=>
149
+ # [{"rel"=>"self", "href"=>"/sid/sites/nancy/deployments/D-751096de-0c33-461a-9d27-56be1b2dd980", "type"=>"application/vnd.grid5000.item+json"},
150
+ class G5KJSON < Hash
151
+
152
+ def items
153
+ return self['items']
154
+ end
155
+
156
+ def nodes
157
+ return self['nodes']
158
+ end
159
+
160
+ def resources
161
+ return self['resources_by_type'].nil?? Hash.new : self['resources_by_type']
162
+ end
163
+
164
+ def rel(r)
165
+ return self['links'].detect { |x| x['rel'] == r }['href']
166
+ end
167
+
168
+ def uid
169
+ return self['uid']
170
+ end
171
+
172
+ def rel_self
173
+ return rel('self')
174
+ end
175
+
176
+ def rel_parent
177
+ return rel('parent')
178
+ end
179
+
180
+ def refresh(g5k)
181
+ return g5k.get_json(rel_self)
182
+ end
183
+
184
+ def self.parse(s)
185
+ return JSON.parse(s, :object_class => G5KJSON, :array_class => G5KArray)
186
+ end
187
+
188
+ end
189
+
190
+ # Manages the low level operations for communicating with the REST API.
191
+ # @api private
192
+ class G5KRest
193
+
194
+ attr_reader :user
195
+ # Initializes a REST connection
196
+ # @param uri [String] resource identifier which normally is the URL of the Rest API
197
+ # @param user [String] user if authentication is needed
198
+ # @param pass [String] password if authentication is needed
199
+ # @param on_error [Symbol] option to deactivate the {Cute::G5K::RequestFailed RequestFailed} exceptions
200
+ def initialize(uri,api_version,user,pass,on_error)
201
+ @user = user
202
+ @pass = pass
203
+ @api_version = api_version.nil? ? "sid" : api_version
204
+ if (user.nil? or pass.nil?)
205
+ @endpoint = uri # Inside Grid'5000
206
+ else
207
+ user_escaped = CGI.escape(user)
208
+ pass_escaped = CGI.escape(pass)
209
+ @endpoint = "https://#{user_escaped}:#{pass_escaped}@#{uri.split("https://")[1]}"
210
+ end
211
+
212
+ machine =`uname -ov`.chop
213
+ @user_agent = "ruby-cute/#{VERSION} (#{machine}) Ruby #{RUBY_VERSION}"
214
+ @api = RestClient::Resource.new(@endpoint, :timeout => 30)
215
+ @on_error = on_error
216
+ test_connection
217
+ end
218
+
219
+ # Returns a resource object
220
+ # @param path [String] this complements the URI to address to a specific resource
221
+ def resource(path)
222
+ path = path[1..-1] if path.start_with?('/')
223
+ return @api[path]
224
+ end
225
+
226
+ # @return [Hash] the HTTP response
227
+ # @param path [String] this complements the URI to address to a specific resource
228
+ def get_json(path)
229
+
230
+ begin
231
+ r = resource(path).get(:content_type => "application/json",
232
+ :user_agent => @user_agent)
233
+ rescue => e
234
+ handle_exception(e)
235
+ end
236
+ return G5KJSON.parse(r)
237
+ end
238
+
239
+ # Creates a resource on the server
240
+ # @param path [String] this complements the URI to address to a specific resource
241
+ # @param json [Hash] contains the characteristics of the resources to be created.
242
+ def post_json(path, json)
243
+
244
+ begin
245
+ r = resource(path).post(json.to_json,
246
+ :content_type => "application/json",
247
+ :accept => "application/json",
248
+ :user_agent => @user_agent)
249
+ rescue => e
250
+ handle_exception(e)
251
+ end
252
+ return G5KJSON.parse(r)
253
+ end
254
+
255
+ # Deletes a resource on the server
256
+ # @param path [String] this complements the URI to address to a specific resource
257
+ def delete_json(path)
258
+ begin
259
+ return resource(path).delete()
260
+ rescue RestClient::InternalServerError => e
261
+ raise RequestFailed.new("Service internal error", e)
262
+ end
263
+ end
264
+
265
+ # @return the parent link
266
+ def follow_parent(obj)
267
+ get_json(obj.rel_parent)
268
+ end
269
+
270
+ private
271
+
272
+ # Tests the connection and raises an error in case of a problem
273
+ def test_connection
274
+ begin
275
+ return get_json("/#{@api_version}/")
276
+ rescue Cute::G5K::Unauthorized
277
+ raise "Your Grid'5000 credentials are not recognized"
278
+ end
279
+ end
280
+
281
+ # Issues a Cute::G5K exception according to the http status code
282
+ def handle_exception(e)
283
+ case e.http_code
284
+ when 500
285
+ # This part deals with bug: https://intranet.grid5000.fr/bugzilla/show_bug.cgi?id=5912
286
+ # Grid'5000 returns 500 error code even though the error was generated by a bad request
287
+ http_body = JSON.parse("{#{e.http_body.split("\n").select{ |x| x.include?("code")}.first}}")
288
+ if http_body["code"] == 400
289
+ raise BadRequest.new("Bad request", e)
290
+ else
291
+ raise RequestFailed.new("Service internal error", e)
292
+ end
293
+ when 400
294
+ raise BadRequest.new("Bad request", e)
295
+ when 404
296
+ raise NotFound.new("Resource not found", e)
297
+ when 401
298
+ raise Unauthorized.new("Authentication problem",e)
299
+ else
300
+ if @on_error == :ignore
301
+ return nil
302
+ else
303
+ raise RequestFailed.new("Service internal error", e)
304
+ end
305
+ end
306
+ end
307
+
308
+ end
309
+
310
+ # This class helps you to access Grid'5000 REST API.
311
+ # Thus, the most common actions such as reservation of nodes and deployment can be easily scripted.
312
+ # To simplify the use of the module, it is better to create a file with the following information:
313
+ #
314
+ # $ cat > ~/.grid5000_api.yml << EOF
315
+ # $ uri: https://api.grid5000.fr/
316
+ # $ username: user
317
+ # $ password: **********
318
+ # $ version: sid
319
+ # $ EOF
320
+ #
321
+ # The *username* and *password* are not necessary if you are using the module from inside Grid'5000.
322
+ # You can take a look at the {Cute::G5K::API#initialize G5K::API constructor} to see more details for
323
+ # this configuration.
324
+ #
325
+ # = Getting started
326
+ #
327
+ # As already said, the goal of {Cute::G5K::API G5K::API} class is to present a high level abstraction to manage the most common activities
328
+ # in Grid'5000 such as: the reservation of resources and the deployment of environments.
329
+ # Consequently, these activities can be easily scripted using Ruby.
330
+ # The advantage of this is that you can use all Ruby constructs (e.g., loops, conditionals, blocks, iterators, etc) to script your experiments.
331
+ # In the presence of error, {Cute::G5K::API G5K::API} raises exceptions (see {Cute::G5K::Error G5K exceptions}),
332
+ # that you can handle to decide the workflow of your experiment
333
+ # (see {Cute::G5K::API#wait_for_deploy wait_for_deploy} and {Cute::G5K::API#wait_for_deploy wait_for_job}).
334
+ # Let's show how {Cute::G5K::API G5K::API} is used through an example, suppose we want to reserve 3 nodes in Nancy site for 1 hour.
335
+ # In order to do that we would write something like this:
336
+ #
337
+ # require 'cute'
338
+ #
339
+ # g5k = Cute::G5K::API.new()
340
+ #
341
+ # job = g5k.reserve(:nodes => 3, :site => 'nancy', :walltime => '01:00:00')
342
+ #
343
+ # puts "Assigned nodes : #{job['assigned_nodes']}"
344
+ #
345
+ # If that is all you want to do, you can write that into a file, let's say *example.rb* and execute it using the Ruby interpreter.
346
+ #
347
+ # $ ruby example.rb
348
+ #
349
+ # The execution will block until you got the reservation. Then, you can interact with the nodes you reserved the way you used to or
350
+ # add more code to the previous script for controlling your experiment with Ruby-Cute as shown in this
351
+ # {http://www.rubydoc.info/github/ruby-cute/ruby-cute/master/file/examples/g5k_exp_virt.rb example}.
352
+ # We have just used the method {Cute::G5K::API#reserve reserve} that allow us to reserve resources in Grid'5000.
353
+ # This method can be used to reserve resources in deployment mode and deploy our own software environment on them using
354
+ # {http://kadeploy3.gforge.inria.fr/ Kadeploy}. For this we use the option *:env* of the {Cute::G5K::API#reserve reserve} method.
355
+ # Therefore, it will first reserve the resources and then deploy the specified environment.
356
+ # The method {Cute::G5K::API#reserve reserve} will block until the deployment is done.
357
+ # The following Ruby script illustrates all we have just said.
358
+ #
359
+ # require 'cute'
360
+ #
361
+ # g5k = Cute::G5K::API.new()
362
+ #
363
+ # job = g5k.reserve(:nodes => 1, :site => 'grenoble', :walltime => '00:40:00', :env => 'wheezy-x64-base')
364
+ #
365
+ # puts "Assigned nodes : #{job['assigned_nodes']}"
366
+ #
367
+ # Your public ssh key located in ~/.ssh will be copied by default on the deployed machines,
368
+ # you can specify another path for your keys with the option *:keys*.
369
+ # In order to deploy your own environment, you have to put the tar file that contains the operating system you want to deploy and
370
+ # the environment description file, under the public directory of a given site.
371
+ # *VLANS* are supported by adding the parameter :vlan => type where type can be: *:routed*, *:local*, *:global*.
372
+ # The following example, reserves 10 nodes in the Lille site, starts the deployment of a custom environment over the nodes
373
+ # and puts the nodes under a routed VLAN. We used the method {Cute::G5K::API#get_vlan_nodes get_vlan_nodes} to get the
374
+ # new hostnames assigned to your nodes.
375
+ #
376
+ # require 'cute'
377
+ #
378
+ # g5k = Cute::G5K::API.new()
379
+ #
380
+ # job = g5k.reserve(:site => "lille", :nodes => 10,
381
+ # :env => 'https://public.lyon.grid5000.fr/~user/debian_custom_img.yaml',
382
+ # :vlan => :routed, :keys => "~/my_ssh_key")
383
+ #
384
+ #
385
+ # puts "Log in into the nodes using the following hostnames: #{g5k.get_vlan_nodes(job)}"
386
+ #
387
+ # If you do not want that the method {Cute::G5K::API#reserve reserve} perform the deployment for you, you have to use the option :type => :deploy.
388
+ # This can be useful when deploying different environments in your reserved nodes. For example deploying the environments for a small HPC cluster.
389
+ # You have to use the method {Cute::G5K::API#deploy deploy} for performing the deploy.
390
+ # This method do not block by default, that is why you have to use the method {Cute::G5K::API#wait_for_deploy wait_for_deploy} in order to block the execution
391
+ # until the deployment is done.
392
+ #
393
+ # require 'cute'
394
+ #
395
+ # g5k = Cute::G5K::API.new()
396
+ #
397
+ # job = g5k.reserve(:site => "lyon", :nodes => 5, :walltime => "03:00:00", :type => :deploy)
398
+ #
399
+ # nodes = job["assigned_nodes"]
400
+ #
401
+ # slaves = nodes[1..4]
402
+ # master = nodes-slaves
403
+ #
404
+ # g5k.deploy(job,:nodes => master, :env => 'https://public.lyon.grid5000.fr/~user/debian_master_img.yaml')
405
+ # g5k.deploy(job,:nodes => slaves, :env => 'https://public.lyon.grid5000.fr/~user/debian_slaves_img.yaml')
406
+ #
407
+ # g5k.wait_for_deploy(job)
408
+ #
409
+ # puts "master node: #{master}"
410
+ # puts "slaves nodes: #{slaves}"
411
+ #
412
+ # You can check out the documentation of {Cute::G5K::API#reserve reserve} and {Cute::G5K::API#deploy deploy} methods
413
+ # to know all the parameters supported and more complex uses.
414
+ #
415
+ # == Another useful methods
416
+ #
417
+ # Let's use *pry* to show other useful methods. As shown in {file:README.md Ruby Cute} the *cute* command will open a
418
+ # pry shell with some modules preloaded and it will create the variable $g5k to access {Cute::G5K::API G5K::API} class.
419
+ # Therefore, we can consult the name of the cluster available in a specific site.
420
+ #
421
+ # [4] pry(main)> $g5k.cluster_uids("grenoble")
422
+ # => ["adonis", "edel", "genepi"]
423
+ #
424
+ # As well as the deployable environments:
425
+ #
426
+ # [6] pry(main)> $g5k.environment_uids("grenoble")
427
+ # => ["squeeze-x64-base", "squeeze-x64-big", "squeeze-x64-nfs", "wheezy-x64-base", "wheezy-x64-big", "wheezy-x64-min", "wheezy-x64-nfs", "wheezy-x64-xen"]
428
+ #
429
+ # For getting a list of sites available in Grid'5000 you can use:
430
+ #
431
+ # [7] pry(main)> $g5k.site_uids()
432
+ # => ["grenoble", "lille", "luxembourg", "lyon",...]
433
+ #
434
+ # We can get the status of nodes in a given site by using:
435
+ #
436
+ # [8] pry(main)> $g5k.nodes_status("lyon")
437
+ # => {"taurus-2.lyon.grid5000.fr"=>"besteffort", "taurus-16.lyon.grid5000.fr"=>"besteffort", "taurus-15.lyon.grid5000.fr"=>"besteffort", ...}
438
+ #
439
+ # We can get information about our submitted jobs by using:
440
+ #
441
+ # [11] pry(main)> $g5k.get_my_jobs("grenoble")
442
+ # => [{"uid"=>1679094,
443
+ # "user_uid"=>"cruizsanabria",
444
+ # "user"=>"cruizsanabria",
445
+ # "walltime"=>3600,
446
+ # "queue"=>"default",
447
+ # "state"=>"running", ...}, ...]
448
+ #
449
+ # If we are done with our experiment, we can release the submitted job or all jobs in a given site as follows:
450
+ #
451
+ # [12] pry(main)> $g5k.release(job)
452
+ # [13] pry(main)> $g5k.release_all("grenoble")
453
+ class API
454
+
455
+ # Assigns a logger
456
+ #
457
+ # = Examples
458
+ # You can use this attribute to control how to log all messages produce by {Cute::G5K::API G5K::API}.
459
+ # For example, below we use the logger available in Ruby standard library.
460
+ #
461
+ # require 'cute'
462
+ # require 'logger'
463
+ #
464
+ # g5k = Cute::G5K::API.new()
465
+ #
466
+ # g5k.logger = Logger.new(File.new('experiment_1.log'))
467
+ attr_accessor :logger
468
+ # Initializes a REST connection for Grid'5000 API
469
+ #
470
+ # = Example
471
+ # You can specify another configuration file using the option *:conf_file*, for example:
472
+ #
473
+ # g5k = Cute::G5K::API.new(:conf_file =>"config file path")
474
+ #
475
+ # You can specify other parameter to use:
476
+ #
477
+ # g5k = Cute::G5K::API.new(:uri => "https://api.grid5000.fr", :version => "sid")
478
+ #
479
+ # If you want to ignore {Cute::G5K::RequestFailed ResquestFailed} exceptions you can use:
480
+ #
481
+ # g5k = Cute::G5K::API.new(:on_error => :ignore)
482
+ #
483
+ # @param [Hash] params Contains initialization parameters.
484
+ # @option params [String] :conf_file Path for configuration file
485
+ # @option params [String] :uri REST API URI to contact
486
+ # @option params [String] :version Version of the REST API to use
487
+ # @option params [String] :user Username to access the REST API
488
+ # @option params [String] :pass Password to access the REST API
489
+ # @option params [Symbol] :on_error Set to :ignore if you want to ignore {Cute::G5K::RequestFailed ResquestFailed} exceptions.
490
+ def initialize(params={})
491
+ config = {}
492
+ default_file = "#{ENV['HOME']}/.grid5000_api.yml"
493
+
494
+ if params[:conf_file].nil? then
495
+ params[:conf_file] = default_file if File.exist?(default_file)
496
+ end
497
+
498
+ config = YAML.load(File.open(params[:conf_file],'r')) unless params[:conf_file].nil?
499
+ @user = params[:user] || config["username"]
500
+ @pass = params[:pass] || config["password"]
501
+ @uri = params[:uri] || config["uri"]
502
+ @api_version = params[:version] || config["version"] || "sid"
503
+ @logger = nil
504
+
505
+ begin
506
+ @g5k_connection = G5KRest.new(@uri,@api_version,@user,@pass,params[:on_error])
507
+ rescue
508
+ msg_create_file = ""
509
+ if (not File.exist?(default_file)) && params[:conf_file].nil? then
510
+ msg_create_file = "Please create the file: ~/.grid5000_api.yml and
511
+ put the necessary credentials or use the option
512
+ :conf_file to indicate another file for the credentials"
513
+ end
514
+ raise "Unable to authorize against the Grid'5000 API.
515
+ #{msg_create_file}"
516
+
517
+ end
518
+ end
519
+
520
+ # It returns the site name. Example:
521
+ # site #=> "rennes"
522
+ # This will only work when {Cute::G5K::API G5K::API} is used within Grid'5000.
523
+ # In the other cases it will return *nil*
524
+ # @return [String] the site name where the method is called on
525
+ def site
526
+ p = `hostname`.chop
527
+ res = /^.*\.(.*).*\.grid5000.fr/.match(p)
528
+ res[1] unless res.nil?
529
+ end
530
+
531
+ # @api private
532
+ # @return the rest point for performing low level REST requests
533
+ def rest
534
+ @g5k_connection
535
+ end
536
+
537
+ # @return [String] Grid'5000 user
538
+ def g5k_user
539
+ return @user.nil? ? ENV['USER'] : @user
540
+ end
541
+
542
+ # Returns all sites identifiers
543
+ #
544
+ # = Example:
545
+ # site_uids #=> ["grenoble", "lille", "luxembourg", "lyon",...]
546
+ #
547
+ # @return [Array] all site identifiers
548
+ def site_uids
549
+ return sites.uids
550
+ end
551
+
552
+ # Returns all cluster identifiers
553
+ #
554
+ # = Example:
555
+ # cluster_uids("grenoble") #=> ["adonis", "edel", "genepi"]
556
+ #
557
+ # @return [Array] cluster identifiers
558
+ def cluster_uids(site)
559
+ return clusters(site).uids
560
+ end
561
+
562
+ # Returns the name of the environments deployable in a given site.
563
+ # These can be used with {Cute::G5K::API#reserve reserve} and {Cute::G5K::API#deploy deploy} methods
564
+ #
565
+ # = Example:
566
+ # environment_uids("nancy") #=> ["squeeze-x64-base", "squeeze-x64-big", "squeeze-x64-nfs", ...]
567
+ #
568
+ # @return [Array] environment identifiers
569
+ def environment_uids(site)
570
+ # environments are returned by the API following the format squeeze-x64-big-1.8
571
+ # it returns environments without the version
572
+ environment_uids = environments(site).uids.map{ |e|
573
+ e_match = /(.*)-(.*)/.match(e)
574
+ new_name = e_match.nil? ? "" : e_match[1]
575
+ }
576
+
577
+ return environment_uids.uniq
578
+ end
579
+
580
+ # @return [Hash] all the status information of a given Grid'5000 site
581
+ # @param site [String] a valid Grid'5000 site name
582
+ def site_status(site)
583
+ @g5k_connection.get_json(api_uri("sites/#{site}/status"))
584
+ end
585
+
586
+ # @return [Hash] the nodes state (e.g, free, busy, etc) that belong to a given Grid'5000 site
587
+ # @param site [String] a valid Grid'5000 site name
588
+ def nodes_status(site)
589
+ nodes = {}
590
+ site_status(site).nodes.each do |node|
591
+ name = node[0]
592
+ status = node[1]["soft"]
593
+ nodes[name] = status
594
+ end
595
+ return nodes
596
+ end
597
+
598
+ # @return [Array] the description of all Grid'5000 sites
599
+ def sites
600
+ @g5k_connection.get_json(api_uri("sites")).items
601
+ end
602
+
603
+ # @return [Array] the description of clusters that belong to a given Grid'5000 site
604
+ # @param site [String] a valid Grid'5000 site name
605
+ def clusters(site)
606
+ @g5k_connection.get_json(api_uri("sites/#{site}/clusters")).items
607
+ end
608
+
609
+ # @return [Array] the description of all environments registered in a Grid'5000 site
610
+ def environments(site)
611
+ @g5k_connection.get_json(api_uri("sites/#{site}/environments")).items
612
+ end
613
+
614
+ # @return [Hash] all the jobs submitted in a given Grid'5000 site,
615
+ # if a uid is provided only the jobs owned by the user are shown.
616
+ # @param site [String] a valid Grid'5000 site name
617
+ # @param uid [String] user name in Grid'5000
618
+ # @param state [String] jobs state: running, waiting
619
+ def get_jobs(site, uid = nil, state = nil)
620
+ filter = "?"
621
+ filter += state.nil? ? "" : "state=#{state}"
622
+ filter += uid.nil? ? "" : "&user=#{uid}"
623
+ filter += "limit=25" if (state.nil? and uid.nil?)
624
+ jobs = @g5k_connection.get_json(api_uri("/sites/#{site}/jobs/#{filter}")).items
625
+ jobs.map{ |j| @g5k_connection.get_json(j.rel_self)}
626
+ # This request sometime is could take a little long when all jobs are requested
627
+ # The API return by default 50 the limit was set to 25 (e.g., 23 seconds).
628
+ end
629
+
630
+ # @return [Hash] the last 50 deployments performed in a Grid'5000 site
631
+ # @param site [String] a valid Grid'5000 site name
632
+ # @param uid [String] user name in Grid'5000
633
+ def get_deployments(site, uid = nil)
634
+ @g5k_connection.get_json(api_uri("sites/#{site}/deployments/?user=#{uid}")).items
635
+ end
636
+
637
+ # @return [Hash] information concerning a given job submitted in a Grid'5000 site
638
+ # @param site [String] a valid Grid'5000 site name
639
+ # @param jid [Fixnum] a valid job identifier
640
+ def get_job(site, jid)
641
+ @g5k_connection.get_json(api_uri("/sites/#{site}/jobs/#{jid}"))
642
+ end
643
+
644
+ # @return [Hash] switches information available in a given Grid'5000 site.
645
+ # @param site [String] a valid Grid'5000 site name
646
+ def get_switches(site)
647
+ items = @g5k_connection.get_json(api_uri("/sites/#{site}/network_equipments")).items
648
+ items = items.select { |x| x['kind'] == 'switch' }
649
+ # extract nodes connected to those switches
650
+ items.each { |switch|
651
+ conns = switch['linecards'].detect { |c| c['kind'] == 'node' }
652
+ next if conns.nil? # IB switches for example
653
+ nodes = conns['ports'] \
654
+ .select { |x| x != {} } \
655
+ .map { |x| x['uid'] } \
656
+ .map { |x| "#{x}.#{site}.grid5000.fr"}
657
+ switch['nodes'] = nodes
658
+ }
659
+ return items.select { |it| it.key?('nodes') }
660
+ end
661
+
662
+ # @return [Hash] information of a specific switch available in a given Grid'5000 site.
663
+ # @param site [String] a valid Grid'5000 site name
664
+ # @param name [String] a valid switch name
665
+ def get_switch(site, name)
666
+ s = get_switches(site).detect { |x| x.uid == name }
667
+ raise "Unknown switch '#{name}'" if s.nil?
668
+ return s
669
+ end
670
+
671
+ # Returns information of all my jobs submitted in a given site.
672
+ # By default it only shows the jobs in state *running*.
673
+ # You can specify another state like this:
674
+ #
675
+ # = Example
676
+ # get_my_jobs("nancy", state="waiting")
677
+ # Valid states are specified in {https://api.grid5000.fr/doc/4.0/reference/spec.html Grid'5000 API spec}
678
+ # @return [Array] all my submitted jobs to a given site and their associated deployments.
679
+ # @param site [String] a valid Grid'5000 site name
680
+ def get_my_jobs(site, state = "running")
681
+ jobs = get_jobs(site, g5k_user, state)
682
+ deployments = get_deployments(site, g5k_user)
683
+ # filtering deployments only the job in state running make sense
684
+ jobs.map{ |j| j["deploy"] = deployments.select{ |d| d["created_at"] > j["started_at"]} if j["state"] == "running"}
685
+ return jobs
686
+ end
687
+
688
+ # Returns an Array with all subnets reserved by a given job.
689
+ # Each element of the Array is a {https://github.com/bluemonk/ipaddress IPAddress::IPv4} object which we can interact with to obtain
690
+ # the details of our reserved subnets:
691
+ #
692
+ # = Example
693
+ # require 'cute'
694
+ #
695
+ # g5k = Cute::G5K::API.new()
696
+ #
697
+ # job = g5k.reserve(:site => "lyon", :resources => "/slash_22=1+{virtual!='none'}/nodes=1")
698
+ #
699
+ # subnet = g5k.get_subnets(job).first #=> we use 'first' because it is an array and we only reserved one subnet.
700
+ #
701
+ # ips = subnet.map{ |ip| ip.to_s }
702
+ #
703
+ # @return [Array] all the subnets defined in a given job
704
+ # @param job [G5KJSON] as described in {Cute::G5K::G5KJSON job}
705
+ def get_subnets(job)
706
+ subnets = job.resources["subnets"]
707
+ subnets.map{|s| IPAddress::IPv4.new s }
708
+ end
709
+
710
+ # @return [Array] all the nodes in the VLAN
711
+ # @param job [G5KJSON] as described in {Cute::G5K::G5KJSON job}
712
+ def get_vlan_nodes(job)
713
+ if job.resources["vlans"].nil?
714
+ return nil
715
+ else
716
+ vlan_id = job.resources["vlans"].first
717
+ end
718
+ nodes = job["assigned_nodes"]
719
+ reg = /^(\w+-\d+)(\..*)$/
720
+ nodes.map {
721
+ |name| reg.match(name)[1]+"-kavlan-"+vlan_id.to_s+reg.match(name)[2] unless reg.match(name).nil?
722
+ }
723
+ end
724
+
725
+ # Releases all jobs on a site
726
+ # @param site [String] a valid Grid'5000 site name
727
+ def release_all(site)
728
+ Timeout.timeout(20) do
729
+ jobs = get_my_jobs(site,"running") + get_my_jobs(site,"waiting")
730
+ break if jobs.empty?
731
+ begin
732
+ jobs.each { |j| release(j) }
733
+ rescue Cute::G5K::RequestFailed => e
734
+ raise unless e.response.include?('already killed')
735
+ end
736
+ end
737
+ return true
738
+ end
739
+
740
+ # Releases a resource, it can be a job or a deploy.
741
+ def release(r)
742
+ begin
743
+ return @g5k_connection.delete_json(r.rel_self)
744
+ rescue Cute::G5K::RequestFailed => e
745
+ raise unless e.response.include?('already killed')
746
+ end
747
+ end
748
+
749
+ # Performs a reservation in Grid'5000.
750
+ #
751
+ # = Examples
752
+ #
753
+ # By default this method blocks until the reservation is ready,
754
+ # if we want this method to return after creating the reservation we set the option *:wait* to *false*.
755
+ # Then, you can use the method {Cute::G5K::API#wait_for_job wait_for_job} to wait for the reservation.
756
+ #
757
+ # job = g5k.reserve(:nodes => 25, :site => 'luxembourg', :walltime => '01:00:00', :wait => false)
758
+ #
759
+ # job = g5k.wait_for_job(job, :wait_time => 100)
760
+ #
761
+ # == Reserving with properties
762
+ #
763
+ # job = g5k.reserve(:site => 'lyon', :nodes => 2, :properties => "wattmeter='YES'")
764
+ #
765
+ # job = g5k.reserve(:site => 'nancy', :nodes => 1, :properties => "switch='sgraphene1'")
766
+ #
767
+ # job = g5k.reserve(:site => 'nancy', :nodes => 1, :properties => "cputype='Intel Xeon E5-2650'")
768
+ #
769
+ # == Subnet reservation
770
+ #
771
+ # The example below reserves 2 nodes in the cluster *chirloute* located in Lille for 1 hour as well as 2 /22 subnets.
772
+ # We will get 2048 IP addresses that can be used, for example, in virtual machines.
773
+ # If walltime is not specified, 1 hour walltime will be assigned to the reservation.
774
+ #
775
+ # job = g5k.reserve(:site => 'lille', :cluster => 'chirloute', :nodes => 2,
776
+ # :env => 'wheezy-x64-xen', :keys => "~/my_ssh_jobkey",
777
+ # :subnets => [22,2])
778
+ #
779
+ # == Before using OAR hierarchy
780
+ # All non-deploy reservations are submitted by default with the OAR option "-allow_classic_ssh"
781
+ # which does not take advantage of the CPU/core management level.
782
+ # Therefore, in order to take advantage of this capability, SSH keys have to be specified at the moment of reserving resources.
783
+ # This has to be used whenever we perform a reservation with cpu and core hierarchy.
784
+ # Users are encouraged to create a pair of SSH keys for managing jobs, for instance the following command can be used:
785
+ #
786
+ # ssh-keygen -N "" -t rsa -f ~/my_ssh_jobkey
787
+ #
788
+ # The reserved nodes can be accessed using "oarsh" or by configuring the SSH connection as shown in {https://www.grid5000.fr/mediawiki/index.php/OAR2 OAR2}.
789
+ # You have to specify different keys per reservation if you want several jobs running at the same time in the same site.
790
+ # Example using the OAR hierarchy:
791
+ #
792
+ # job = g5k.reserve(:site => "grenoble", :switches => 3, :nodes => 1, :cpus => 1, :cores => 1, :keys => "~/my_ssh_jobkey")
793
+ #
794
+ # == Using OAR syntax
795
+ #
796
+ # The parameter *:resources* can be used instead of parameters such as: *:cluster*, *:nodes*, *:cpus*, *:walltime*, *:vlan*, *:subnets*, *:properties*, etc,
797
+ # which are shortcuts for OAR syntax. These shortcuts are ignored if the the parameter *:resources* is used.
798
+ # Using the parameter *:resources* allows to express more flexible and complex reservations by using directly the OAR syntax.
799
+ # Therefore, the two examples shown below are equivalent:
800
+ #
801
+ # job = g5k.reserve(:site => "grenoble", :switches => 3, :nodes => 1, :cpus => 1, :cores => 1, :keys => "~/my_ssh_jobkey")
802
+ # job = g5k.reserve(:site => "grenoble", :resources => "/switch=3/nodes=1/cpu=1/core=1", :keys => "~/my_ssh_jobkey")
803
+ #
804
+ # Combining OAR hierarchy with properties:
805
+ #
806
+ # job = g5k.reserve(:site => "grenoble", :resources => "{ib10g='YES' and memnode=24160}/cluster=1/nodes=2/core=1", :keys => "~/my_ssh_jobkey")
807
+ #
808
+ # If we want 2 nodes with the following constraints:
809
+ # 1) nodes on 2 different clusters of the same site, 2) nodes with virtualization capability enabled
810
+ # 3) 1 /22 subnet. The reservation will be like:
811
+ #
812
+ # job = g5k.reserve(:site => "rennes", :resources => "/slash_22=1+{virtual!='none'}/cluster=2/nodes=1")
813
+ #
814
+ # Another reservation for two clusters:
815
+ #
816
+ # job = g5k.reserve(:site => "nancy", :resources => "{cluster='graphene'}/nodes=2+{cluster='griffon'}/nodes=3")
817
+ #
818
+ # Reservation using a local VLAN
819
+ #
820
+ # job = g5k.reserve(:site => 'nancy', :resources => "{type='kavlan-local'}/vlan=1,nodes=1", :env => 'wheezy-x64-xen')
821
+ #
822
+ # @return [G5KJSON] as described in {Cute::G5K::G5KJSON job}
823
+ # @param [Hash] opts Options for reservation in Grid'5000
824
+ # @option opts [Numeric] :nodes Number of nodes to reserve
825
+ # @option opts [String] :walltime Walltime of the reservation
826
+ # @option opts [String] :site Grid'5000 site
827
+ # @option opts [Symbol] :type Type of reservation: :deploy, :allow_classic
828
+ # @option opts [String] :name Reservation name
829
+ # @option opts [String] :cmd The command to execute when the job starts (e.g. ./my-script.sh).
830
+ # @option opts [String] :cluster Valid Grid'5000 cluster
831
+ # @option opts [Array] :subnets 1) prefix_size, 2) number of subnets
832
+ # @option opts [String] :env Environment name for {http://kadeploy3.gforge.inria.fr/ Kadeploy}
833
+ # @option opts [Symbol] :vlan Vlan type: :routed, :local, :global
834
+ # @option opts [String] :properties OAR properties defined in the cluster
835
+ # @option opts [String] :resources OAR syntax for complex submissions
836
+ # @option opts [String] :reservation Request a job to be scheduled a specific date.
837
+ # The date format is "YYYY-MM-DD HH:MM:SS".
838
+ # @option opts [Boolean] :wait Whether or not to wait until the job is running (default is true)
839
+ def reserve(opts)
840
+
841
+ # checking valid options
842
+ valid_opts = [:site, :cluster, :switches, :cpus, :cores, :nodes, :walltime, :cmd,
843
+ :type, :name, :subnets, :env, :vlan, :properties, :resources, :reservation, :wait, :keys]
844
+ unre_opts = opts.keys - valid_opts
845
+ raise ArgumentError, "Unrecognized option #{unre_opts}" unless unre_opts.empty?
846
+
847
+ nodes = opts.fetch(:nodes, 1)
848
+ walltime = opts.fetch(:walltime, '01:00:00')
849
+ site = opts[:site]
850
+ type = opts[:type]
851
+ name = opts.fetch(:name, 'rubyCute job')
852
+ command = opts[:cmd]
853
+ opts[:wait] = true if opts[:wait].nil?
854
+ cluster = opts[:cluster]
855
+ switches = opts[:switches]
856
+ cpus = opts[:cpus]
857
+ cores = opts[:cores]
858
+ subnets = opts[:subnets]
859
+ properties = opts[:properties]
860
+ reservation = opts[:reservation]
861
+ resources = opts.fetch(:resources, "")
862
+ type = :deploy if opts[:env]
863
+ keys = opts[:keys]
864
+
865
+ vlan_opts = {:routed => "kavlan",:global => "kavlan-global",:local => "kavlan-local"}
866
+ vlan = nil
867
+ unless opts[:vlan].nil?
868
+ if vlan_opts.include?(opts[:vlan]) then
869
+ vlan = vlan_opts.fetch(opts[:vlan])
870
+ else
871
+ raise ArgumentError, 'Option for vlan not recognized'
872
+ end
873
+ end
874
+
875
+ raise 'At least nodes, time and site must be given' if [nodes, walltime, site].any? { |x| x.nil? }
876
+
877
+ secs = walltime.to_secs
878
+ walltime = walltime.to_time
879
+
880
+ raise 'Nodes must be an integer.' unless nodes.is_a?(Integer)
881
+
882
+ command = "sleep #{secs}" if command.nil?
883
+ type = type.to_sym unless type.nil?
884
+
885
+ if resources == ""
886
+ resources = "/switch=#{switches}" unless switches.nil?
887
+ resources += "/nodes=#{nodes}"
888
+ resources += "/cpu=#{cpus}" unless cpus.nil?
889
+ resources += "/core=#{cores}" unless cores.nil?
890
+ resources = "{cluster='#{cluster}'}" + resources unless cluster.nil?
891
+ resources = "{type='#{vlan}'}/vlan=1+" + resources unless vlan.nil?
892
+ resources = "slash_#{subnets[0]}=#{subnets[1]}+" + resources unless subnets.nil?
893
+ end
894
+
895
+ resources += ",walltime=#{walltime}" unless resources.include?("walltime")
896
+
897
+ payload = {
898
+ 'resources' => resources,
899
+ 'name' => name,
900
+ 'command' => command
901
+ }
902
+
903
+ info "Reserving resources: #{resources} (type: #{type}) (in #{site})"
904
+
905
+ payload['properties'] = properties unless properties.nil?
906
+ payload['types'] = [ type.to_s ] unless type.nil?
907
+
908
+ if not type == :deploy
909
+ if opts[:keys]
910
+ payload['import-job-key-from-file'] = [ File.expand_path(keys) ]
911
+ else
912
+ payload['types'] = [ 'allow_classic_ssh' ]
913
+ end
914
+ end
915
+
916
+ if reservation
917
+ payload['reservation'] = reservation
918
+ info "Starting this reservation at #{reservation}"
919
+ end
920
+
921
+ begin
922
+ # Support for the option "import-job-key-from-file"
923
+ # The request has to be redirected to the OAR API given that Grid'5000 API
924
+ # does not support some OAR options.
925
+ if payload['import-job-key-from-file'] then
926
+ # Adding double quotes otherwise we have a syntax error from OAR API
927
+ payload["resources"] = "\"#{payload["resources"]}\""
928
+ temp = @g5k_connection.post_json(api_uri("sites/#{site}/internal/oarapi/jobs"),payload)
929
+ sleep 1 # This is for being sure that our job appears on the list
930
+ r = get_my_jobs(site,nil).select{ |j| j["uid"] == temp["id"] }.first
931
+ else
932
+ r = @g5k_connection.post_json(api_uri("sites/#{site}/jobs"),payload) # This makes reference to the same class
933
+ end
934
+ rescue Error => e
935
+ info "Fail to submit job"
936
+ info e.message
937
+ e.http_body.split("\\n").each{ |line| info line}
938
+ raise
939
+ end
940
+
941
+ job = @g5k_connection.get_json(r.rel_self)
942
+ job = wait_for_job(job) if opts[:wait] == true
943
+ opts.delete(:nodes) # to not collapse with deploy options
944
+ deploy(job,opts) unless opts[:env].nil? #type == :deploy
945
+ return job
946
+
947
+ end
948
+
949
+ # Blocks until job is in *running* state
950
+ #
951
+ # = Example
952
+ # You can pass the parameter *:wait_time* that allows you to timeout the submission (by default is 10h).
953
+ # The method will throw a {Cute::G5K::EventTimeout Timeout} exception
954
+ # that you can catch and react upon.
955
+ # The following example shows how can be used, let's suppose we want to find 5 nodes available for
956
+ # 3 hours. We can try in each site using the script below.
957
+ #
958
+ # require 'cute'
959
+ #
960
+ # g5k = Cute::G5K::API.new()
961
+ #
962
+ # sites = g5k.site_uids
963
+ #
964
+ # sites.each{ |site|
965
+ # job = g5k.reserve(:site => site, :nodes => 5, :wait => false, :walltime => "03:00:00")
966
+ # begin
967
+ # job = g5k.wait_for_job(job, :wait_time => 60)
968
+ # puts "Nodes assigned #{job['assigned_nodes']}"
969
+ # break
970
+ # rescue Cute::G5K::EventTimeout
971
+ # puts "We waited too long in site #{site} let's release the job and try in another site"
972
+ # g5k.release(job)
973
+ # end
974
+ # }
975
+ #
976
+ # @param job [G5KJSON] as described in {Cute::G5K::G5KJSON job}
977
+ # @param opts [Hash] options
978
+ def wait_for_job(job,opts={})
979
+ opts[:wait_time] = 36000 if opts[:wait_time].nil?
980
+ jid = job['uid']
981
+ info "Waiting for reservation #{jid}"
982
+ begin
983
+ Timeout.timeout(opts[:wait_time]) do
984
+ while true
985
+ job = job.refresh(@g5k_connection)
986
+ t = job['scheduled_at']
987
+ if !t.nil?
988
+ t = Time.at(t)
989
+ secs = [ t - Time.now, 0 ].max.to_i
990
+ info "Reservation #{jid} should be available at #{t} (#{secs} s)"
991
+ end
992
+ break if job['state'] == 'running'
993
+ raise "Job is finishing." if job['state'] == 'finishing'
994
+ Kernel.sleep(5)
995
+ end
996
+ end
997
+ rescue Timeout::Error
998
+ raise EventTimeout.new("Event timeout")
999
+ end
1000
+
1001
+ info "Reservation #{jid} ready"
1002
+ return job
1003
+ end
1004
+
1005
+ # Deploys an environment in a set of reserved nodes using {http://kadeploy3.gforge.inria.fr/ Kadeploy}.
1006
+ # A job structure returned by {Cute::G5K::API#reserve reserve} or {Cute::G5K::API#get_my_jobs get_my_jobs} methods
1007
+ # is mandatory as a parameter as well as the environment to deploy.
1008
+ # By default this method does not block, for that you have to set the option *:wait* to *true*.
1009
+ #
1010
+ # = Examples
1011
+ # Deploying the production environment *wheezy-x64-base* on all the reserved nodes and wait until the deployment is done:
1012
+ #
1013
+ # deploy(job, :env => "wheezy-x64-base", :wait => true)
1014
+ #
1015
+ # Other parameters you can specify are *:nodes* [Array] for deploying on specific nodes within a job and
1016
+ # *:keys* [String] to specify the public key to use during the deployment.
1017
+ #
1018
+ # deploy(job, :nodes => ["genepi-2.grid5000.fr"], :env => "wheezy-x64-xen", :keys => "~/my_key")
1019
+ #
1020
+ # @param job [G5KJSON] as described in {Cute::G5K::G5KJSON job}
1021
+ # @param [Hash] opts Deploy options
1022
+ # @option opts [String] :env {http://kadeploy3.gforge.inria.fr/ Kadeploy} environment to deploy
1023
+ # @option opts [String] :nodes Specifies the nodes to deploy on
1024
+ # @option opts [String] :keys Specifies the SSH keys to copy for the deployment
1025
+ # @option opts [Boolean] :wait Whether or not to wait until the deployment is done (default is false)
1026
+ # @return [G5KJSON] a job with deploy information as described in {Cute::G5K::G5KJSON job}
1027
+ def deploy(job, opts = {})
1028
+
1029
+ # checking valid options, same as reserve option even though some option dont make any sense
1030
+ valid_opts = [:site, :cluster, :switches, :cpus, :cores, :nodes, :walltime, :cmd,
1031
+ :type, :name, :subnets, :env, :vlan, :properties, :resources, :reservation, :wait, :keys]
1032
+
1033
+ unre_opts = opts.keys - valid_opts
1034
+ raise ArgumentError, "Unrecognized option #{unre_opts}" unless unre_opts.empty?
1035
+
1036
+ raise ArgumentError, "Unrecognized job format" unless job.is_a?(G5KJSON)
1037
+
1038
+ env = opts[:env]
1039
+ raise ArgumentError, "Environment must be given" if env.nil?
1040
+
1041
+ nodes = opts[:nodes].nil? ? job['assigned_nodes'] : opts[:nodes]
1042
+ raise "Unrecognized nodes format, use an Array" unless nodes.is_a?(Array)
1043
+
1044
+ site = @g5k_connection.follow_parent(job).uid
1045
+
1046
+ if opts[:keys].nil? then
1047
+ public_key_path = File.expand_path("~/.ssh/id_rsa.pub")
1048
+ public_key_file = File.exist?(public_key_path) ? File.read(public_key_path) : ""
1049
+ else
1050
+ public_key_file = File.read("#{File.expand_path(opts[:keys])}.pub")
1051
+ end
1052
+
1053
+ payload = {
1054
+ 'nodes' => nodes,
1055
+ 'environment' => env,
1056
+ 'key' => public_key_file,
1057
+ }
1058
+
1059
+ if !job.resources["vlans"].nil?
1060
+ vlan = job.resources["vlans"].first
1061
+ payload['vlan'] = vlan
1062
+ info "Found VLAN with uid = #{vlan}"
1063
+ end
1064
+
1065
+ info "Creating deployment"
1066
+
1067
+ begin
1068
+ r = @g5k_connection.post_json(api_uri("sites/#{site}/deployments"), payload)
1069
+ rescue Error => e
1070
+ info "Fail to deploy"
1071
+ info e.message
1072
+ e.http_body.split("\\n").each{ |line| info line}
1073
+ raise
1074
+ end
1075
+
1076
+ job["deploy"] = [] if job["deploy"].nil?
1077
+
1078
+ job["deploy"].push(r)
1079
+
1080
+ job = wait_for_deploy(job) if opts[:wait] == true
1081
+
1082
+ return job
1083
+
1084
+ end
1085
+
1086
+ # Returns the status of all deployments performed within a job.
1087
+ # The results can be filtered using a Hash with valid deployment properties
1088
+ # described in {https://api.grid5000.fr/doc/4.0/reference/spec.html Grid'5000 API spec}.
1089
+ #
1090
+ # = Example
1091
+ #
1092
+ # deploy_status(job, :nodes => ["adonis-10.grenoble.grid5000.fr"], :status => "terminated")
1093
+ #
1094
+ # @return [Array] status of deploys within a job
1095
+ # @param job [G5KJSON] as described in {Cute::G5K::G5KJSON job}
1096
+ # @param filter [Hash] filter the deployments to be returned.
1097
+ def deploy_status(job,filter = {})
1098
+
1099
+ job["deploy"].map!{ |d| d.refresh(@g5k_connection) }
1100
+
1101
+ filter.keep_if{ |k,v| v} # removes nil values
1102
+ if filter.empty?
1103
+ status = job["deploy"].map{ |d| d["status"] }
1104
+ else
1105
+ status = job["deploy"].map{ |d| d["status"] if filter.select{ |k,v| d[k.to_s] != v }.empty? }
1106
+ end
1107
+ return status.compact
1108
+
1109
+ end
1110
+
1111
+ # Blocks until deployments have *terminated* status
1112
+ #
1113
+ # = Examples
1114
+ # This method requires a job as a parameter and it will blocks by default until all deployments
1115
+ # within the job pass form *processing* status to *terminated* status.
1116
+ #
1117
+ # wait_for_deploy(job)
1118
+ #
1119
+ # You can wait for specific deployments using the option *:nodes*. This can be useful when performing different deployments on the reserved resources.
1120
+ #
1121
+ # wait_for_deploy(job, :nodes => ["adonis-10.grenoble.grid5000.fr"])
1122
+ #
1123
+ # Another parameter you can specify is *:wait_time* that allows you to timeout the deployment (by default is 10h).
1124
+ # The method will throw a {Cute::G5K::EventTimeout Timeout} exception
1125
+ # that you can catch and react upon. This example illustrates how this can be used.
1126
+ #
1127
+ # require 'cute'
1128
+ #
1129
+ # g5k = Cute::G5K::API.new()
1130
+ #
1131
+ # job = g5k.reserve(:nodes => 1, :site => 'lyon', :env => 'wheezy-x64-base')
1132
+ #
1133
+ # begin
1134
+ # g5k.wait_for_deploy(job,:wait_time => 100)
1135
+ # rescue Cute::G5K::EventTimeout
1136
+ # puts "We waited too long let's release the job"
1137
+ # g5k.release(job)
1138
+ # end
1139
+ #
1140
+ # @param job [G5KJSON] as described in {Cute::G5K::G5KJSON job}
1141
+ # @param opts [Hash] options
1142
+ def wait_for_deploy(job,opts = {})
1143
+
1144
+ raise "Deploy information not present in the given job" if job["deploy"].nil?
1145
+
1146
+ opts.merge!({:wait_time => 36000}) if opts[:wait_time].nil?
1147
+ nodes = opts[:nodes]
1148
+
1149
+ begin
1150
+ Timeout.timeout(opts[:wait_time]) do
1151
+ # it will ask just for processing status
1152
+ status = deploy_status(job,{:nodes => nodes, :status => "processing"})
1153
+ until status.empty? do
1154
+ info "Waiting for #{status.length} deployment"
1155
+ sleep 4
1156
+ status = deploy_status(job,{:nodes => nodes, :status => "processing"})
1157
+ end
1158
+ info "Deployment finished"
1159
+ return job
1160
+ end
1161
+ rescue Timeout::Error
1162
+ raise EventTimeout.new("Timeout triggered")
1163
+ end
1164
+
1165
+ end
1166
+
1167
+ private
1168
+ # Handles the output of messages within the module
1169
+ # @param msg [String] message to show
1170
+ def info(msg)
1171
+ if @logger.nil? then
1172
+ t = Time.now
1173
+ s = t.strftime('%Y-%m-%d %H:%M:%S.%L')
1174
+ puts "#{s} => #{msg}"
1175
+ else
1176
+ @logger.info(msg)
1177
+ end
1178
+ end
1179
+
1180
+ # @return a valid Grid'5000 resource
1181
+ # it avoids "//"
1182
+ def api_uri(path)
1183
+ path = path[1..-1] if path.start_with?('/')
1184
+ return "#{@api_version}/#{path}"
1185
+ end
1186
+
1187
+ end
1188
+
1189
+ end
1190
+ end