ruby-cute 0.6 → 0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4bee03262cac39274dd43426589fce86de6962eb
-  data.tar.gz: b0262a03d84c76f2557550f4a0c05bcefc7c2a09
+  metadata.gz: c25c70220c9af96769ac92522d76cb09ee237846
+  data.tar.gz: 98596abba4e8569122cf00369fafc0b7ea8eefb1
 SHA512:
-  metadata.gz: 2241ac02410656a7dbf16006d2f28ea9d02767e1b4e395fc0861798e6891eb60d60ede93b7899d87f86fc3dbe355da0071a8e591760f828b5a6e14737f8011b2
-  data.tar.gz: ab9c2570faf491c33759ecd1317cf574b3e4779b86df251986b2456d50ba3b22eac475bc9af04e2017d278044f83bf0497f58ac6fb602ecc0be72f8f643f5a4e
+  metadata.gz: 305140e82df29d420897496719de4a71554a10ea7c9f32c90dd8c956576c7c374c6b2e57cac4daa685775631e85a6e6c23954d9686d83d4ccb3a5aa48cac459e
+  data.tar.gz: ae3e6bdb53792be17caeb8ee67824d4c61cb7f64ad15b29e74c099bc60a5ede6993d4051d1854e4b5826d356b1c16bf60d3679baa9ba5cee005afc18b23cb39f

data/README.md

@@ -36,7 +36,6 @@ $ cat > ~/.grid5000_api.yml << EOF
   uri: https://api.grid5000.fr/
   username: user
   password: **********
-  version: sid
   EOF
 
 ```
@@ -106,9 +105,86 @@ Within this shell you have preloaded [G5K Module](http://www.rubydoc.info/github
 [Net::SSH::Multi](http://www.rubydoc.info/github/ruby-cute/ruby-cute/master/Net/SSH/Multi),
 you can go directly to their respective documentation to know how to take advantage of them.
 
-### Experiment example
+### Examples
+
+The following examples show how to use the [G5K Module](http://www.rubydoc.info/github/ruby-cute/ruby-cute/master/Cute/G5K/API) in an experiment.
+
+#### Example 1: automatic experiment bootstrap
+
+This example (also available as
+[examples/xp-bootstrap](http://www.rubydoc.info/github/ruby-cute/ruby-cute/master/file/examples/xp-bootstrap)
+is simple script automating the initial bootstrap of an experiment.
+
+```ruby
+#!/usr/bin/ruby -w
+
+# This script, to be executed on a frontend, automates the initial setup
+# of an experiment, and then sleeps to let the user take over.
+# The same script, run with --reserve, will handle resources reservation
+
+# To make this work:
+# - connect to a frontend
+# - install ruby-cute: gem install --user-install ruby-cute
+# - get this script, make it executable (chmod a+rx xp-bootstrap)
+# - run it: ./xp-bootstrap --reserve
+
+gem 'ruby-cute', ">=0.6"
+require 'cute'
+require 'pp'
+
+g5k = Cute::G5K::API.new
+G5K_SITE = `hostname --fqdn`.split('.')[-3] # get the site name from the `hostname` command
+G5K_ENV = 'jessie-x64-base' # environment to deploy
+NODES = 2
+WALLTIME = '0:30'
+
+# When the script is run with --reserve, use Ruby-Cute to reserve resources and run the script again inside the reservation, when it starts
+if ARGV[0] == '--reserve'
+  # reserve two nodes for 30 mins
+  job = g5k.reserve(:site => G5K_SITE, :nodes => NODES, :walltime => WALLTIME, :type => :deploy, :wait => false,
+                    :name => 'xp-bootstrap',
+                    :cmd => File::realpath(__FILE__)
+                   )
+  puts "Job #{job['uid']} created. Monitor its status with e.g.: oarstat -fj #{job['uid']}"
+  exit(0)
+end
+
+###########################################################################
+#### What follows is what gets executed inside the resources reservation
+
+# for better output, redirect stderr to stdout, make stdout a synchronized output stream
+STDERR.reopen(STDOUT)
+STDOUT.sync = true
+
+jobid = ENV['OAR_JOB_ID']
+raise "OAR_JOB_ID not set. Are you running inside a OAR reservation? Maybe you should use #{__FILE__} --reserve?" if not jobid
+
+# get job details
+job = g5k.get_job(G5K_SITE, jobid)
+nodes = job['assigned_nodes']
+puts "Running on: #{nodes.join(' ')}"
+
+# deploying all nodes, waiting for the end of deployment
+g5k.deploy(job,  :env => G5K_ENV, :wait => true)
+
+raise "Deployment ended with error" if ((job['deploy'].last['status'] == 'error') or (not job['deploy'].last['result'].to_a.all? { |e| e[1]['state'] == 'OK' }))
+
+cmd = 'apt-get update && apt-get -y install nuttcp'
+puts "Running command: #{cmd}"
+# Run a command on each node and analyze result
+ssh = Net::SSH::Multi::Session::new
+nodes.each { |n| ssh.use "root@#{n}" }
+r = ssh.exec!(cmd)
+raise "Command failed on at least one node\n#{r}" if not r.to_a.all? { |e| e[1][:status] == 0 }
+
+# Sleep for a very long time to avoid reservation termination 
+puts "Experiment preparation finished."
+puts "Nodes: #{nodes.join(' ')}"
+sleep 86400*365
+```
+
+#### Example 2: running MPI
 
-The following example shows how to use the [G5K Module](http://www.rubydoc.info/github/ruby-cute/ruby-cute/master/Cute/G5K/API) in an experiment.
 This example implements the experiment described in
 [MPI on Grid5000](https://www.grid5000.fr/mediawiki/index.php/Run_MPI_On_Grid%275000#Setting_up_and_starting_Open_MPI_to_use_high_performance_interconnect).
 
@@ -155,16 +231,16 @@ g5k.release(job) # Frees resources.
 
 ## Contact information
 
-Ruby-Cute is maintained by the Algorille team at LORIA/Inria Nancy - Grand Est, and specifically by:
+Ruby-Cute is maintained by the Madynes team at LORIA/Inria Nancy - Grand Est, and specifically by:
 
-* Cristian Ruiz <cristian.ruiz@inria.fr>
-* Emmanuel Jeanvoine <emmanuel.jeanvoine@inria.fr>
 * Lucas Nussbaum <lucas.nussbaum@loria.fr>
 
 Past contributors include:
 
 * Sébastien Badia <sebastien.badia@inria.fr>
 * Tomasz Buchert <tomasz.buchert@inria.fr>
+* Emmanuel Jeanvoine <emmanuel.jeanvoine@inria.fr>
+* Cristian Ruiz <cristian.ruiz@inria.fr>
 * Luc Sarzyniec <luc.sarzyniec@inria.fr>
 
-Questions/comments should be directed to Lucas Nussbaum and Emmanuel Jeanvoine.
+Questions/comments should be directed to Lucas Nussbaum.

data/debian/changelog

@@ -1,16 +1,16 @@
-ruby-cute (0.6) UNRELEASED; urgency=medium
+ruby-cute (0.6) unstable; urgency=medium
 
   * New release.
 
- -- Cristian Ruiz <cristian.ruiz@inria.fr>  Thu, 20 Oct 2016 10:57:07 +0100
+ -- Lucas Nussbaum <lucas@debian.org>  Fri, 17 Mar 2017 23:38:19 +0100
 
-ruby-cute (0.5) UNRELEASED; urgency=medium
+ruby-cute (0.5) unstable; urgency=medium
 
   * New release.
 
  -- Cristian Ruiz <cristian.ruiz@inria.fr>  Tue, 3 May 2016 08:54:42 +0100
 
-ruby-cute (0.4) UNRELEASED; urgency=medium
+ruby-cute (0.4) unstable; urgency=medium
 
   * Initial release.
 

data/examples/xp-bootstrap

@@ -0,0 +1,65 @@
+#!/usr/bin/ruby -w
+
+# This script, to be executed on a frontend, automates the initial setup
+# of an experiment, and then sleeps to let the user take over.
+# The same script, run with --reserve, will handle resources reservation
+
+# To make this work:
+# - connect to a frontend
+# - install ruby-cute: gem install --user-install ruby-cute
+# - get this script, make it executable (chmod a+rx xp-bootstrap)
+# - run it: ./xp-bootstrap --reserve
+
+gem 'ruby-cute', ">=0.6"
+require 'cute'
+require 'pp'
+
+g5k = Cute::G5K::API.new
+G5K_SITE = `hostname --fqdn`.split('.')[-3] # get the site name from the `hostname` command
+G5K_ENV = 'jessie-x64-base' # environment to deploy
+NODES = 2
+WALLTIME = '0:30'
+
+# When the script is run with --reserve, use Ruby-Cute to reserve resources and run the script again inside the reservation, when it starts
+if ARGV[0] == '--reserve'
+  # reserve two nodes for 30 mins
+  job = g5k.reserve(:site => G5K_SITE, :nodes => NODES, :walltime => WALLTIME, :type => :deploy, :wait => false,
+                    :name => 'xp-bootstrap',
+                    :cmd => File::realpath(__FILE__)
+                   )
+  puts "Job #{job['uid']} created. Monitor its status with e.g.: oarstat -fj #{job['uid']}"
+  exit(0)
+end
+
+###########################################################################
+#### What follows is what gets executed inside the resources reservation
+
+# for better output, redirect stderr to stdout, make stdout a synchronized output stream
+STDERR.reopen(STDOUT)
+STDOUT.sync = true
+
+jobid = ENV['OAR_JOB_ID']
+raise "OAR_JOB_ID not set. Are you running inside a OAR reservation? Maybe you should use #{__FILE__} --reserve?" if not jobid
+
+# get job details
+job = g5k.get_job(G5K_SITE, jobid)
+nodes = job['assigned_nodes']
+puts "Running on: #{nodes.join(' ')}"
+
+# deploying all nodes, waiting for the end of deployment
+g5k.deploy(job,  :env => G5K_ENV, :wait => true)
+
+raise "Deployment ended with error" if ((job['deploy'].last['status'] == 'error') or (not job['deploy'].last['result'].to_a.all? { |e| e[1]['state'] == 'OK' }))
+
+cmd = 'apt-get update && apt-get -y install nuttcp'
+puts "Running command: #{cmd}"
+# Run a command on each node and analyze result
+ssh = Net::SSH::Multi::Session::new
+nodes.each { |n| ssh.use "root@#{n}" }
+r = ssh.exec!(cmd)
+raise "Command failed on at least one node\n#{r}" if not r.to_a.all? { |e| e[1][:status] == 0 }
+
+# Sleep for a very long time to avoid reservation termination 
+puts "Experiment preparation finished."
+puts "Nodes: #{nodes.join(' ')}"
+sleep 86400*365

data/lib/cute.rb

@@ -1,5 +1,4 @@
 require 'cute/version'
-require 'cute/bash'
 require 'cute/execute'
 require 'cute/taktuk'
 require 'cute/g5k_api'

data/lib/cute/g5k_api.rb

@@ -1,7 +1,13 @@
 require 'restclient'
 require 'yaml'
 require 'json'
-require 'ipaddress'
+begin
+  old_verbose, $VERBOSE = $VERBOSE, false
+  require 'ipaddress'
+ensure
+  $VERBOSE = old_verbose
+end
+
 require 'uri'
 
 module Cute
@@ -225,14 +231,24 @@ module Cute
         return @api[path]
       end
 
+      RETRY_503_MAX = 10
+      RETRY_503_SLEEP = 1
+
       # @return [Hash] the HTTP response
       # @param path [String] this complements the URI to address to a specific resource
       def get_json(path)
-
+        retries = 0
         begin
           r = resource(path).get(:content_type => "application/json",
                                  :user_agent => @user_agent)
         rescue => e
+          if e.kind_of?(RestClient::Exception) and e.http_code == 503 and retries < RETRY_503_MAX
+            # the G5K REST API sometimes fail with error 503. In that case we should just wait and retry
+            puts("G5KRest: GET #{path} failed with error 503, retrying after #{RETRY_503_SLEEP} seconds")
+            retries += 1
+            sleep RETRY_503_SLEEP
+            retry
+          end
           handle_exception(e)
         end
         return G5KJSON.parse(r)
@@ -242,13 +258,20 @@ module Cute
       # @param path [String] this complements the URI to address to a specific resource
       # @param json [Hash] contains the characteristics of the resources to be created.
       def post_json(path, json)
-
+        retries = 0
         begin
           r = resource(path).post(json.to_json,
                                   :content_type => "application/json",
                                   :accept => "application/json",
                                   :user_agent => @user_agent)
         rescue => e
+          if e.kind_of?(RestClient::Exception) and e.http_code == 503 and retries < RETRY_503_MAX
+            # the G5K REST API sometimes fail with error 503. In that case we should just wait and retry
+            puts("G5KRest: POST #{path} failed with error 503, retrying after #{RETRY_503_SLEEP} seconds")
+            retries += 1
+            sleep RETRY_503_SLEEP
+            retry
+          end
           handle_exception(e)
         end
         return G5KJSON.parse(r)
@@ -257,9 +280,17 @@ module Cute
       # Deletes a resource on the server
       # @param path [String] this complements the URI to address to a specific resource
       def delete_json(path)
+        retries = 0
         begin
           return resource(path).delete()
         rescue  => e
+          if e.kind_of?(RestClient::Exception) and e.http_code == 503 and retries < RETRY_503_MAX
+            # the G5K REST API sometimes fail with error 503. In that case we should just wait and retry
+            puts("G5KRest: DELETE #{path} failed with error 503, retrying after #{RETRY_503_SLEEP} seconds")
+            retries += 1
+            sleep RETRY_503_SLEEP
+            retry
+          end
           handle_exception(e)
         end
       end
@@ -616,6 +647,7 @@ module Cute
         environment_uids = environments(site).uids.map{ |e|
           e_match = /(.*)-(.*)/.match(e)
           new_name = e_match.nil? ? "" : e_match[1]
+          new_name
         }
 
         return environment_uids.uniq
@@ -1160,7 +1192,7 @@ module Cute
       # @param [Hash] opts Deploy options
       # @option opts [String] :env {http://kadeploy3.gforge.inria.fr/ Kadeploy} environment to deploy
       # @option opts [Array] :nodes Specifies the nodes to deploy on
-      # @option opts [String] :keys Specifies the SSH keys to copy for the deployment
+      # @option opts [String] :keys Specifies the SSH keys to copy for the deployment. By default, the content of ~/.ssh/id_rsa.pub is used.
       # @option opts [Boolean] :wait Whether or not to wait until the deployment is done (default is false)
       # @return [G5KJSON] a job with deploy information as described in {Cute::G5K::G5KJSON job}
       def deploy(job, opts = {})

data/lib/cute/net-ssh.rb

@@ -1,4 +1,9 @@
-require 'net/ssh/multi'
+begin # load it without warnings
+  old_verbose, $VERBOSE = $VERBOSE, false
+  require 'net/ssh/multi'
+ensure
+  $VERBOSE = old_verbose
+end
 require 'logger'
 
 module Net; module SSH
@@ -91,6 +96,7 @@ module SessionActions
   #
   def exec!(command, &block)
 
+    Multi.logger.debug "SSH execution: #{command}"
     results = {}
 
     main =open_channel do |channel|
@@ -100,36 +106,38 @@ module SessionActions
 
         results[ch.connection.host] ||= {}
 
-        channel.on_data do |ch, data|
+        channel.on_data do |c, data|
           if block
-            block.call(ch, :stdout, data)
+            block.call(c, :stdout, data)
           else
-            results[ch.connection.host][:stdout] = data.strip
-            Multi.logger.debug("[#{ch.connection.host}] #{data.strip}")
+            results[c.connection.host][:stdout] ||= ""
+            results[c.connection.host][:stdout] += data.strip
+            Multi.logger.debug("[#{c.connection.host}] #{data.strip}")
           end
         end
-        channel.on_extended_data do |ch, type, data|
+        channel.on_extended_data do |c, type, data|
           if block
-            block.call(ch, :stderr, data)
+            block.call(c, :stderr, data)
           else
-            results[ch.connection.host][:stderr] = data.strip
-            Multi.logger.debug("[#{ch.connection.host}] #{data.strip}")
+            results[c.connection.host][:stderr] ||= ""
+            results[c.connection.host][:stderr] += data.strip
+            Multi.logger.debug("[#{c.connection.host}] #{data.strip}")
           end
         end
-        channel.on_request("exit-status") do |ch, data|
-          ch[:exit_status] = data.read_long
-          results[ch.connection.host][:status] = ch[:exit_status]
-          if ch[:exit_status] != 0
-            Multi.logger.info("execution of '#{command}' on #{ch.connection.host}
-                            failed with return status #{ch[:exit_status].to_s}")
-            if results[ch.connection.host][:stdout]
+        channel.on_request("exit-status") do |c, data|
+          c[:exit_status] = data.read_long
+          results[c.connection.host][:status] = c[:exit_status]
+          if c[:exit_status] != 0
+            Multi.logger.info("execution of '#{command}' on #{c.connection.host}
+                            failed with return status #{c[:exit_status].to_s}")
+            if results[c.connection.host][:stdout]
               Multi.logger.info("--- stdout dump ---")
-              Multi.logger.info(results[ch.connection.host][:stdout])
+              Multi.logger.info(results[c.connection.host][:stdout])
             end
 
-            if  results[ch.connection.host][:stderr]
+            if  results[c.connection.host][:stderr]
               Multi.logger.info("--stderr dump ---")
-              Multi.logger.info(results[ch.connection.host][:stderr])
+              Multi.logger.info(results[c.connection.host][:stderr])
             end
           end
         end

data/lib/cute/taktuk.rb

@@ -184,7 +184,7 @@ module Cute
       # Return just the value 0:(.*)
       def treat_value(string)
         tmp = string.split(":",2)
-        value = tmp[1].nil? ? "" : tmp[1]
+        return tmp[1].nil? ? "" : tmp[1]
       end
 
       def to_cmd

data/lib/cute/version.rb

@@ -1,3 +1,3 @@
 module Cute
-  VERSION = "0.6"
+  VERSION = "0.7"
 end

data/ruby-cute.gemspec

@@ -23,10 +23,10 @@ Gem::Specification.new do |s|
   s.add_development_dependency "yard", "~> 0.8"
   s.add_development_dependency "simplecov", "~> 0.7"
 
-  s.add_dependency 'rest-client', '1.6.7'
-  s.add_dependency 'json', '~> 1.8'
-  s.add_dependency 'ipaddress', '~>0.8'
-  s.add_dependency 'net-ssh-multi', '~>1.2'
+  s.add_dependency 'rest-client', '>= 1.6'
+  s.add_dependency 'json', '>= 1.8'
+  s.add_dependency 'ipaddress', '>= 0.8'
+  s.add_dependency 'net-ssh-multi', '>= 1.2'
 
   s.extra_rdoc_files = ['README.md', 'LICENSE']
   s.license = 'CeCILL-B'

data/spec/g5k_api_spec.rb

@@ -220,4 +220,17 @@ describe Cute::G5K::API do
     expect(subject.release_all(@rand_site)).to be true
   end
 
+  it "retries GET automatically when there is an error" do
+    expect(subject.get_jobs("tmpfail")).to be_truthy
+  end
+
+  it "retries POST automatically when there is an error" do
+    job = subject.reserve(:site => 'tmpfail', :wait => false)
+    expect(job).to be_truthy
+  end
+
+  it "retries DELETE automatically when there is an error" do
+    job = subject.rest.delete_json("3.0/sites/tmpfail/aa")
+    expect(job).to be_truthy
+  end
 end

data/spec/spec_helper.rb

@@ -55,6 +55,10 @@ RSpec.configure do |config|
     stub_request(:any,/^https:\/\/.*\:.*@api.grid5000.fr\/...\/sites\/non-found\/.*/).
       to_return(:status => 404)
 
+    stub_request(:any,/^https:\/\/.*\:.*@api.grid5000.fr\/...\/sites\/tmpfail\/.*/).
+      to_return(:status => 503).
+      to_return(:status => 200, :body => g5k_media_type.to_json, :headers => {})
+
     stub_request(:get,/^https:\/\/.*\:.*@api.grid5000.fr\/...\/sites\/.*vlans$/).
       to_return(:status => 200, :body => {'total' => 3, 'items' => [{'type' => "kavlan-local"},{'type' => "kvlan"}]}.to_json)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-cute
 version: !ruby/object:Gem::Version
-  version: '0.6'
+  version: '0.7'
 platform: ruby
 authors:
 - Algorille team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-20 00:00:00.000000000 Z
+date: 2017-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -112,56 +112,56 @@ dependencies:
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.7
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.7
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.8'
 - !ruby/object:Gem::Dependency
   name: ipaddress
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.8'
 - !ruby/object:Gem::Dependency
   name: net-ssh-multi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.2'
 description: Ruby library for controlling experiments
@@ -194,6 +194,7 @@ files:
 - examples/g5k-tutorial.md
 - examples/g5k_exp1.rb
 - examples/g5k_exp_virt.rb
+- examples/xp-bootstrap
 - lib/cute.rb
 - lib/cute/bash.rb
 - lib/cute/configparser.rb
@@ -233,7 +234,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Critically Useful Tools for Experiments
@@ -245,4 +246,3 @@ test_files:
 - spec/taktuk_spec.rb
 - test/test_bash.rb
 - test/test_execute.rb
-has_rdoc: