ruby-cute 0.13 → 0.24

data/lib/cute/g5k_api.rb

@@ -24,7 +24,7 @@ module Cute
     # - {Cute::G5K::Unauthorized} it means that there is an authentication problem.
     # - {Cute::G5K::EventTimeout} this exception is triggered by the methods that wait for events such as:
     #   job submission and environment deployment.
-    class Error < Exception
+    class Error < StandardError
       attr_accessor :orig # Original exception
 
       def initialize(message = nil, object = nil)
@@ -63,6 +63,7 @@ module Cute
     #
     #    end
     class BadRequest < Error
+      attr_accessor :inner_url, :inner_code, :inner_title, :inner_message
     end
 
     # It wraps all Restclient exceptions with http codes: 403, 405,406, 412, 415, 500, 502, 503 and 504.
@@ -205,7 +206,8 @@ module Cute
       # @param user [String] user if authentication is needed
       # @param pass [String] password if authentication is needed
       # @param on_error [Symbol] option to deactivate the {Cute::G5K::RequestFailed RequestFailed} exceptions
-      def initialize(uri,api_version,user,pass,on_error)
+      # @param timeout [Integer] timeout for Rest request
+      def initialize(uri,api_version,user,pass,on_error,timeout)
         @user = user
         @pass = pass
         @api_version = api_version.nil? ? "stable" : api_version
@@ -217,9 +219,8 @@ module Cute
           @endpoint = "https://#{user_escaped}:#{pass_escaped}@#{uri.split("https://")[1]}"
         end
 
-        machine =`uname -ov`.chop
         @user_agent = "ruby-cute/#{VERSION} Ruby/#{RUBY_VERSION}"
-        @api = RestClient::Resource.new(@endpoint, :timeout => 30,:verify_ssl => false)
+        @api = RestClient::Resource.new(@endpoint, :timeout => timeout,:verify_ssl => false)
         # some versions of restclient do not verify by default SSL certificates , :verify_ssl => true)
         # SSL verify is disabled due to Grid'5000 API certificate problem
         @on_error = on_error
@@ -241,7 +242,7 @@ module Cute
       def get_json(path)
         return G5KJSON.parse(get_raw(path))
       end
-      
+
       # @return [String] the HTTP response
       # @param path [String] this complements the URI to address to a specific resource
       def get_raw(path)
@@ -320,7 +321,7 @@ module Cute
       end
 
       # Issues a Cute::G5K exception according to the http status code
-      def handle_exception(e, req = nil)
+      def handle_exception(e, _req = nil)
         puts("Error: #{$!}")
         puts("Backtrace:\n\t"+e.backtrace.join("\n\t"))
         if e.respond_to? :http_code
@@ -338,11 +339,24 @@ module Cute
         case e.http_code
 
         when  400
-          raise BadRequest.new("Bad request", e)
+          br = BadRequest.new("Bad request (error 400): #{e.http_body}", e)
+          if e.http_body =~ /^Request to ([^ ]*) failed with status 400: (.*)$/m
+            br.inner_url = $1
+            json = $2
+            begin
+              d = JSON::parse(json)
+              br.inner_code = d['code']
+              br.inner_title = d['title']
+              br.inner_message = d['message']
+            rescue JSON::ParserError
+              # Ignore error
+            end
+          end
+          raise br
         when 404
-          raise NotFound.new("Resource not found", e)
+          raise NotFound.new("Resource not found (error 404): #{e.http_body}", e)
         when 401
-          raise Unauthorized.new("Authentication problem",e)
+          raise Unauthorized.new("Authentication problem (error 401): #{e.http_body}",e)
         else
           if @on_error == :ignore
             return nil
@@ -555,6 +569,7 @@ module Cute
       # @option params [String] :password Password to access the REST API
       # @option params [Symbol] :on_error Set to :ignore if you want to ignore {Cute::G5K::RequestFailed ResquestFailed} exceptions.
       # @option params [Boolean] :debug Activate the debug mode
+      # @option params [Integer] :timeout Set the timeout in sec, default is 30 sec
       def initialize(params={})
         config = {}
         default_file = "#{ENV['HOME']}/.grid5000_api.yml"
@@ -573,9 +588,10 @@ module Cute
         @api_version = params[:version] || config["version"] || "stable"
         @logger = nil
         @debug = params[:debug] || false
+        @timeout = params[:timeout] || 30
 
         begin
-          @g5k_connection = G5KRest.new(@uri,@api_version,@user,@pass,params[:on_error])
+          @g5k_connection = G5KRest.new(@uri,@api_version,@user,@pass,params[:on_error], @timeout)
         rescue => e
 
           msg_create_file = ""
@@ -701,7 +717,9 @@ module Cute
       # @param site [String] a valid Grid'5000 site name
       # @param uid [String] user name in Grid'5000
       # @param states [Array] or [String] jobs state: running, waiting (multiple states can be specified)
-      def get_jobs(site, uid = nil, states = nil)
+      # @param details [String] pass "resources=yes to the query to get the list of resources
+      # @param recurse [String] after fetching the list of jobs, fetch details about each job individualy
+      def get_jobs(site, uid = nil, states = nil, details = false, recurse = true)
 
         parameters = []
         if states then
@@ -709,12 +727,18 @@ module Cute
           parameters.push("state=#{states.join(",")}")
         end
         parameters.push("user=#{uid}") if uid
-        parameters.push("limit=25") if (states.nil? and uid.nil?)
+        parameters.push("limit=1000000")
+        parameters.push("resources=yes") if details
 
+        info(debug_cmd(api_uri("/sites/#{site}/jobs?#{parameters.join("&")}"),"GET"), :debug)
         jobs = @g5k_connection.get_json(api_uri("/sites/#{site}/jobs?#{parameters.join("&")}")).items
-        jobs.map{ |j| @g5k_connection.get_json(j.rel_self)}
-        # This request sometime is could take a little long when all jobs are requested
-        # The API return by default 50 the limit was set to 25 (e.g., 23 seconds).
+        if recurse
+          jobs.map! do |j|
+            info(debug_cmd(j.rel_self, "GET"), :debug)
+            @g5k_connection.get_json(j.rel_self)
+          end
+        end
+        jobs.to_a
       end
 
       # @return [Hash] the last 50 deployments performed in a Grid'5000 site
@@ -976,6 +1000,7 @@ module Cute
       # @option opts [String] :cmd The command to execute when the job starts (e.g. ./my-script.sh).
       # @option opts [String] :cluster Valid Grid'5000 cluster
       # @option opts [String] :queue A specific job queue
+      # @option opts [String] :project A specific OAR project
       # @option opts [Array]  :subnets 1) prefix_size, 2) number of subnets
       # @option opts [String] :env Environment name for {http://kadeploy3.gforge.inria.fr/ Kadeploy}
       # @option opts [String] :vlan VLAN type and number: kavlan-local, kavlan, kavlan-topo, etc
@@ -990,12 +1015,12 @@ module Cute
         # checking valid options
         valid_opts = [:site, :cluster, :switches, :cpus, :cores, :nodes, :walltime, :cmd,
                       :type, :name, :subnets, :env, :vlan, :num_vlan,:properties, :resources,
-                      :reservation, :wait, :keys, :queue, :env_user]
+                      :reservation, :wait, :keys, :queue, :project, :env_user]
         unre_opts = opts.keys - valid_opts
         raise ArgumentError, "Unrecognized option #{unre_opts}" unless unre_opts.empty?
 
         nodes = opts.fetch(:nodes, 1)
-        walltime = opts.fetch(:walltime, '01:00:00')
+        walltime = opts[:walltime]
         site = opts[:site]
         type = opts.fetch(:type, [])
         name = opts.fetch(:name, 'rubyCute job')
@@ -1013,6 +1038,7 @@ module Cute
         type = [type] if type.is_a?(Symbol)
         keys = opts[:keys]
         queue = opts[:queue]
+        project = opts[:project]
         vlan = opts[:vlan]
         num_vlan = opts.fetch(:num_vlan, 1)
 
@@ -1027,14 +1053,11 @@ module Cute
           raise ArgumentError, "VLAN type not available in site #{site}" unless available_vlans.include?(vlan)
         end
 
-        raise 'At least nodes, time and site must be given'  if [nodes, walltime, site].any? { |x| x.nil? }
-
-        raise 'nodes should be an integer or a string containing either ALL or BEST' unless (nodes.is_a?(Fixnum) or ["ALL","BEST"].include?(nodes))
+        raise 'At least nodes and site must be given'  if [nodes, site].any? { |x| x.nil? }
 
-        secs = walltime.to_secs
-        walltime = walltime.to_time
+        raise 'nodes should be an integer or a string containing either ALL or BEST' unless (nodes.is_a?(Integer) or ["ALL","BEST"].include?(nodes))
 
-        command = "sleep #{secs}" if command.nil?
+        command = "sleep infinity" if command.nil?
 
         if resources == ""
           resources = "/switch=#{switches}" unless switches.nil?
@@ -1043,14 +1066,16 @@ module Cute
           resources += "/core=#{cores}" unless cores.nil?
 
           if cluster
-            resources = (cluster.is_a?(Fixnum) ? "/cluster=#{cluster}" : "{cluster='#{cluster}'}") + resources
+            resources = (cluster.is_a?(Integer) ? "/cluster=#{cluster}" : "{cluster='#{cluster}'}") + resources
           end
 
           resources = "{type='#{vlan}'}/vlan=#{num_vlan}+" + resources unless vlan.nil?
           resources = "slash_#{subnets[0]}=#{subnets[1]}+" + resources unless subnets.nil?
         end
 
-        resources += ",walltime=#{walltime}" unless resources.include?("walltime")
+        if walltime
+          resources += ",walltime=#{walltime}" unless resources.include?("walltime")
+        end
 
         payload = {
                    'resources' => resources,
@@ -1058,12 +1083,13 @@ module Cute
                    'command' => command
                   }
 
-        info "Reserving resources: #{resources} (type: #{type}) (in #{site})"
+        info "Reserving resources: #{resources} (types: #{type.join(' ')}) (in #{site})"
 
         payload['properties'] = properties unless properties.nil?
         payload['types'] = type.map{ |t| t.to_s} unless type.nil?
         type.map!{|t| t.to_sym}  unless type.nil?
         payload['queue'] = queue if queue
+        payload['project'] = project if project
 
         unless type.include?(:deploy)
           if opts[:keys]
@@ -1073,18 +1099,10 @@ module Cute
 
         if reservation
           payload['reservation'] = reservation
-          info "Starting this reservation at #{reservation}"
         end
 
-        begin
-          info debug_cmd(api_uri("sites/#{site}/jobs"),"POST",payload.to_json), :debug
-          r = @g5k_connection.post_json(api_uri("sites/#{site}/jobs"),payload)  # This makes reference to the same class
-        rescue Error => e
-          info "Fail to submit job"
-          info e.message
-          e.http_body.split("\\n").each{ |line| info line}
-          raise
-        end
+        info debug_cmd(api_uri("sites/#{site}/jobs"),"POST",payload.to_json), :debug
+        r = @g5k_connection.post_json(api_uri("sites/#{site}/jobs"),payload)  # This makes reference to the same class
 
         job = @g5k_connection.get_json(r.rel_self)
         job = wait_for_job(job) if opts[:wait] == true
@@ -1274,7 +1292,7 @@ module Cute
 
         job["deploy"].map!{  |d| d.refresh(@g5k_connection) }
 
-        filter.keep_if{ |k,v| v} # removes nil values
+        filter.keep_if{ |_k,v| v} # removes nil values
         if filter.empty?
           status = job["deploy"].map{ |d| d["status"] }
         else
@@ -1352,7 +1370,7 @@ module Cute
       # @return [Array] machines that did not deploy successfully
       # @param deploy_info [Hash] deployment structure information
       def check_deployment(deploy_info)
-        deploy_info["result"].select{ |p,v|  v["state"] == "KO"}.keys
+        deploy_info["result"].select{ |_p,v|  v["state"] == "KO"}.keys
       end
 
       # Returns a valid  URI using the current G5K API version.

data/lib/cute/net-ssh-exec3.rb

@@ -14,17 +14,18 @@ class Net::SSH::Connection::Session
     res[:stderr] = ""
     res[:exit_code] = nil
     res[:exit_signal] = nil
+    ts = Time::now
     open_channel do |channel|
-      channel.exec(command) do |ch, success|
+      channel.exec(command) do |_ch, success|
         unless success
           abort "FAILED: couldn't execute command (ssh.channel.exec)"
         end
-        channel.on_data do |ch,data|
+        channel.on_data do |_ch,data|
           print data unless o[:no_output]
           res[:stdout]+=data
         end
 
-        channel.on_extended_data do |ch,type,data|
+        channel.on_extended_data do |_ch,_type,data|
           print data unless o[:no_output]
           if o[:merge_outputs]
             res[:stdout]+=data
@@ -33,14 +34,16 @@ class Net::SSH::Connection::Session
           end
         end
 
-        channel.on_request("exit-status") do |ch,data|
+        channel.on_request("exit-status") do |_ch,data|
           res[:exit_code] = data.read_long
-          puts "EXITCODE: #{res[:exit_code]}" unless o[:no_log]
+          d = sprintf("%.1f", Time::now - ts)
+          puts "EXITCODE: #{res[:exit_code]} (duration: #{d}s)" unless o[:no_log]
         end
 
-        channel.on_request("exit-signal") do |ch, data|
+        channel.on_request("exit-signal") do |_ch, data|
           res[:exit_signal] = data.read_long
-          puts "EXITSIGNAL: #{res[:exit_signal]}" unless o[:no_log]
+          d = sprintf("%.1f", Time::now - ts)
+          puts "EXITSIGNAL: #{res[:exit_signal]} (duration: #{d}s)" unless o[:no_log]
         end
       end
     end

data/lib/cute/net-ssh.rb

@@ -115,7 +115,7 @@ module SessionActions
             Multi.logger.debug("[#{c.connection.host}] #{data.strip}")
           end
         end
-        channel.on_extended_data do |c, type, data|
+        channel.on_extended_data do |c, _type, data|
           if block
             block.call(c, :stderr, data)
           else
@@ -129,7 +129,7 @@ module SessionActions
           results[c.connection.host][:status] = c[:exit_status]
           if c[:exit_status] != 0
             Multi.logger.info("execution of '#{command}' on #{c.connection.host}
-                            failed with return status #{c[:exit_status].to_s}")
+                            failed with return status #{c[:exit_status]}")
             if results[c.connection.host][:stdout]
               Multi.logger.info("--- stdout dump ---")
               Multi.logger.info(results[c.connection.host][:stdout])

data/lib/cute/net.rb

@@ -13,14 +13,12 @@ module Cute
     end
 
     def Network::wait_open_port(host, port, timeout = 120)
-      def now()
-        return Time.now.to_f
-      end
-      bound = now() + timeout
-      while now() < bound do
-        t = now()
+      now = -> { return Time.now.to_f }
+      bound = now.call + timeout
+      while now.call < bound do
+        t = now.call
         return true if port_open?(host, port)
-        dt = now() - t
+        dt = now.call - t
         sleep(0.5 - dt) if dt < 0.5
       end
       return false

data/lib/cute/synchronization.rb

@@ -1,8 +1,6 @@
 module Cute
    module Synchronization
-    require 'thread'
-
-    class Semaphore
+     class Semaphore
       def initialize(max)
         @lock = Mutex.new
         @cond = ConditionVariable.new

data/lib/cute/taktuk.rb

@@ -123,7 +123,6 @@ module Cute
           yield  taktuk_cmd
           taktuk_cmd.loop unless taktuk_cmd.commands.empty?
           taktuk_cmd.free! if taktuk_cmd
-          taktuk_cmd = nil
         end
       else
         return taktuk_cmd
@@ -372,9 +371,9 @@ module Cute
       def to_cmd
         self.inject([]) do |ret,val|
           if val =~ /^\[(.*)\]$/
-            ret += ['[',Regexp.last_match(1).strip,']']
+            ret + ['[',Regexp.last_match(1).strip,']']
           else
-            ret += val.split(' ')
+            ret + val.split(' ')
           end
         end
       end
@@ -417,7 +416,7 @@ module Cute
 
         @streams.types.each{ |name|
           @args << '-o'
-          @args << "#{name.to_s}=#{@streams.to_cmd}"
+          @args << "#{name}=#{@streams.to_cmd}"
         }
 
         connector = build_connector
@@ -563,7 +562,7 @@ module Cute
       #    tak.input(:file => "test_file.tar")
       def input(opts = {})
         mode = "broadcast"
-        @commands << "#{mode} input #{opts.keys.first.to_s}"
+        @commands << "#{mode} input #{opts.keys.first}"
         @commands << "[ #{opts.values.first} ]"
         @commands << ';'
       end

data/lib/cute/version.rb

@@ -1,3 +1,3 @@
 module Cute
-  VERSION = "0.13"
+  VERSION = "0.24"
 end

data/ruby-cute.gemspec

@@ -5,9 +5,9 @@ require 'cute/version'
 Gem::Specification.new do |s|
   s.name        = "ruby-cute"
   s.version     = Cute::VERSION
-  s.authors     = ["Algorille team"]
-  s.email       = "ruby-cute-staff@lists.gforge.inria.fr"
-  s.homepage    = "http://ruby-cute.gforge.inria.fr/"
+  s.authors     = ["Algorille/Madynes/RESIST teams at Inria/LORIA"]
+  s.email       = "lucas.nussbaum@inria.fr"
+  s.homepage    = "http://ruby-cute.github.io/"
   s.summary     = "Critically Useful Tools for Experiments"
   s.description = "Ruby library for controlling experiments"
   s.required_rubygems_version = ">= 1.3.6"
@@ -26,7 +26,10 @@ Gem::Specification.new do |s|
   s.add_dependency 'rest-client', '>= 1.6'
   s.add_dependency 'json', '>= 1.8'
   s.add_dependency 'ipaddress', '>= 0.8'
+  s.add_dependency 'net-ssh', '>= 3.2'
   s.add_dependency 'net-ssh-multi', '>= 1.2'
+  s.add_dependency 'net-scp', '>= 1.2'
+  s.add_dependency 'peach', '>= 0.5.1'
 
   s.extra_rdoc_files = ['README.md', 'LICENSE']
   s.license = 'CeCILL-B'

data/spec/g5k_api_check_spec.rb

@@ -4,7 +4,7 @@ require 'spec_helper'
 
 describe Cute::G5K::API do
 
-  subject { g5k = ENV['TEST_REAL'].nil?? Cute::G5K::API.new(:user => "test") : Cute::G5K::API.new() }
+  subject { ENV['TEST_REAL'].nil?? Cute::G5K::API.new(:user => "test") : Cute::G5K::API.new() }
 
   before :each do
     if ENV['TEST_REAL']

data/spec/g5k_api_spec.rb

@@ -4,9 +4,9 @@ require 'spec_helper'
 describe Cute::G5K::API do
 
   if ENV['TEST_REAL']
-    subject { g5k = ENV['DEBUG'].nil?? Cute::G5K::API.new() : Cute::G5K::API.new(:debug => true) }
+    subject { ENV['DEBUG'].nil?? Cute::G5K::API.new() : Cute::G5K::API.new(:debug => true) }
   else
-    subject { g5k = ENV['DEBUG'].nil?? Cute::G5K::API.new(:user => "test") : Cute::G5K::API.new(:user => "test",:debug => true) }
+    subject { ENV['DEBUG'].nil?? Cute::G5K::API.new(:user => "test") : Cute::G5K::API.new(:user => "test",:debug => true) }
   end
 
   let(:sites) { subject.site_uids}
@@ -79,15 +79,6 @@ describe Cute::G5K::API do
     expect{subject.get_jobs("not-found")}.to raise_error(Cute::G5K::NotFound)
   end
 
-  it "raises a bad request error" do
-    expect{ subject.reserve(:site => @rand_site, :resources =>"/slash_22=1+{nonsense}")}.to raise_error(Cute::G5K::BadRequest)
-#    expect{ subject.reserve(:site => @rand_site, :resources =>"{ib30g='YES'}/nodes=2")}.to raise_error(Cute::G5K::BadRequest)
-  end
-
-  it "raises a bad request using OAR API" do
-    expect{subject.reserve(:site => @rand_site, :resources =>"nodes=1",:keys => "~/jobkey_nonexisting")}.to raise_error(Cute::G5K::BadRequest)
-  end
-
 #   it "raises an exception at deploying" do
 #     expect{ subject.reserve(:site => @rand_site, :nodes => 1, :env => "nonsense")}.to raise_error(Cute::G5K::RequestFailed)
 #   end
@@ -199,7 +190,6 @@ describe Cute::G5K::API do
   end
 
   it "submit and does not wait for the reservation" do
-    cluster = subject.cluster_uids(@rand_site).first
     job = subject.reserve(:site => @rand_site, :wait => false)
     job = subject.wait_for_job(job, :wait_time => 600)
     expect(job).to include('state' => "running")

data/spec/spec_helper.rb

@@ -30,6 +30,7 @@ class FakeG5KResponse < Hash
                 'nodes' => {"node1" => {"hard"=> "alive", "soft"=>"busy"}}
                }
   def initialize(num_items = 2)
+    super
     MEDIA_TYPE.each { |key,value| self[key] = value}
     self['items'] = []
     num_items.times.each{ self['items'].push(MEDIA_TYPE) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-cute
 version: !ruby/object:Gem::Version
-  version: '0.13'
+  version: '0.24'
 platform: ruby
 authors:
-- Algorille team
-autorequire: 
+- Algorille/Madynes/RESIST teams at Inria/LORIA
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2024-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: net-ssh-multi
   requirement: !ruby/object:Gem::Requirement
@@ -164,10 +178,39 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: peach
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.1
 description: Ruby library for controlling experiments
-email: ruby-cute-staff@lists.gforge.inria.fr
+email: lucas.nussbaum@inria.fr
 executables:
 - cute
+- grd
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -181,8 +224,9 @@ files:
 - README.md
 - Rakefile
 - bin/cute
+- bin/grd
+- debian/.gitattributes
 - debian/changelog
-- debian/compat
 - debian/control
 - debian/copyright
 - debian/ruby-cute.docs
@@ -216,11 +260,11 @@ files:
 - spec/taktuk_spec.rb
 - test/test_bash.rb
 - test/test_execute.rb
-homepage: http://ruby-cute.gforge.inria.fr/
+homepage: http://ruby-cute.github.io/
 licenses:
 - CeCILL-B
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -235,9 +279,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.3.15
+signing_key:
 specification_version: 4
 summary: Critically Useful Tools for Experiments
 test_files:

data/debian/compat

@@ -1 +0,0 @@
-8