xp5k 0.0.10 → 0.0.11

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MTY5MDQ3MTllYzcxZjVmZGQyMjcxYzBhNzdiNmQ2MTJhNjQ0YjA5ZQ==
-  data.tar.gz: !binary |-
-    MWQ2ZjBkZWU3MjI0OTMxZDUwOGRjYjViNjlkYTY4NjQ2ZDUwMTk1OA==
+SHA1:
+  metadata.gz: 64a41367917d4c9b4445f795f9d71eb1a092b901
+  data.tar.gz: 26ea5183d5e2e675bccdb938443d2e464efbeda6
 SHA512:
-  metadata.gz: !binary |-
-    NDkwMDgwOTlkN2QzYWYxYjYxNzFkZTJhMWJhYjQ5M2M2Mjk3OGU1YTU3NjAz
-    MzlmMTMzNzMxNmNlZjJlMGVkNzM3MzA0NGQyY2QxYTcwNTJmY2ZmZjQ1N2Ez
-    YzU0NGM2Nzc2YzA5YzBjNTI0ZDg5Yzc0MDBiOWNkZGFkZGQwYmI=
-  data.tar.gz: !binary |-
-    OTMzOWE0N2I5MTZhMmYzNDE5YzFiMTllNWJmMzA1YWRiM2I3YjQ1ZmFiOTI3
-    NjUwMDIzZTkzODQ4YzY0ODQwNWNkODM1MzQxY2NmMzQxMDNhZTRiMmM5NjVl
-    NGE3MDNiZmI4MDdiNDUzNzg0Y2Q5NDFiMmQ2OWQ4MzQ2ZjY3YzI=
+  metadata.gz: a4e3e3c59d52bf281a1e0c8a55f328c5e7ecc4fd4791d63aff1e5a099255c46f11fbdb2267861ccde607a5ea7467d0d55de1d37d542c592ecb0822e34214903d
+  data.tar.gz: 84348914d0e589ef7859de46a980806b98a0067fcef293e5290aa8f8d67b6bb7c53b7b6d8f29e62235bf7c3d449c210bef4e08deda64b04a8409ef93daa4bf5b

data/Gemfile.lock

@@ -1,38 +1,79 @@
 PATH
   remote: .
   specs:
-    xp5k (0.0.1)
-      json (>= 1.5.1)
-      restfully (~> 0.6)
-      term-ansicolor (>= 1.0.7)
+    xp5k (0.0.11)
+      json (~> 1.8)
+      net-ssh-multi (~> 1.2)
+      rest-client (~> 1.8)
+      restfully (~> 1.1)
+      term-ansicolor (~> 1.3)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    addressable (2.3.2)
-    backports (2.6.5)
-    json (1.7.5)
-    mime-types (1.19)
-    rack (1.4.1)
-    rack-cache (1.2)
+    addressable (2.3.8)
+    backports (3.6.7)
+    bond (0.4.3)
+    domain_name (0.5.25)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    json (1.8.3)
+    mime-types (2.6.2)
+    net-ssh (3.0.1)
+    net-ssh-gateway (1.2.0)
+      net-ssh (>= 2.6.5)
+    net-ssh-multi (1.2.1)
+      net-ssh (>= 2.6.5)
+      net-ssh-gateway (>= 1.2.0)
+    netrc (0.11.0)
+    rack (1.6.4)
+    rack-cache (1.5.1)
       rack (>= 0.4)
-    rest-client (1.6.7)
-      mime-types (>= 1.16)
-    rest-client-components (1.2.0)
+    rb-readline (0.5.3)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    rest-client-components (1.4.0)
       rack (>= 1.0.1)
-      rest-client (>= 1.6.0, < 1.7.0)
-    restfully (0.8.8)
+      rest-client (>= 1.6.0)
+    restfully (1.1.1)
       addressable
       backports
       json (~> 1.5)
       rack-cache
+      rb-readline
       rest-client (~> 1.6)
       rest-client-components
-    term-ansicolor (1.0.7)
+      ripl (= 0.6.1)
+      ripl-color_streams
+      ripl-multi_line
+      ripl-play (~> 0.2.1)
+      ripl-short_errors
+    ripl (0.6.1)
+      bond (~> 0.4.0)
+    ripl-color_streams (0.1.2)
+      ripl (>= 0.2.8)
+    ripl-multi_line (0.3.1)
+      ripl (>= 0.3.6)
+    ripl-play (0.2.1)
+      ripl (>= 0.3.0)
+    ripl-short_errors (0.1.0)
+      ripl (>= 0.2.8)
+    term-ansicolor (1.3.2)
+      tins (~> 1.0)
+    tins (1.6.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (>= 1.0.0)
+  bundler (~> 1.10)
   xp5k!
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -1,18 +1,18 @@
 #### Table of Contents
 
-1. [Getting started](#Getting started)
-  * [Gemfile](#Gemfile)
-  * [Restfully](#Restfully)
-  * [Ssh config](#Ssh config)
-2. [Samples](#Samples)
-  * [Hello date](#Hello date)
-  * [Xp5k Roles](#Xp5k Roles)
-  * [Nested roels](#Nested roles)
-  * [Patterns](#Patterns)
-  * [Get the deployed nodes](#Get the deployed nodes)
-  * [Automatic redeployment](#Automatic redeployment)
-  * [Vlan support](#Vlan support)
-  * [Non deploy jobs](#Non deploy jobs)
+1. [Getting started](#getting-started)
+  * [Gemfile](#gemfile)
+  * [Restfully](#restfully)
+  * [Ssh config](#ssh-config)
+2. [Samples](#samples)
+  * [Hello date](#hello-date)
+  * [Xp5k Roles](#xp5k-roles)
+  * [Nested roels](#nested-roles)
+  * [Patterns](#patterns)
+  * [Get the deployed nodes](#get-the-deployed-nodes)
+  * [Automatic redeployment](#automatic-redeployment)
+  * [Vlan support](#vlan-support)
+  * [Non deploy jobs](#non-deploy-jobs)
 
 # Getting started
 
@@ -83,9 +83,9 @@ set :g5k_user, "msimonin"
 # gateway
 set :gateway, "#{g5k_user}@access.grid5000.fr"
 # These keys will used to access the gateway and nodes
-ssh_options[:keys]= [File.join(ENV["HOME"], ".ssh", "id_rsa"), File.join(ENV["HOME"], ".ssh", "id_rsa_insideg5k")]
+ssh_options[:keys]= [File.join(ENV["HOME"], ".ssh", "id_rsa")]
 # # This key will be installed on nodes
-set :ssh_public,  File.join(ENV["HOME"], ".ssh", "id_rsa_insideg5k.pub")
+set :ssh_public,  File.join(ENV["HOME"], ".ssh", "id_rsa.pub")
 
 XP5K::Config.load
 
@@ -191,11 +191,11 @@ role :server do
   @myxp.role_with_name('server').servers
 end
 
-role :server do
+role :frontend do
   @myxp.role_with_name('frontend').servers
 end
 
-role :server do
+role :clients do
   @myxp.role_with_name('clients').servers
 end
 ```
@@ -223,7 +223,7 @@ You can also define nested roles (only 1 level) :
 
 You can select nodes matching a pattern (`String` or `Regexp`) :
 
-```Ruby
+```ruby
 roles = []
 scenario['clusters'].each do |cluster|
   roles << XP5K::Role.new({
@@ -295,14 +295,12 @@ You can control this behaviour passing special keys in the deployment hash.
 
 ## Non deploy jobs
 
-Here we fill the ```types``` field with ```allow_classic_ssh```. 
+Here we fill the ```types``` field with ```allow_classic_ssh```.
 
 ```ruby
 @myxp.define_job({
   :resources  => ["nodes=1, walltime=1"],
   :site       => "rennes",
-  :retry      => true,
-  :goal       => "100%",
   :types      => ["allow_classic_ssh"],
   :name       => "init" ,
   :command    => "sleep 86400"

data/lib/xp5k/rake/dsl.rb

@@ -0,0 +1,163 @@
+require 'xp5k/role'
+require 'net/ssh/multi'
+require 'timeout'
+require 'thread'
+
+module XP5K
+  module Rake
+    module DSL
+
+      private
+
+      def role(*args, &block)
+        hosts = []
+        procblock = nil
+        if block_given?
+          raise 'Arguments not allowed with block' unless args[1].nil?
+          procblock = block
+        else
+          case args[1]
+          when String
+            hosts = [args[1]]
+          when Array
+            hosts = args[1]
+          else
+            raise "Role <#{args.first}> argument must be a String or an Array"
+          end
+        end
+
+        XP5K::Role.new(name: args.first, size: hosts.length, servers: hosts, proc: procblock).add
+
+      end
+
+
+      def roles(*args)
+        hosts = []
+        args.each do |rolename|
+          hosts << XP5K::Role.findByName(rolename).servers
+        end
+        hosts.flatten!
+      end
+
+      #
+      def on(hosts, *args, &block)
+
+        logs = Hash.new { |h,k| h[k] = '' }
+        errors = Hash.new { |h,k| h[k] = '' }
+        ssh_session = {}
+        current_server = ""
+        all_connected = false
+        failed_servers = []
+        hosts = [hosts] if hosts.class == String
+
+        XP5K::Config[:user] ||= ENV["USER"]
+        XP5K::Config[:gateway] ||= XP5K::Config[:user] + "@access.grid5000.fr"
+
+        commands = (args.last.class == String or args.last.class == Array) ? (args.last.class == String ? [args.pop] : args.pop ) : []
+        options = args.last.class == Hash ? args.pop : {}
+        options[:user] ||= 'root'
+
+        if block_given?
+          case result = yield
+          when String
+            commands << result
+          when Array
+            commands.push(result).flatten!
+          else
+            raise "<on> block must return String or Array"
+          end
+        end
+
+        until all_connected
+          failed = false
+          gateway_options = {}
+          gateway_options[:config] == XP5K::Config[:ssh_config] ? true : XP5K::Config[:ssh_config]
+          gateway_user, gateway_host = XP5K::Config[:gateway].match(/^(?:([^;,:=]+)@|)(.*?)$/)[1,2]
+          gateway = Net::SSH::Gateway.new(gateway_host, gateway_user)
+          workq = Queue.new
+          hosts.each{ |host| workq << host }
+          workers = (0...10).map do
+            Thread.new do
+              begin
+                while host = workq.pop(true)
+                  begin
+                    timeout(5) do
+                      ssh_session[host] = gateway.ssh(host, options[:user])
+                      puts "Connected to #{host}..."
+                    end
+                  rescue Timeout::Error, Net::SSH::Disconnect, Exception => e
+                    puts "Removing #{host} (#{e.message})..."
+                    hosts.delete host
+                    failed_servers << host
+                    failed = true
+                  end
+                end
+
+              rescue ThreadError
+              end
+
+            end
+          end; "ok"
+          workers.map(&:join); "ok"
+
+          all_connected = true if !failed
+        end
+
+        workq = Queue.new
+        hosts.each{ |host| workq << host }
+        workers = (0...10).map do
+          Thread.new do
+            begin
+              while host = workq.pop(true)
+                begin
+                  commands.each do |command|
+                    puts "[command][#{host}] #{command}"
+                    ssh_session[host].exec!(command) do |channel, stream, data|
+                      logs[host] << data
+                      errors[host] << data if stream == :err
+                      puts "[#{stream}][#{host}] #{data}" if data.chomp != ""
+                    end
+                  end
+                rescue Exception => e
+                  puts "[#{host}] " + e.message
+                end
+              end
+            rescue ThreadError
+            end
+          end
+        end; "ok"
+        workers.map(&:join); "ok"
+
+        # Print the result sorting by hostname
+        errors.sort.each do |error|
+          puts "---- stderr on #{error.first} #{"-" * (get_width - error.first.length - 16)} "
+          puts "#{error[1]}"
+        end
+        logs.sort.each do |key, value|
+          puts "---- #{key} #{"-" * (get_width - key.length - 6)}"
+          puts value
+        end
+        puts "Servers unreachable : #{failed_servers.inspect}" if !failed_servers.empty?
+
+        # Clean all ssh connections
+        puts "Closing ssh connections..."
+        hosts.each do |host|
+          ssh_session[host].close
+          gateway.close ssh_session[host].transport.port
+        end
+        gateway.shutdown!
+
+      end
+
+      def run(command)
+
+      end
+
+      def get_width
+        result = `tput cols`
+        result.to_i
+      end
+
+    end
+  end
+end

data/lib/xp5k/rake.rb

@@ -0,0 +1,3 @@
+require 'xp5k/rake/dsl'
+
+self.extend XP5K::Rake::DSL

data/lib/xp5k/role.rb

@@ -1,6 +1,6 @@
 class XP5K::Role
 
-  attr_accessor :name, :size, :desc, :servers, :jobid, :inner, :pattern
+  attr_accessor :name, :size, :desc, :servers, :jobid, :inner, :pattern, :proc
 
   @@roles = []
 
@@ -9,6 +9,7 @@ class XP5K::Role
     @inner   = false
     @servers = []
     @desc    = ""
+    @proc    = nil
 
     # Required parameters
     %w{ name size }.each do |param|
@@ -20,11 +21,36 @@ class XP5K::Role
     end
 
     # Optional parameters
-    %w{ desc servers inner pattern }.each do |param|
+    %w{ desc servers inner pattern proc }.each do |param|
       instance_variable_set("@#{param}", options[param.to_sym]) if options[param.to_sym]
     end
   end
 
+  def add
+    @@roles << self
+  end
+
+  def servers
+    if self.proc.class == Proc
+      self.callproc
+    end
+    @servers
+  end
+
+  def callproc
+    case result = self.proc.call
+    when String
+      hosts = [result]
+    when Array
+      hosts = result
+    else
+      raise "Role <#{args.first}> block must return String or Array"
+    end
+    self.size = hosts.length
+    self.servers = hosts
+  end
+
+
   def self.create_roles(job, job_definition)
     # Definition will return list of roles
     roles = []
@@ -81,6 +107,10 @@ class XP5K::Role
     @@roles
   end
 
+  def self.listnames()
+    @@roles.map { |role| role.name }
+  end
+
   def self.findByName(name)
     roles = @@roles.select { |x| x.name == name }
     roles.empty? ? nil : roles.first

data/lib/xp5k/version.rb

@@ -1,3 +1,3 @@
 module XP5K
-  VERSION='0.0.10'
+  VERSION='0.0.11'
 end

data/xp5k.gemspec

@@ -10,17 +10,18 @@ Gem::Specification.new do |s|
   s.email       = ['pascal.morillon@irisa.fr']
   s.license     = 'MIT'
   s.homepage    = 'https://github.com/pmorillon/xp5k'
-  s.summary     = %q{A small Grid'5000 helper to submit jobs and deploy environments via REST API}
+  s.summary     = %q{A small Grid'5000 helper}
   s.description = %q{A small Grid'5000 helper to submit jobs and deploy environments via REST API}
 
   s.required_rubygems_version = '>= 1.3.6'
 
-  s.add_dependency 'rest-client', '1.6.8' # SSL problems with newer version
-  s.add_dependency 'restfully', '>1.0.0'
-  s.add_dependency 'term-ansicolor', '>= 1.0.7'
-  s.add_dependency 'json', '>= 1.5.1'
+  s.add_dependency 'rest-client', '~> 1.8' # SSL problems with newer version
+  s.add_dependency 'restfully', '~>1.1'
+  s.add_dependency 'term-ansicolor', '~> 1.3'
+  s.add_dependency 'json', '~> 1.8'
+  s.add_dependency 'net-ssh-multi', '~> 1.2'
 
-  s.add_development_dependency 'bundler', '>= 1.0.0'
+  s.add_development_dependency 'bundler', '~> 1.10'
 
   s.files         = `git ls-files`.split("\n")
   s.require_paths = ['lib']

metadata

@@ -1,85 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: xp5k
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - Pascal Morillon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-03 00:00:00.000000000 Z
+date: 2015-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.8
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.8
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: restfully
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>'
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>'
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.7
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.7
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.1
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.1
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: net-ssh-multi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.10'
 description: A small Grid'5000 helper to submit jobs and deploy environments via REST
   API
 email:
@@ -88,7 +102,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - Gemfile.lock
 - LICENCE
@@ -96,6 +110,8 @@ files:
 - lib/xp5k.rb
 - lib/xp5k/config.rb
 - lib/xp5k/exceptions.rb
+- lib/xp5k/rake.rb
+- lib/xp5k/rake/dsl.rb
 - lib/xp5k/role.rb
 - lib/xp5k/version.rb
 - lib/xp5k/xp.rb
@@ -110,12 +126,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
@@ -123,6 +139,5 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: A small Grid'5000 helper to submit jobs and deploy environments via REST
-  API
+summary: A small Grid'5000 helper
 test_files: []