ruby-cute 0.0.1 → 0.0.2

data/examples/g5k_exp1.rb

@@ -0,0 +1,41 @@
+require 'cute'
+
+# This example tests two libraries for the execution of commands over several machines.
+# These two  libraries use two different approaches: 1) SSH, 1) TakTuk.
+# This example uses Grid'5000 but it can be used with any set of machines that can be accessed through SSH.
+
+g5k = Cute::G5K::API.new()
+# We reuse a job if there is one available.
+if g5k.get_my_jobs("grenoble").empty? then
+  job = g5k.reserve(:nodes => 5, :site => 'grenoble', :walltime => '00:30:00')
+else
+  job =g5k.get_my_jobs("grenoble").first
+end
+
+nodes = job["assigned_nodes"]
+
+results = {}
+
+# please change user by your Grid'5000 user.
+Net::SSH::Multi.start do |session|
+
+  nodes.each{ |node| session.use "user@#{node}" }
+  session.exec 'hostname'
+  session.loop
+  results = session.exec! 'df'
+  session.exec 'uptime'
+end
+
+puts results
+
+Cute::TakTuk.start(nodes,:user => "user" ) do |tak|
+
+  results = tak.exec!("hostname")
+  tak.loop()
+  tak.exec("df")
+  tak.exec("uname -r")
+
+end
+
+
+puts results

data/examples/g5k_exp_virt.rb

@@ -0,0 +1,129 @@
+# = Virtualization example on Grid'5000
+
+# This script implements the example virtualization and subnet reservation in Grid'5000.
+# In a nutshell, a machine and a sub network is reserved per site.
+# Configuration files are generated to make kvm vms take the IPs addresses reserved, then several virtual
+# machines are booted per machine (default 2). At the end all virtual machines are contacted via SSH.
+# The example is described in https://www.grid5000.fr/mediawiki/index.php/Virtualization_on_Grid%275000
+
+require 'grid5000/subnets'
+require 'cute'
+require 'net/scp'
+
+g5k = Cute::G5K::API.new()
+# We reuse a job if there is one available.
+G5K_SITES = [:lille, :rennes, :lyon, :grenoble]
+
+threads = []
+jobs = {}
+grid5000_opt = {:user => "oar", :keys => ["~/.ssh/id_rsa"], :port => 6667 }
+num_vm = 2 # Number of vms per site
+
+G5K_SITES.each{ |site|
+  threads << Thread.new {
+    if g5k.get_my_jobs(site).empty?
+      # As the platform could be busy and we could wait for a long time.
+      # here, we set wait_time to 200 seconds, like that we would just use the sites that are free.
+      begin
+      jobs[site] = g5k.reserve(:site => site, :resources => "slash_22=1+{virtual!='none'}/nodes=1",
+                               :walltime =>"01:00:00",:keys => "~/.ssh/id_rsa", :wait_time => 200)
+      rescue Cute::G5K::EventTimeout
+        puts "We waited long enough, releasing job in site #{site}"
+        g5k.release(jobs[site]) # we release the job
+        jobs.delete(site)
+      end
+    else
+      jobs[site] = g5k.get_my_jobs(site).first
+    end
+  }
+}
+
+threads.each{ |t| t.join}
+
+nodes = []
+jobs.each{ |k,v| nodes+=v["assigned_nodes"]}
+
+puts("Nodes reserved: #{nodes.inspect}")
+
+# Creating vm configuration files
+vm_dir = Dir.mktmpdir("vm_def")
+
+system("wget -O #{vm_dir}/vm-template.xml http://public.nancy.grid5000.fr/~cruizsanabria/vm-template.xml")
+
+template = ERB.new(File.read("#{vm_dir}/vm-template.xml"))
+
+vms = []
+
+G5K_SITES.each{ |site|
+  subnet  = g5k.get_subnets(jobs[site]).first
+  ips = subnet.map{ |ip| ip.to_s }
+  num_vm.times{ |n|
+    @vm_name = "node#{n}.#{site}"
+    @vm_mac = ip2mac(ips[n+1])
+    vms.push(ips[n+1]) # avoiding .0 last octet
+    @tap_device = "tap#{n}"
+    File.open("#{vm_dir}/node_#{n}.#{site}.xml",'w+') do |f|
+      f.puts(template.result()) # ERB replaces @vm_name, @vm_mac and, @tap_device in the file.
+    end
+  }
+}
+
+puts("vm's ip assigned #{vms.inspect}")
+
+# Setting up VMs
+Cute::TakTuk.start(nodes, grid5000_opt) do |tak|
+
+  tak.exec!("mkdir -p ~/vm_definitions")
+
+  tak.exec("wget -q -O /tmp/wheezy-x64-base.qcow2 http://public.nancy.grid5000.fr/~cruizsanabria/wheezy-x64-base.qcow2")
+  puts("Transfering configuration files")
+  Dir.entries(vm_dir).each{ |vm_file|
+    next if vm_file[0] =="." # avoid . and .. files
+    puts File.join(vm_dir,vm_file)
+    tak.put(File.join(vm_dir,vm_file),"/tmp/#{vm_file}")
+  }
+  # Creates a number of tap devices number of vms/number of machines
+  puts("Creating TAP devices")
+  num_vm.times{ tak.exec!("sudo create_tap") }
+
+  # Creating contextualization script to copy our ssh key
+  tak.exec!("mkdir -p ~/kvm-context")
+  tak.exec!("cp ~/.ssh/id_rsa.pub ~/kvm-context/")
+  File.open("/tmp/post-install","w+") do |f|
+    f.puts("#!/bin/sh")
+    f.puts("mkdir -p /root/.ssh")
+    f.puts("cat /mnt/id_rsa.pub >> /root/.ssh/authorized_keys")
+  end
+  tak.put("/tmp/post-install","/tmp/post-install")
+  tak.exec!("cp /tmp/post-install ~/kvm-context/post-install")
+  tak.exec!("chmod 755 ~/kvm_context/post-install")
+  tak.exec!("genisoimage -r -o /tmp/kvm-context.iso ~/kvm-context/")
+end
+
+# Starting vms
+Net::SSH::Multi.start do |session|
+  jobs.each{ |site,job|
+    # We create a group per site
+    session.group site do
+      job["assigned_nodes"].each{ |node|
+        session.use node, grid5000_opt
+      }
+    end
+    num_vm.times{ |n| puts session.with(site).exec!("virsh create /tmp/node_#{n}.#{site}.xml")}
+  }
+end
+
+puts("Waiting for the machines to start")
+sleep 60
+
+# Executing some commands on the vms
+
+Net::SSH::Multi.start do |session|
+
+  vms.each{ |vm|
+    session.use("root@#{vm}")
+  }
+  session.exec("hostname")
+  session.exec("uptime")
+
+end

data/lib/cute.rb

@@ -1,2 +1,7 @@
-module Cute
-end
+require 'cute/version'
+require 'cute/bash'
+require 'cute/execute'
+require 'cute/taktuk'
+require 'cute/g5k_api'
+require 'cute/extensions'
+require 'cute/net-ssh'

data/lib/cute/bash.rb

@@ -0,0 +1,337 @@
+
+#
+# Features a cool class to interface with Bash.
+#
+
+require 'digest'
+require 'open3'
+
+module Cute; module Bash
+
+    class BashError < StandardError; end
+    class BashTimeout < BashError; end
+    class BashPaddingError < BashError; end
+
+    class StatusError < BashError
+
+        attr_reader :status
+        attr_reader :output
+
+        def initialize(cmd, status, output)
+            super("'#{cmd}' returned with status = #{status}")
+            @status = status
+            @cmd = cmd
+            @output = output
+        end
+
+    end
+
+    class Bash
+
+        def initialize(stdin, stdout, debug = false)
+            @stdin = stdin
+            @stdout = stdout
+            @debug = debug
+            @buff = ''
+        end
+
+        def parse(&block)
+            return self.instance_exec(&block)
+        end
+
+        def _loop
+            while true do
+                x = IO::select([@stdout], [], [], 120.0)
+                raise BashTimeout.new if x.nil?
+                bytes = @stdout.sysread(1024)
+                $stderr.write("\nBASH IN: #{bytes}\n") if @debug
+                @buff << bytes
+                break if yield
+            end
+        end
+
+        def _nonce
+            randee = 4.times.map { rand().to_s }.join('|')
+            return Digest::SHA512.hexdigest(randee).to_s
+        end
+
+        def _run(cmd, opts)
+            # it's a kind of magic
+            $stderr.write("\nBASH CMD: #{cmd}\n") if @debug
+            nonce = _nonce()
+            @stdin.write("#{cmd}; printf '%04d#{nonce}' $?\n")
+            @stdin.flush
+            _loop do
+                @buff.include?(nonce)
+            end
+            raise BashPaddingError.new if !@buff.end_with?(nonce)
+            output = @buff
+            @buff = ''
+            # treat the output
+            output.slice!(-nonce.length..-1)
+            status = output.slice!(-4..-1)
+            raise "Status #{status} > 255?" if status.slice(0..0) != '0'
+            return output, status.to_i
+        end
+
+        def _run_block(cmd, opts)
+            @stdin.write("#{cmd}; printf '%04d#{nonce}' $?\n")
+        end
+
+        def _extend(path, suffix)
+            path = path.chomp('/') if path.end_with?('/')
+            return path + suffix
+        end
+
+        def _unlines(s)
+            return s.lines.map { |l| l.chomp("\n") }
+        end
+
+        def _escape(args)
+            return args.map { |x| "'#{x}'" }.join(' ')
+        end
+
+        # TESTING METHODS
+
+        def assert(condition, msg = 'Assertion error')
+            raise msg if condition != true
+        end
+
+        # PUBLIC METHODS
+
+        def export(name, value)
+            run("export #{name}=#{value}")
+        end
+
+        def run(cmd, opts = {})
+            out, status = _run(cmd, opts)
+            raise StatusError.new(cmd, status, out) if status != 0
+            return out
+        end
+
+        def run_status(cmd, opts = {})
+            out, status = _run(cmd, opts)
+            return status
+        end
+
+        def cd(path)
+            run("cd #{path}")
+        end
+
+        def ls
+            run("ls -1").lines.map { |line| line.chomp("\n") }
+        end
+
+        def pwd
+            run("pwd").strip
+        end
+
+        def untar(name, where = nil)
+            if where.nil?
+                run("tar xvf #{name}")
+            else
+                run("tar -C #{where} -xvf #{name}")
+            end
+        end
+
+        def echo(text)
+            run("echo #{text}")
+        end
+
+        def bc(text)
+            run("echo '#{text}' | bc").strip
+        end
+
+        def cp(a, b)
+            run("cp #{a} #{b}")
+        end
+
+        def mv(a, b)
+            run("mv #{a} #{b}")
+        end
+
+        def rm(*args)
+            run("rm #{_escape(args)}")
+        end
+
+        def remove_dirs(path)
+            rm "-rf", path
+        end
+
+        def build
+            # builds a standard Unix software
+            run("./configure")
+            run("make")
+        end
+
+        def abspath(path)
+            run("readlink -f #{path}").strip
+        end
+
+        def build_tarball(tarball, path)
+            # builds a tarball containing a std Unix software
+            tarball = abspath(tarball)
+            path = abspath(path)
+            remove_dirs(path)
+            tmp = _extend(path, '-tmp')
+            remove_dirs(tmp)
+            make_dirs(tmp)
+            untar(tarball, tmp)
+            cd tmp
+            # we are in the temp dir
+            if exists('./configure')
+                cd '/'
+                mv tmp, path
+            else
+                ds = dirs()
+                raise 'Too many dirs?' if ds.length != 1
+                mv ds.first, path
+                cd '/'
+                remove_dirs(tmp)
+            end
+            cd path
+            build
+        end
+
+        def tmp_file
+            return run('mktemp').strip
+        end
+
+        def save_machines(machines, path = nil)
+            path = tmp_file if path.nil?
+            append_lines(path, machines.map { |m| m.to_s })
+            return path
+        end
+
+        def mpirun(machines, params)
+            machines = save_machines(machines) if machines.is_a?(Array)
+            return run("mpirun --mca btl ^openib -machinefile #{machines} #{params}")
+        end
+
+        def join(*args)
+            return File.join(*args)
+        end
+
+        def exists(path)
+            run_status("[[ -e #{path} ]]") == 0
+        end
+
+        def make_dirs(path)
+            run("mkdir -p #{path}")
+        end
+
+        def mkdir(path)
+            run("mkdir #{path}") unless exists(path)  # TODO: this changes semantics of mkdir...
+        end
+
+        def files(ignore = true, type = 'f')
+            fs = run("find . -maxdepth 1 -type #{type}")
+            fs = _unlines(fs).reject { |f| f == '.' }.map { |f| f[2..-1] }
+            fs = fs.reject { |f| f.end_with?('~') or f.start_with?('.') } if ignore
+            return fs
+        end
+
+        def which(prog)
+            return run("which #{prog}").strip
+        end
+
+        def dirs(ignore = true)
+            return files(ignore, 'd')
+        end
+
+        def get_type(name)
+            return :dir if run_status("[[ -d #{name} ]]") == 0
+            return :file if run_status("[[ -f #{name} ]]") == 0
+            raise "'#{name}' is neither file nor directory"
+        end
+
+        def expand_path(path)
+            return run("echo #{path}").strip
+        end
+
+        def append_line(path, line)
+            return run("echo '#{line}' >> #{path}")
+        end
+
+        def append_lines(path, lines)
+            lines.each { |line|
+                append_line(path, line)
+            }
+        end
+
+        def contents(name)
+            run("cat #{name}")
+        end
+
+        alias cat contents
+
+        def hostname
+            return run('hostname').strip
+        end
+
+        def touch(path)
+            run("touch #{path}")
+        end
+
+        # BELOW ARE SPECIFIC METHODS FOR XPFLOW
+
+
+        def packages
+            list = run("dpkg -l")
+            list = _unlines(list).map do |p|
+                s, n, v = p.split
+                { :status => s, :name => n, :version => v }
+            end
+        end
+
+        def aptget(*args)
+            raise 'Command not given' if args.length == 0
+            cmd = args.first.to_sym
+            args = args.map { |x| x.to_s }.join(' ')
+            status = run_status("DEBIAN_FRONTEND=noninteractive apt-get -y #{args}")
+            if cmd == :purge and status == 100
+                return # ugly hack for the case when the package is not installed
+            end
+            raise StatusError.new('aptget', status, 'none') if status != 0
+        end
+
+        def distribute(path, dest, *nodes)
+            # distributes a file to the given nodes
+            nodes.flatten.each { |node|
+                run("scp -o 'StrictHostKeyChecking no' #{path} #{node}:#{dest}")
+            }
+        end
+
+        def glob(pattern)
+            out, status = _run("ls -1 #{pattern}", {})
+            return [] if status != 0
+            return out.strip.lines.map { |x| x.strip }
+        end
+
+    end
+
+
+    def self.bash(cmd = 'bash', debug = false, &block)
+        if not block_given?
+            sin, sout, serr, thr = Open3.popen3(cmd)
+            return Bash.new(sin, sout, debug)
+        end
+        # run bash interpreter using this command
+        result = nil
+        Open3.popen3(cmd) do |sin, sout, serr, thr|
+            dsl = Bash.new(sin, sout, debug)
+            dsl.cd('~')   # go to the home dir
+            result = dsl.parse(&block)
+        end
+        return result
+    end
+
+end; end
+
+if __FILE__ == $0
+    Cute::Bash.bash("ssh localhost bash") do
+        cd '/tmp'
+        puts files.inspect
+        run 'rm /'
+    end
+end