nera 0.0.4 → 0.1.0

data/History.txt

@@ -1,3 +1,9 @@
+== 0.1.0
+* 1 major enahncement:
+  * remote connection function is supported
+* 2 bug fixes
+  * bug fix for multi-lined job scripts.
+
 == 0.0.4
 * 1 major enhancement:
   * multi-lined simulation command becomes available

data/Manifest.txt

@@ -3,6 +3,7 @@ Manifest.txt
 README.rdoc
 Rakefile
 bin/nera
+bin/nera_addhost
 bin/nera_addsim
 lib/nera.rb
 lib/nera/nera_cui.rb
@@ -14,11 +15,13 @@ lib/nera/nera_job_records.rb
 lib/nera/nera_job_script.rb
 lib/nera/nera_parameter_layer_controller.rb
 lib/nera/nera_parameter_records.rb
+lib/nera/nera_remote_connector.rb
 lib/nera/nera_run_layer_controller.rb
 lib/nera/nera_run_records.rb
 lib/nera/nera_simulator.rb
 lib/nera/nera_simulator_layer_controller.rb
 lib/nera/nera_simulator_records.rb
+lib/nera_addhost/add_hosts.rb
 lib/nera_addsim/make_simulator.rb
 scripts/console
 scripts/destroy
@@ -33,6 +36,7 @@ test/test_nera_job_layer_controller.rb
 test/test_nera_job_records.rb
 test/test_nera_parameter_layer_controller.rb
 test/test_nera_parameter_records.rb
+test/test_nera_remote_connector.rb
 test/test_nera_run_layer_controller.rb
 test/test_nera_run_records.rb
 test/test_nera_simulator.rb

data/README.rdoc

@@ -17,6 +17,10 @@ To add your own simulators,
 
   nera_addsim 'neradb'
 
+To edit your host informations,
+
+  nera_addhost 'neradb'
+
 == REQUIREMENTS:
 
 Ruby 1.8.0 or later

data/Rakefile

@@ -7,9 +7,9 @@ $hoe = Hoe.new('nera', NERA::VERSION) do |p|
   p.developer('Yohsuke Murase', 'murase@serow.t.u-tokyo.ac.jp')
   p.changes              = p.paragraphs_of("History.txt", 0..1).join("\n\n")
   p.rubyforge_name       = p.name # TODO this is default value
-  # p.extra_deps         = [
-  #   ['activesupport','>= 2.0.2'],
-  # ]
+  p.extra_deps         = [
+    ['net-ssh','>= 2.0.11'],
+  ]
   p.extra_dev_deps = [
     ['newgem', ">= #{::Newgem::VERSION}"]
   ]

data/bin/nera_addhost

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+#
+#  Created by Yohsuke Murase on 2009-4-28.
+#  Copyright (c) 2009. All rights reserved.
+require 'optparse'
+require File.expand_path(File.dirname(__FILE__) + "/../lib/nera")
+
+parser = OptionParser.new do |opts|
+  opts.banner = <<-BANNER.gsub(/^    /,'')
+    
+    Usage: #{File.basename($0)} [options] 'nera_db_folder'
+    
+    BANNER
+  opts.separator ""
+  opts.on("-h", "--help",
+          "Show this help message.") { $stdout.puts opts; exit }
+  opts.on("-v", "--version", "Show version.") { $stdout.puts NERA::VERSION; exit}
+  opts.parse!(ARGV)
+end
+
+unless ARGV.size == 1
+  raise "\n\nUsage :\t nera_addhost 'nera_db_folder'"
+end
+
+require File.expand_path(File.dirname(__FILE__) + "/../lib/nera_addhost/add_hosts")
+NERA::CUI_Hosts_Editor.new(ARGV[0])

data/lib/nera/nera_cui.rb

@@ -351,9 +351,13 @@ HEADER
       action_list = ["Back to simulator layer",
                      "Show job infos",
                      "Cancel jobs",
+                     "Show host infos",
+                     "Upload jobs",
+                     "Download data",
                      "Include data",
                      "Exit"]
       jlc = NERA::JobLayerController.new( @path_db_folder)
+      rc = NERA::RemoteConnector.new( @path_db_folder)
 
       selected = 0
       Dir.chdir( jlc.path_to_job_layer) {
@@ -388,6 +392,80 @@ HEADER
           end
         end
         return :job_layer
+      when "Show host infos"
+        h_list = ["Cancel", "All"] + rc.hostnames
+        sel = Dialog::select_many( h_list)
+        if sel.include?(0)
+          return :job_layer
+        elsif sel.include?(1)
+          rc.hostnames.each do |hname|
+            Dialog::message("Information of the host #{hname} =========================")
+            Dialog::message( rc.show_status( hname) )
+          end
+        else
+          sel.each do |num|
+            Dialog::message("Information of the host #{h_list[num]} =========================")
+            Dialog::message( rc.show_status( h_list[num]) )
+          end
+        end
+        return :job_layer
+      when "Upload jobs"
+        header,list = jlc.not_finished_list_in_csv
+        list.unshift("Cancel")
+        selected_jobs = Dialog::select_many( list, header).map do |num|
+          list[num].split(',')[0].to_i
+        end
+        return :job_layer if selected_jobs.include?(0)
+
+        h_list = ["Cancel"] + rc.hostnames
+        sel = Dialog::select_one( h_list, "Select the host")
+        return :job_layer if sel == 0
+        selected_host = h_list[sel]
+        
+        Dialog::message("Information of the host #{selected_host} =========================")
+        Dialog::message( rc.show_status( selected_host) + "\n" )
+        if Dialog::ask("Will you upload the script?", true)
+          if rc.submittable_hostnames.include?(selected_host) and Dialog::ask("Will you submit the jobs?", true)
+            v = rc.submit( selected_jobs, selected_host)
+            Dialog::message(v.to_s)
+          else
+            v = rc.transfer( selected_jobs, selected_host)
+            Dialog::message(v.to_s)
+          end
+        end
+        return :job_layer
+      when "Download data"
+        d_hash = rc.check_completion_all
+        list = ["Cancel", "All"]
+        d_hash.each_pair do |host, d_files|
+          if d_files.is_a?(Array)
+            d_files.each do |f|
+              list << "#{f} @ #{host}"
+            end
+          end
+        end
+        sel = Dialog::select_many( list)
+        if sel.include?(0)
+          return :job_layer
+        elsif sel.include?(1)
+          d_hash.each_pair do |host, d_files|
+            res = rc.download( d_files, host)
+            if res.is_a?(Array)
+              Dialog::message("#{res.join(', ')} have been downloaded.")
+            else
+              Dialog::message(res.to_s)
+            end
+          end
+        else
+          downloaded = []
+          sel.each do |num|
+            a = list[num].split('@')
+            res = rc.download( [ a[0].strip ], a[1].strip)
+            downloaded += res if res.is_a?(Array)
+          end
+          Dialog::message("#{downloaded.join(', ')} have been downloaded.")
+        end
+        return :job_layer
       when "Include data"
         list = jlc.include_list
         list = ["Cancel","All"] + list

data/lib/nera/nera_db_folders.rb

@@ -220,7 +220,11 @@ module NERA
     def path_to_finished_jobs_file
       return @db_folder + "Tables/finished_jobs.yml"
     end
-    
+
+    def path_to_hosts_yaml
+      return @db_folder + "Tables/hosts.yml"
+    end
+
   end
 end
     

data/lib/nera/nera_dialog.rb

@@ -200,6 +200,33 @@ module Dialog
       end
     end
   end
+
+  # -----------------
+  def ask( message = "Input y or n.", default = true)
+    while true
+      m = "(y/N)"
+      m = "(Y/n)" if default
+      $stdout.puts( "#{message} : #{m}")
+
+      buf = Readline.readline(PROMPT).chomp.strip.downcase
+      if buf == ''
+        if default
+          return true
+        else
+          return false
+        end
+      end
+      
+      if buf.match(/^y$/) or buf.match(/^yes$/)
+        return true
+      elsif buf.match(/^n$/) or buf.match(/^no$/)
+        return false
+      else
+        $stdout.puts "The input is not valid. Try again"
+        redo
+      end
+    end
+  end
   
-  module_function :select_one, :select_one_or_return_string, :select_many, :message, :set_multiple_values, :get_integer
+  module_function :select_one, :select_one_or_return_string, :select_many, :message, :set_multiple_values, :get_integer, :ask
 end

data/lib/nera/nera_job_script.rb

@@ -66,7 +66,7 @@ echo \"NERA_START_AT\" >> nera_status.rb
 echo \"#{job_info[:execs][i]}\"
 mkdir -p #{run_id}
 cd #{run_id}
-{ time -p #{job_info[:execs][i]}; } 2>> ../nera_status.rb
+{ time -p { #{job_info[:execs][i]};} } 2>> ../nera_status.rb
 if test $? -ne 0; then { echo \"Exit abnormally!\" >> ../nera_status.rb; exit; } fi
 cd ..
 echo \"NERA_OUTPUT\" >> nera_status.rb
@@ -146,7 +146,7 @@ BODY
         :run_ids => run_ids,
         :seeds => seeds,
         :execs => seeds.map do |sed|
-          sim_inst.simulate_command( sed).split("\n").join('; ').sub(/;$/,'')
+          sim_inst.simulate_command( sed).split("\n").join(' && ').sub(/;$/,'')
         end
       }
 

data/lib/nera/nera_remote_connector.rb

@@ -0,0 +1,337 @@
+require 'nera_db_folders'
+require 'nera_job_records'
+require 'nera_job_script'
+require 'fileutils'
+require 'yaml'
+require 'rubygems'
+require 'net/ssh'
+require 'net/sftp'
+
+module NERA
+
+  # A class for remote connection
+  class RemoteConnector
+
+    # instance of NERA::DbFolders
+    @db_folders
+    # instance of NERA::JobRecords
+    @job_records
+    # hosts (Array of Hashes)
+    @hosts
+
+    public
+    def initialize( path_db_folder)
+      raise ArgumentError unless path_db_folder.is_a?(String)
+      
+      @db_folders = NERA::DbFolders.new( path_db_folder)
+      if File.exist?( @db_folders.path_to_hosts_yaml)
+        File.open( @db_folders.path_to_hosts_yaml, 'r') do |io|
+          @hosts = YAML.load( io)
+        end
+      else
+        @hosts = []
+      end
+      raise "Invalid format of hosts.yml" unless valid_hosts?
+      @job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
+    end
+
+    private
+    def valid_hosts?
+      @hosts.each do |host|
+        return false unless host.has_key?(:name)
+        return false unless host[:name].is_a?(String)
+        return false unless host.has_key?(:host_url)
+        return false unless host[:host_url].is_a?(String)
+        return false unless host.has_key?(:user)
+        return false unless host[:user].is_a?(String)
+        host[:job_path] = '~/' unless host.has_key?(:job_path)
+        host[:port] = 22 unless host.has_key?(:port)
+      end
+    end
+
+    public
+    def hostnames
+      @hosts.map do |host| host[:name] end
+    end
+
+    def submittable_hostnames
+      arr = []
+      @hosts.each do |host|
+        arr << host[:name] if host[:submission_command]
+      end
+      return arr
+    end
+
+    def show_status( hostname)
+      found = @hosts.find do |host|
+        host[:name] == hostname
+      end
+      return :connection_failed unless found
+      h = { :port => found[:port], :timeout => 10 }
+      h[:keys] = found[:keys] if found.has_key?(:keys)
+      cmd = 'ps au'
+      cmd = found[:show_status_command] if found.has_key?(:show_status_command)
+      str = ''
+      begin
+        $stderr.print "Connecting with #{found[:name]}.....\t"
+        Net::SSH.start( found[:host_url], found[:user], h) do |ssh|
+          str = ssh.exec!( cmd)
+        end
+      rescue Errno::ECONNREFUSED
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue Errno::ENETUNREACH
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue Net::SSH::Exception
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue SocketError
+        $stderr.puts "failed"
+        return :connection_failed
+      end
+      $stderr.puts "done"
+      return str
+    end
+
+    def transfer( jobids, hostname)
+      raise ArgumentError unless jobids.is_a?(Array)
+      jobids.each do |jid|
+        raise ArgumentError unless jid.is_a?(Integer)
+      end
+      raise ArgumentError unless hostname.is_a?(String)
+
+      host = @hosts.find do |h| h[:name] == hostname end
+      return :connection_failed unless host
+      
+      @job_records.transaction {
+        jids = []
+        jobids.each do |jid|
+          jids << jid if @job_records.find_by_id(jid)
+        end
+        return :no_such_jobs if jids.size == 0
+        
+        h = { :port => host[:port] }
+        h[:keys] = host[:keys] if host.has_key?(:keys)
+        
+        begin
+        $stderr.print "Connecting with #{host[:name]}.....\t"
+          Net::SSH.start( host[:host_url], host[:user], h) do |ssh|
+            ssh.exec("mkdir -p #{host[:job_path]}")
+            target_home = ssh.exec!('echo $HOME').chomp
+            target_dir = host[:job_path].sub(/^~/,target_home)
+            ssh.sftp.connect do |sftp|
+              jids.each do |jid|
+                pjs = @db_folders.path_to_job_script(jid)
+                target_path = target_dir + '/' + File.basename(pjs)
+                next unless File.exist?( pjs)
+                contents = ''
+                File.open( pjs, 'r') do |io| contents = io.readlines end
+                contents[0] += "cd #{target_dir}\n"
+                sftp.file.open(target_path, "w") do |f| f.puts contents end
+                ssh.exec!("chmod +x #{target_path}")
+                @job_records.update_to_state_copied( jid, host[:name])
+              end
+            end
+          end
+        rescue Errno::ECONNREFUSED
+          $stderr.puts "failed"
+          return :connection_failed
+        rescue Errno::ENETUNREACH
+          $stderr.puts "failed"
+          return :connection_failed
+        rescue SocketError
+          $stderr.puts "failed"
+          return :connection_failed
+        rescue Net::SSH::Exception
+          $stderr.puts "failed"
+          return :connection_failed
+        end
+        $stderr.puts "done"
+      }
+      return :succeeded
+    end
+
+    private
+    def convert_shell_script
+      
+    end
+
+    public
+    def submit( jobids, hostname)
+      raise ArgumentError unless jobids.is_a?(Array)
+      jobids.each do |jid|
+        raise ArgumentError unless jid.is_a?(Integer)
+      end
+      raise ArgumentError unless hostname.is_a?(String)
+
+      host = @hosts.find do |h| h[:name] == hostname end
+      return :connection_failed unless host
+
+      return :no_submission_command unless host.has_key?(:submission_command)
+      
+      @job_records.transaction {
+        jids = []
+        jobids.each do |jid|
+          jids << jid if @job_records.find_by_id(jid)
+        end
+        return :no_such_jobs if jids.size == 0
+        
+        h = { :port => host[:port] }
+        h[:keys] = host[:keys] if host.has_key?(:keys)
+        
+        begin
+        $stderr.print "Connecting with #{host[:name]}.....\t"
+          Net::SSH.start( host[:host_url], host[:user], h) do |ssh|
+            ssh.exec("mkdir -p #{host[:job_path]}")
+            target_home = ssh.exec!('echo $HOME').chomp
+            target_dir = host[:job_path].sub(/^~/,target_home)
+            ssh.sftp.connect do |sftp|
+              jids.each do |jid|
+                pjs = @db_folders.path_to_job_script(jid)
+                next unless File.exist?( pjs)
+                target_path = target_dir + '/' + File.basename(pjs)
+                contents = ''
+                File.open( pjs, 'r') do |io| contents = io.readlines end
+                contents[0] += "cd #{target_dir}\n"
+                sftp.file.open(target_path, "w") do |f| f.puts contents end
+                ssh.exec!("chmod +x #{target_path}")
+                @job_records.update_to_state_copied( jid, host[:name])
+                ssh.exec!("cd #{target_dir} && #{host[:submission_command]} #{target_path} &")
+                @job_records.update_to_state_submitted( jid, host[:name])
+              end
+            end
+          end
+        rescue Errno::ECONNREFUSED
+          $stderr.puts "failed"
+          return :connection_failed
+        rescue Errno::ENETUNREACH
+          $stderr.puts "failed"
+          return :connection_failed
+        rescue SocketError
+          $stderr.puts "failed"
+          return :connection_failed
+        rescue Net::SSH::Exception
+          $stderr.puts "failed"
+          return :connection_failed
+        end
+        $stderr.puts "done"
+      }
+      return :succeeded
+    end
+
+    def check_completion( hostname)
+      raise ArgumentError unless hostname.is_a?(String)
+
+      host = @hosts.find do |h| h[:name] == hostname end
+      return :connection_failed unless host
+
+      h = { :port => host[:port] }
+      h[:keys] = host[:keys] if host.has_key?(:keys)
+
+      ls_str = ''
+      begin
+        $stderr.print "Connecting with #{host[:name]}.....\t"
+        Net::SSH.start( host[:host_url], host[:user], h) do |ssh|
+          ls_str = ssh.exec!("ls #{host[:job_path]}")
+        end
+      rescue Errno::ECONNREFUSED
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue Errno::ENETUNREACH
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue SocketError
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue Net::SSH::Exception
+        $stderr.puts "failed"
+        return :connection_failed
+      end
+      $stderr.puts "done"
+      return finished_list( ls_str)
+    end
+
+    private
+    def finished_list( ls_str)
+      ls_list = ls_str.to_s.split("\n")
+      return ls_list.find_all do |fname|
+        flag = false
+        if fname =~ /^(j[0-9]+).tar.bz2$/
+          flag = true unless ls_list.find do |f| f == $1 end
+        end
+        flag
+      end
+    end
+
+    public
+    def check_completion_all
+      h = {}
+      hostnames.each do |hname|
+        h[hname] = check_completion(hname)
+      end
+      return h
+    end
+
+
+    def download( filenames, hostname)
+      raise ArgumentError unless filenames.is_a?(Array)
+      filenames.each do |fname|
+        raise ArgumentError unless fname.is_a?(String)
+        raise ArgumentError unless fname =~ /.tar.bz2$/
+      end
+      raise ArgumentError unless hostname.is_a?(String)
+
+      host = @hosts.find do |h| h[:name] == hostname end
+      return :connection_failed unless host
+
+      h = { :port => host[:port] }
+      h[:keys] = host[:keys] if host.has_key?(:keys)
+
+      downloaded = []
+      
+      begin
+        $stderr.puts "Connecting with #{host[:name]}.....\t"
+        Net::SSH.start( host[:host_url], host[:user], h) do |ssh|
+          target_home = ssh.exec!('echo $HOME').chomp
+          target_dir = host[:job_path].sub(/^~/,target_home)
+          ls_str = ssh.exec!("ls #{target_dir}")
+          d_list = filenames.find_all do |fname| ls_str.include?( fname) end
+          ssh.sftp.connect do |sftp|
+            d_list.each do |d_file|
+              #download
+              $stderr.print "  downloading #{d_file}.....\t"
+              sftp.download!( target_dir + '/' + d_file, @db_folders.path_to_include_layer + d_file)
+              $stderr.puts "done"
+              downloaded << d_file
+              #delete data file and job script
+              $stderr.print "  deleting #{d_file}.....\t"
+              sftp.remove!(target_dir + '/' + d_file)
+              $stderr.puts "done"
+              sh_file = d_file.sub(/tar.bz2$/,'sh')
+              $stderr.print "  deleting #{sh_file}.....\t"
+              sftp.remove!(target_dir + '/' + sh_file)
+              $stderr.puts "done"
+            end
+          end
+        end
+      rescue Errno::ECONNREFUSED
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue Errno::ENETUNREACH
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue SocketError
+        $stderr.puts "failed"
+        return :connection_failed
+      rescue Net::SSH::Exception
+        $stderr.puts "failed"
+        return :connection_failed
+      end
+      $stderr.puts "done"
+      
+      return downloaded
+    end
+    
+  end
+end

data/lib/nera.rb

@@ -19,7 +19,8 @@ require 'nera_run_records'
 require 'nera_simulator'
 require 'nera_simulator_layer_controller'
 require 'nera_simulator_records'
+require 'nera_remote_connector'
 
 module NERA
-  VERSION = '0.0.4'
+  VERSION = '0.1.0'
 end

data/lib/nera_addhost/add_hosts.rb

@@ -0,0 +1,193 @@
+require 'nera_dialog'
+require 'yaml'
+require 'fileutils'
+
+module NERA
+
+  class CUI_Hosts_Editor
+
+    HOST_INFO = [ [:host_url,String,"localhost"], [:user,String,ENV['USER']], [:job_path,String,"~/"],
+                   [:port,Integer,22], [:keys,String,"~/.ssh/id_rsa"],
+                   [:submission_command,String,"nohup"], [:show_status_command,String,"ps au"]]
+
+
+    #----- the unique module funciton -----
+    def initialize( db_folder_path)
+      unless valid_nera_db?( db_folder_path)
+        raise "\n\n#{db_folder_path} is not a valid nera_db folder."
+      end
+
+      dbf = NERA::DbFolders.new( db_folder_path)
+      @@hosts = Hash.new
+      h_yml = dbf.path_to_hosts_yaml
+      @@hosts = YAML.load( File.open( h_yml,'r') ) if File.exists?( h_yml)
+
+      edit_hosts_yaml
+      YAML.dump( @@hosts, File.open(h_yml,'w') )
+    end
+
+    private
+    #------- private methods ------------
+    def valid_nera_db?( db_folder_path)
+      return false unless File.directory?( db_folder_path)
+      return false unless File.directory?( db_folder_path + '/Tables/')
+      return false unless File.directory?( db_folder_path + '/Jobs/')
+      return false unless File.directory?( db_folder_path + '/Simulator_classes/')
+      return false unless File.exist?( db_folder_path + '/simulators.yml')
+      return true
+    end
+    
+    def edit_hosts_yaml
+      loop do
+        names = @@hosts.map do |host| host[:name] end
+        names = [:"add a new host"] + names + [:exit]
+        sel = Dialog::select_one( names, "Select the host to edit.")
+        case names[sel]
+        when :"add a new host"
+          add_a_new_host
+        when :exit
+          return true
+        else
+          edit_host_info( names[sel])
+        end
+      end
+    end
+
+    def add_a_new_host
+      $stdout.puts <<HEA
+==========================================
+Enter the new hostname (1/3)
+==========================================
+HEA
+      used_names = @@hosts.map do |host| host[:name] end
+      name = Readline.readline(">").chomp.strip
+      while name.size == 0 or used_names.include?(name)
+        $stdout.puts "The given hostname is not valid."
+        name = Readline.readline(">").chomp.strip
+      end
+      new_host = { :name => name}
+
+      $stdout.puts <<HEA
+==========================================
+Edit the host information of #{name} (2/3)
+==========================================
+HEA
+      info = Dialog.set_multiple_values( HOST_INFO, "Set host information")
+      new_host.merge!( info)
+
+      $stdout.puts <<HEA
+==========================================
+Check connection with #{name} (3/3)
+==========================================
+HEA
+      h = { :port => new_host[:port], :keys => new_host[:keys] }
+      flag = false
+      begin
+        Net::SSH.start( new_host[:host_url], new_host[:user], h) do |ssh|
+          $stdout.puts ssh.exec!( new_host[:show_status_command])
+        end
+      rescue Errno::ECONNREFUSED then
+        puts "Connection refused. Check server state again."
+        flag = false
+      rescue Errno::ENETUNREACH then
+        puts "Network is unreachable."
+        flag = false
+      rescue SocketError then
+        puts "Can not resolve host name. Is network active?"
+        flag = false
+      rescue Net::SSH::Exception then
+        puts "Connection failed."
+        flag = false
+      rescue Exception then
+        puts "Conncetion error."
+        flag = false
+      else
+        puts "Connect successfully."
+        flag = true
+      end
+
+      unless flag
+        unless Dialog::ask( "Do you continue anyway?", true)
+          $stdout.puts "Addition of #{name} is aborted."
+          return nil
+        end
+      end
+
+      @@hosts << new_host
+      $stdout.puts <<HEA
+==========================================
+Host #{name} is successfully added.
+==========================================
+HEA
+    end
+
+    # ==============================
+    def edit_host_info( hostname)
+      $stdout.puts <<HEA
+==========================================
+Edit the host information of #{hostname} (1/2)
+==========================================
+HEA
+      h_info = HOST_INFO.dup
+      found = @@hosts.find do |h| h[:name] == hostname end
+      h_info.each_with_index do |info,i|
+        if found[info[0]]
+          info[2] = found[info[0]]
+        else
+          info[2] = HOST_INFO[i][2]
+        end
+      end
+
+      new_host = Dialog.set_multiple_values( h_info, "Set host information")
+      new_host[:name] = hostname
+
+      $stdout.puts <<HEA
+==========================================
+Check connection with #{hostname} (2/2)
+==========================================
+HEA
+      h = { :port => new_host[:port], :keys => new_host[:keys] }
+      flag = false
+      begin
+        Net::SSH.start( new_host[:host_url], new_host[:user], h) do |ssh|
+          $stdout.puts ssh.exec!( new_host[:show_status_command])
+        end
+      rescue Errno::ECONNREFUSED then
+        puts "Connection refused. Check server state again."
+        flag = false
+      rescue Errno::ENETUNREACH then
+        puts "Network is unreachable."
+        flag = false
+      rescue SocketError then
+        puts "Can not resolve host name. Is network active?"
+        flag = false
+      rescue Net::SSH::Exception then
+        puts "Connection failed."
+        flag = false
+      rescue Exception then
+        puts "Conncetion error."
+        flag = false
+      else
+        puts "Connect successfully."
+        flag = true
+      end
+
+      unless flag
+        unless Dialog::ask( "Do you continue anyway?", true)
+          $stdout.puts "Addition of #{name} is aborted."
+          return nil
+        end
+      end
+
+      idx = @@hosts.index do |h| h[:name] == hostname end
+      @@hosts[idx] = new_host.dup
+      $stdout.puts <<HEA
+==========================================
+Information of #{hostname} is successfully updated.
+==========================================
+HEA
+
+    end
+    
+  end
+end

data/test/test_nera_db_folders.rb

@@ -205,5 +205,10 @@ class TC_NERA_DB_FOLDERS < Test::Unit::TestCase
     assert_equal( @db_folder+"/Tables/finished_jobs.yml", p)
   end
 
+  def test_path_to_hosts_yaml
+    p = @dbf.path_to_hosts_yaml
+    assert_equal( @db_folder+"/Tables/hosts.yml",p)
+  end
+
 end
   

data/test/test_nera_dialog.rb

@@ -130,5 +130,11 @@ class TC_NERA_DIALOG < Test::Unit::TestCase
       Dialog::get_integer( "Input a number", "hoge")
     }
   end
-    
-  end
+
+  def test_ask
+    s = Dialog::ask("Input yes or no", true)
+    assert_equal( s, true)
+    s = Dialog::ask("Input yes or no", false)
+    assert_equal( s, false)
+  end    
+end

data/test/test_nera_job_layer_controller.rb

@@ -32,7 +32,7 @@ class TC_NERA_JOB_LAYER_CONTROLLER < Test::Unit::TestCase
   def test_path_to_job_layer
     assert_equal( @dbf.path_to_job_layer, @jlc.path_to_job_layer)
   end
-
+  
   def test_not_finished_list_in_csv
     h, l = @jlc.not_finished_list_in_csv
     assert_equal( "id, state, simulator, param_id, run_id, num_runs, created_at, updated_at, host_name", h)

data/test/test_nera_remote_connector.rb

@@ -0,0 +1,239 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class TC_NERA_RM_CONNECTOR < Test::Unit::TestCase
+
+  def setup
+    @testdir = 'test_remote_connector'
+    FileUtils.mkdir(@testdir)
+    FileUtils.chdir(@testdir)
+    @db_folder = "nera_db"
+    NERA::DbFolders.create_db( @db_folder)
+    @dbf = NERA::DbFolders.new( @db_folder)
+    @slc = NERA::SimulatorLayerController.new( @db_folder)
+    @plc = NERA::ParameterLayerController.new( @db_folder, Ising)
+    pid = @plc.create_a_new_parameter_set({:L => 32, :K => 0.01, :tmax => 1})
+    @rlc = NERA::RunLayerController.new( @db_folder, Ising, pid)
+    @rlc.create_jobs( 3, 1)
+    @jlc = NERA::JobLayerController.new( @db_folder)
+    
+    @hosts = []
+    @hosts << { :name => "localhost",:host_url => "localhost",:user => ENV['USER'], :job_path => FileUtils.pwd + '/job1'}
+    @hosts << { :name => "localhost_submittable",:host_url => "localhost",:user => ENV['USER'], :job_path => FileUtils.pwd + '/job2', :submission_command => "nohup", :show_status_command => 'ps au' }
+    @hosts << { :name => "unconnectable",:host_url => "unconnectable_host",:user => "hoge", :job_path => '~/job_test', :submission_command => 'qsub', :show_status_command => 'qstat; qstatlist;'}
+    File.open( @dbf.path_to_hosts_yaml, 'w') do |io|
+      YAML.dump( @hosts, io)
+      io.flush
+    end
+
+    @rc = NERA::RemoteConnector.new( @db_folder)
+  end
+  
+  def teardown
+    FileUtils.chdir('..')
+    FileUtils.rm_r(@testdir)
+  end
+  
+  def test_constructor
+    assert_raise( ArgumentError) {
+      NERA::RemoteConnector.new( @dbf)
+    }
+  end
+  
+  def test_hostnames
+    a = @rc.hostnames
+    names = @hosts.map do |h| h[:name] end
+    assert_equal( a, names)
+  end
+
+  def test_submittable_hostnames
+    a = @rc.submittable_hostnames
+    assert_equal( ["localhost_submittable","unconnectable"], a)
+  end
+
+  def test_show_status
+    s = @rc.show_status("localhost")
+    assert( s.is_a?(String) )
+    ret = @rc.show_status("unconnectable")
+    assert_equal( :connection_failed, ret)
+  end
+
+  def test_transfer
+    assert_raise( ArgumentError) {
+      @rc.transfer( 1, "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.transfer( [1,"j000001.sh"], "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.transfer( [1,2], 1)
+    }
+    
+    ret = @rc.transfer( [999,1000], "localhost")
+    assert_equal( :no_such_jobs, ret)
+
+    ret = @rc.transfer( [1,2], "unknown_host")
+    assert_equal( :connection_failed, ret)
+
+    ret = @rc.transfer( [1], "unconnectable")
+    assert_equal( :connection_failed, ret)
+
+    ret = @rc.transfer( [1,2], "localhost")
+    assert_equal( :succeeded, ret)
+    h,l = @jlc.not_finished_list_in_csv
+    stat = l[0].split(',')[1].strip
+    assert_equal( :copied.to_s, stat)
+    stat = l[0].split(',')[-1].strip
+    assert_equal( "localhost", stat)
+    stat = l[1].split(',')[1].strip
+    assert_equal( :copied.to_s, stat)
+    stat = l[1].split(',')[-1].strip
+    assert_equal( "localhost", stat)
+    stat = l[2].split(',')[1].strip
+    assert_equal( :created.to_s, stat)
+    stat = l[2].split(',')[-1].strip
+    assert_equal( '', stat)
+
+
+    assert( File.directory?("job1") )
+    assert( File.exist?("job1/j000001.sh") )
+    assert( File.exist?("job1/j000002.sh") )
+    assert( ! File.exist?("job1/j000003.sh") )
+  end
+
+  def test_submit
+    assert_raise( ArgumentError) {
+      @rc.submit( 1, "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.submit( ["j000001.sh"], "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.submit( [1,2], 1)
+    }
+    
+    ret = @rc.submit( [999,1000], "localhost_submittable")
+    assert_equal( :no_such_jobs, ret)
+
+    ret = @rc.submit( [1,2], "localhost")
+    assert_equal( :no_submission_command, ret)
+
+    ret = @rc.submit( [1,2], "unknown_host")
+    assert_equal( :connection_failed, ret)
+
+    ret = @rc.submit( [1], "unconnectable")
+    assert_equal( :connection_failed, ret)
+
+    ret = @rc.submit( [1,2], "localhost_submittable")
+    assert_equal( :succeeded, ret)
+
+    h,l = @jlc.not_finished_list_in_csv
+    stat = l[0].split(',')[1].strip
+    assert_equal( :submitted.to_s, stat)
+    stat = l[0].split(',')[-1].strip
+    assert_equal( "localhost_submittable", stat)
+    stat = l[1].split(',')[1].strip
+    assert_equal( :submitted.to_s, stat)
+    stat = l[1].split(',')[-1].strip
+    assert_equal( "localhost_submittable", stat)
+    stat = l[2].split(',')[1].strip
+    assert_equal( :created.to_s, stat)
+    stat = l[2].split(',')[-1].strip
+    assert_equal( '', stat)
+
+    assert( File.directory?("job2") )
+    assert( File.exist?("job2/j000001.sh") )
+    assert( File.exist?("job2/j000002.sh") )
+    assert( ! File.exist?("job2/j000003.sh") )
+
+    sleep 3 # wait for completion
+
+    assert( File.exist?("job2/j000001.tar.bz2") )
+    assert( File.exist?("job2/j000002.tar.bz2") )
+
+  end
+
+  def test_check_completion
+    fs = @rc.check_completion( "localhost_submittable")
+    assert_equal( [], fs)
+    fs = @rc.check_completion( "unconnectable")
+    assert_equal( :connection_failed, fs)
+    fs = @rc.check_completion( "unknown_host")
+    assert_equal( :connection_failed, fs)
+    assert_raise( ArgumentError) {
+      @rc.check_completion( 1)
+    }
+
+    ret = @rc.submit( [2,3], "localhost_submittable")
+    sleep 3 # wait for completion
+    fs = @rc.check_completion( "localhost_submittable")
+    assert( fs, ["j000001.tar.bz2", "j000002.tar.bz2"] )
+  end
+
+  
+  def test_check_completion_all
+    ret = @rc.submit( [1,2], "localhost_submittable")
+    sleep 3 # wait for completion
+    comp = @rc.check_completion_all
+    comp.has_key?( "localhost_submittable")
+    assert( comp["localhost_submittable"], ["j000001.tar.bz2", "j000002.tar.bz2"] )
+  end
+
+  def test_download
+    assert_raise( ArgumentError) {
+      @rc.download( "j000001.tar.bz2", "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.download( [1,"j000001.tar.bz2"], "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.download( [1,2], "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.download( ["j000001.sh"], "localhost")
+    }
+    assert_raise( ArgumentError) {
+      @rc.download( ["j000001.tar.bz2","j000002.tar.bz2"], 1)
+    }
+
+    ret = @rc.download( ["j000001.tar.bz2"], "unknown_host")
+    assert_equal( :connection_failed, ret)
+
+    ret = @rc.download( ["j000001.tar.bz2"], "unconnectable")
+    assert_equal( :connection_failed, ret)
+
+    ret = @rc.download( ["j000001.tar.bz2"], "localhost_submittable")
+    assert_equal( [], ret)
+
+    ret = @rc.submit( [1,2], "localhost_submittable")
+    sleep 3 # wait for completion
+    downloadable = @rc.check_completion("localhost_submittable")
+    ret = @rc.download( downloadable, "localhost_submittable")
+    assert_equal( ["j000001.tar.bz2", "j000002.tar.bz2"], ret)
+
+    downloadable = @rc.check_completion("localhost_submittable")
+    assert_equal( downloadable, [])
+
+    assert( FileTest.exist?(@db_folder+"/Jobs/Include/j000001.tar.bz2") )
+    assert( FileTest.exist?(@db_folder+"/Jobs/Include/j000002.tar.bz2") )
+  end
+
+=begin
+  def test_remote
+    @hosts << { :name => "cruijff",:host_url => "cruijff.t.u-tokyo.ac.jp",:user => 'murase', :job_path => '~/job1', :submission_command => 'qsub', :show_status_command => 'qstatlist'}
+    File.open( "hosts.yml", 'w') do |io|
+      YAML.dump( @hosts, io)
+      io.flush
+    end
+    @rc = NERA::RemoteConnector.new( @db_folder, "hosts.yml")
+
+    puts @rc.show_status("cruijff")
+    
+    @rc.submit( [1,2], "cruijff")
+    sleep 3
+    a = @rc.check_completion("cruijff")
+    @rc.download( a, "cruijff")
+    
+  end
+=end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nera
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors: 
 - Yohsuke Murase
@@ -9,9 +9,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-16 00:00:00 +09:00
+date: 2009-04-30 00:00:00 +09:00
 default_executable: 
 dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: net-ssh
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.11
+    version: 
 - !ruby/object:Gem::Dependency 
   name: newgem
   type: :development
@@ -37,6 +47,7 @@ email:
 - murase@serow.t.u-tokyo.ac.jp
 executables: 
 - nera
+- nera_addhost
 - nera_addsim
 extensions: []
 
@@ -50,6 +61,7 @@ files:
 - README.rdoc
 - Rakefile
 - bin/nera
+- bin/nera_addhost
 - bin/nera_addsim
 - lib/nera.rb
 - lib/nera/nera_cui.rb
@@ -61,11 +73,13 @@ files:
 - lib/nera/nera_job_script.rb
 - lib/nera/nera_parameter_layer_controller.rb
 - lib/nera/nera_parameter_records.rb
+- lib/nera/nera_remote_connector.rb
 - lib/nera/nera_run_layer_controller.rb
 - lib/nera/nera_run_records.rb
 - lib/nera/nera_simulator.rb
 - lib/nera/nera_simulator_layer_controller.rb
 - lib/nera/nera_simulator_records.rb
+- lib/nera_addhost/add_hosts.rb
 - lib/nera_addsim/make_simulator.rb
 - scripts/console
 - scripts/destroy
@@ -80,6 +94,7 @@ files:
 - test/test_nera_job_records.rb
 - test/test_nera_parameter_layer_controller.rb
 - test/test_nera_parameter_records.rb
+- test/test_nera_remote_connector.rb
 - test/test_nera_run_layer_controller.rb
 - test/test_nera_run_records.rb
 - test/test_nera_simulator.rb
@@ -121,6 +136,7 @@ test_files:
 - test/test_nera_job_records.rb
 - test/test_nera_parameter_layer_controller.rb
 - test/test_nera_parameter_records.rb
+- test/test_nera_remote_connector.rb
 - test/test_nera_run_layer_controller.rb
 - test/test_nera_run_records.rb
 - test/test_nera_simulator.rb