nera 0.2.1 → 0.2.2

data/History.txt

@@ -1,3 +1,8 @@
+== 0.2.2
+* 1 major enhancement:
+* 2 bug fixes:
+  * fix the bug of the module for remote connections
+
 == 0.2.1
 * 1 major enhancement:
   * "show log" menu is implemented into the CUI program.

data/README.rdoc

@@ -23,7 +23,7 @@ To edit your host informations,
 
 == REQUIREMENTS:
 
-Ruby 1.8.0 or later
+Ruby 1.8.6 or later
 
 == INSTALL:
 

data/lib/nera/nera_cui.rb

@@ -252,7 +252,7 @@ HEADER
         }
         return :run_layer
       when "Create jobs"
-        num_job = Dialog::get_integer( "Input the number of jobs (must be equal or larget than zero)") { |i| i >= 0 }
+        num_job = Dialog::get_integer( "Input the number of jobs (must be equal or larger than zero)") { |i| i >= 0 }
         return :run_layer if num_job == 0
         num_run_per_job = Dialog::get_integer( "Input the number of runs per job (must be larger than zero)") { |i| i > 0 }
         jobids = rlc.create_jobs( num_job, num_run_per_job)

data/lib/nera/nera_job_layer_controller.rb

@@ -113,8 +113,8 @@ module NERA
           next if File.directory?( @db_folders.path_to_include_layer+File.basename( js_path).sub(/\.sh$/,'') ) or File.exist?( @db_folders.path_to_include_layer + File.basename( js_path).sub(/\.sh$/,'.tar.bz2'))
           Dir.chdir( @db_folders.path_to_include_layer) {
             system "nohup #{js_path} &"
-            submitted_jobs << jid
           }
+          submitted_jobs << jid
           @job_records.update_to_state_submitted( jid, "localhost")
           @db_folders.logger.info( self.class.to_s) {
             "submitted a job (jobid:#{jid}) to localhost"
@@ -162,7 +162,7 @@ module NERA
         system(cmd)
         raise "command : #{cmd} failed" unless $? == 0
         tarpath = bz2_path.sub(/.bz2$/,'')
-        cmd = "tar xvf #{tarpath}"
+        cmd = "tar xf #{tarpath}"
         system(cmd)
         raise "command : #{cmd} failed" unless $? == 0
         stat_path = tarpath.sub(/.tar$/,'/nera_status.rb')

data/lib/nera/nera_job_script.rb

@@ -44,7 +44,7 @@ echo \"    :parameters => {\" >> nera_status.rb
 BODY
 
 job_info[:params].each_pair do |key, value|
-job_script += "echo \"        #{key.to_sym.inspect} => #{value.inspect.gsub(/\"/,'\"')},\" >> nera_status.rb\n"
+job_script += "echo \"        #{key.to_sym.inspect.gsub(/\"/,'\"')} => #{value.inspect.gsub(/\"/,'\"')},\" >> nera_status.rb\n"
 end
 
 job_script +=<<"BODY"
@@ -63,7 +63,7 @@ echo \"    #{run_id} => {\" >> nera_status.rb
 echo \"        :seed => #{job_info[:seeds][i]}, :start_at => DateTime.parse( <<'NERA_START_AT'.chomp), :output => <<'NERA_OUTPUT', :finish_at => DateTime.parse( <<'NERA_FINISH_AT'.chomp)\" >> nera_status.rb
 date >> nera_status.rb
 echo \"NERA_START_AT\" >> nera_status.rb
-echo \"#{job_info[:execs][i]}\"
+echo \"#{job_info[:execs][i]}\" >&2
 mkdir -p #{run_id}
 cd #{run_id}
 { time -p { #{job_info[:execs][i]};} } 2>> ../nera_status.rb

data/lib/nera/nera_parameter_layer_controller.rb

@@ -99,8 +99,8 @@ module NERA
           flag = @param_records.update_to_state_trashbox(id)
           return nil unless flag
           r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
-          cmd = "tar cvjf #{r_path}.tar.bz2 #{r_path}"
-          system(cmd)
+          cmd = "tar cjf #{File.basename(r_path)}.tar.bz2 #{File.basename(r_path)}"
+          Dir.chdir( File.dirname(r_path)) { system(cmd) }
           raise "Command #{cmd} failed" unless $? == 0
           FileUtils.rm_r(r_path)
           @db_folders.logger.info(self.class.to_s) {
@@ -138,8 +138,10 @@ module NERA
         flag = @param_records.update_to_state_active(id)
         return nil unless flag
         r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
-        cmd = "tar xvjf #{r_path}.tar.bz2"
-        system(cmd)
+        cmd = "tar xjf #{File.basename(r_path)}.tar.bz2"
+        Dir.chdir( File.dirname(r_path) ) {
+          system(cmd)
+        }
         raise "Command #{cmd} failed" unless $? == 0
         FileUtils.rm(r_path+".tar.bz2")
         @db_folders.logger.info(self.class.to_s) {

data/lib/nera/nera_remote_connector.rb

@@ -62,13 +62,14 @@ module NERA
       return arr
     end
 
-    def show_status( hostname)
+    def show_status( hostname, passphrase = nil)
       found = @hosts.find do |host|
         host[:name] == hostname
       end
       return :connection_failed unless found
       h = { :port => found[:port], :timeout => 10 }
       h[:keys] = found[:keys] if found.has_key?(:keys)
+      h[:passphrase] = passphrase if passphrase
       cmd = 'ps au'
       cmd = found[:show_status_command] if found.has_key?(:show_status_command)
       str = ''
@@ -79,26 +80,26 @@ module NERA
         end
       rescue Errno::ECONNREFUSED => ex
         $stderr.puts "failed"
-        @db_folders.logger.error(self.class.to_s) { ex.to_s }
+        @db_folders.logger.error(self.class.to_s) { Errno::ECONNREFUSED }
         return :connection_failed
       rescue Errno::ENETUNREACH => ex
         $stderr.puts "failed"
-        @db_folders.logger.error(self.class.to_s) { ex.to_s }
+        @db_folders.logger.error(self.class.to_s) { Errno::ENETUNREACH }
         return :connection_failed
       rescue SocketError => ex
         $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { SocketError }
         return :connection_failed
       rescue Net::SSH::Exception => ex
         $stderr.puts "failed"
-        @db_folders.logger.error(self.class.to_s) { ex.to_s }
+        @db_folders.logger.error(self.class.to_s) { Net::SSH::Exception }
         return :connection_failed
       end
       $stderr.puts "done"
       return str
     end
 
-    def transfer( jobids, hostname)
+    def transfer( jobids, hostname, passphrase = nil)
       raise ArgumentError unless jobids.is_a?(Array)
       jobids.each do |jid|
         raise ArgumentError unless jid.is_a?(Integer)
@@ -107,6 +108,8 @@ module NERA
 
       host = @hosts.find do |h| h[:name] == hostname end
       return :connection_failed unless host
+
+      succeeded_jobids = []
       
       @job_records.transaction {
         jids = []
@@ -116,8 +119,9 @@ module NERA
         return :no_such_jobs if jids.size == 0
         
         h = { :port => host[:port] }
+        h[:passphrase] = passphrase if passphrase
         h[:keys] = host[:keys] if host.has_key?(:keys)
-        
+
         begin
         $stderr.print "Connecting with #{host[:name]}.....\t"
           Net::SSH.start( host[:host_url], host[:user], h) do |ssh|
@@ -138,29 +142,30 @@ module NERA
                 @db_folders.logger.info(self.class.to_s) {
                   "transfered a file (#{File.basename(pjs)}) to #{host[:name]}"
                 }
+                succeeded_jobids << jid
               end
             end
           end
         rescue Errno::ECONNREFUSED => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         rescue Errno::ENETUNREACH => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         rescue SocketError => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         rescue Net::SSH::Exception => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         end
         $stderr.puts "done"
       }
-      return :succeeded
+      return succeeded_jobids
     end
 
     private
@@ -169,7 +174,7 @@ module NERA
     end
 
     public
-    def submit( jobids, hostname)
+    def submit( jobids, hostname, passphrase = nil)
       raise ArgumentError unless jobids.is_a?(Array)
       jobids.each do |jid|
         raise ArgumentError unless jid.is_a?(Integer)
@@ -180,7 +185,8 @@ module NERA
       return :connection_failed unless host
 
       return :no_submission_command unless host.has_key?(:submission_command)
-      
+
+      succeeded_jobids = []
       @job_records.transaction {
         jids = []
         jobids.each do |jid|
@@ -189,7 +195,9 @@ module NERA
         return :no_such_jobs if jids.size == 0
         
         h = { :port => host[:port] }
+        h[:passphrase] = passphrase if passphrase
         h[:keys] = host[:keys] if host.has_key?(:keys)
+
         
         begin
         $stderr.print "Connecting with #{host[:name]}.....\t"
@@ -208,43 +216,45 @@ module NERA
                 sftp.file.open(target_path, "w") do |f| f.puts contents end
                 ssh.exec!("chmod +x #{target_path}")
                 @job_records.update_to_state_copied( jid, host[:name])
-                ssh.exec!("cd #{target_dir} && #{host[:submission_command]} #{target_path} &")
+                ssh.exec!("cd #{target_dir} && #{host[:submission_command]} #{target_path}")
                 @job_records.update_to_state_submitted( jid, host[:name])
                 @db_folders.logger.info(self.class.to_s) {
                   "submitted a file (#{File.basename(pjs)}) to #{host[:name]}"
                 }
+                succeeded_jobids << jid
               end
             end
           end
         rescue Errno::ECONNREFUSED => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         rescue Errno::ENETUNREACH => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         rescue SocketError => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         rescue Net::SSH::Exception => ex
           $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
           return :connection_failed
         end
         $stderr.puts "done"
       }
-      return :succeeded
+      return succeeded_jobids
     end
 
-    def check_completion( hostname)
+    def check_completion( hostname, passphrase = nil)
       raise ArgumentError unless hostname.is_a?(String)
 
       host = @hosts.find do |h| h[:name] == hostname end
       return :connection_failed unless host
 
       h = { :port => host[:port] }
+      h[:passphrase] = passphrase if passphrase
       h[:keys] = host[:keys] if host.has_key?(:keys)
 
       ls_str = ''
@@ -287,16 +297,16 @@ module NERA
     end
 
     public
-    def check_completion_all
+    def check_completion_all( passphrase = nil)
       h = {}
       hostnames.each do |hname|
-        h[hname] = check_completion(hname)
+        h[hname] = check_completion(hname, passphrase)
       end
       return h
     end
 
 
-    def download( filenames, hostname)
+    def download( filenames, hostname, passphrase = nil)
       raise ArgumentError unless filenames.is_a?(Array)
       filenames.each do |fname|
         raise ArgumentError unless fname.is_a?(String)
@@ -308,6 +318,7 @@ module NERA
       return :connection_failed unless host
 
       h = { :port => host[:port] }
+      h[:passphrase] = passphrase if passphrase
       h[:keys] = host[:keys] if host.has_key?(:keys)
 
       downloaded = []
@@ -343,19 +354,19 @@ module NERA
         end
       rescue Errno::ECONNREFUSED => ex
         $stderr.puts "failed"
-        @db_folders.logger.error(self.class.to_s) { ex.to_s }
+        @db_folders.logger.error(self.class.to_s) { ex.inspect }
         return :connection_failed
       rescue Errno::ENETUNREACH => ex
         $stderr.puts "failed"
-        @db_folders.logger.error(self.class.to_s) { ex.to_s }
+        @db_folders.logger.error(self.class.to_s) { ex.inspect }
         return :connection_failed
       rescue SocketError => ex
         $stderr.puts "failed"
-          @db_folders.logger.error(self.class.to_s) { ex.to_s }
+          @db_folders.logger.error(self.class.to_s) { ex.inspect }
         return :connection_failed
       rescue Net::SSH::Exception => ex
         $stderr.puts "failed"
-        @db_folders.logger.error(self.class.to_s) { ex.to_s }
+        @db_folders.logger.error(self.class.to_s) { ex.inspect }
         return :connection_failed
       end
       $stderr.puts "done"

data/lib/nera.rb

@@ -22,5 +22,5 @@ require 'nera_simulator_records'
 require 'nera_remote_connector'
 
 module NERA
-  VERSION = '0.2.1'
+  VERSION = '0.2.2'
 end

data/lib/nera_addhost/add_hosts.rb

@@ -6,9 +6,9 @@ module NERA
 
   class CUI_Hosts_Editor
 
-    HOST_INFO = [ [:host_url,String,"localhost"], [:user,String,ENV['USER']], [:job_path,String,"~/"],
+    HOST_INFO = [ [:host_url,String,"host_url"], [:user,String,ENV['USER']], [:job_path,String,"~/"],
                    [:port,Integer,22], [:keys,String,"~/.ssh/id_rsa"],
-                   [:submission_command,String,"nohup"], [:show_status_command,String,"ps au"]]
+                   [:submission_command,String,"qsub"], [:show_status_command,String,"qstat"]]
 
 
     #----- the unique module funciton -----

data/lib/nera_addsim/make_simulator.rb

@@ -41,7 +41,6 @@ BODY
         $stdout.puts "Input the simulator name again."
       end
       return ["seed",nil,nil] if name == "seed"
-      #array << ":#{name}"
       array << name.to_sym.inspect
       $stdout.puts
       num = Dialog.select_one(type, "++Select parameter type+++")
@@ -115,7 +114,8 @@ BODY
             $stdout.puts "Input the simulator name again."
           end
           return ["seed",nil,nil] if name == "seed"
-          param[0] = ":#{name}"
+          # param[0] = ":#{name}"
+          param[0] = name.to_sym.inspect
         when 2
           $stdout.puts
           num = Dialog.select_one(type, "+++Select new parameter type+++ \n #{param[1]} ->")

data/test/test_nera_remote_connector.rb

@@ -18,7 +18,7 @@ class TC_NERA_RM_CONNECTOR < Test::Unit::TestCase
     
     @hosts = []
     @hosts << { :name => "localhost",:host_url => "localhost",:user => ENV['USER'], :job_path => FileUtils.pwd + '/job1'}
-    @hosts << { :name => "localhost_submittable",:host_url => "localhost",:user => ENV['USER'], :job_path => FileUtils.pwd + '/job2', :submission_command => "nohup", :show_status_command => 'ps au' }
+    @hosts << { :name => "localhost_submittable",:host_url => "localhost",:user => ENV['USER'], :job_path => FileUtils.pwd + '/job2', :submission_command => 'nohup', :show_status_command => 'ps au' }
     @hosts << { :name => "unconnectable",:host_url => "unconnectable_host",:user => "hoge", :job_path => '~/job_test', :submission_command => 'qsub', :show_status_command => 'qstat; qstatlist;'}
     File.open( @dbf.path_to_hosts_yaml, 'w') do |io|
       YAML.dump( @hosts, io)
@@ -85,7 +85,7 @@ class TC_NERA_RM_CONNECTOR < Test::Unit::TestCase
     assert_equal( s2+1, s3)
 
     ret = @rc.transfer( [1,2], "localhost")
-    assert_equal( :succeeded, ret)
+    assert_equal( [1,2], ret)
     s3 = File.open(logp,'r').readlines.size
     assert_equal( s2+3, s3)
 
@@ -138,7 +138,7 @@ class TC_NERA_RM_CONNECTOR < Test::Unit::TestCase
     assert_equal( s1+1, s2)
 
     ret = @rc.submit( [1,2], "localhost_submittable")
-    assert_equal( :succeeded, ret)
+    assert_equal( [1,2], ret)
     s2 = File.open(logp,'r').readlines.size
     assert_equal( s1+3, s2)
 
@@ -242,23 +242,4 @@ class TC_NERA_RM_CONNECTOR < Test::Unit::TestCase
     assert( FileTest.exist?(@db_folder+"/Jobs/Include/j000002.tar.bz2") )
   end
 
-=begin
-  def test_remote
-    @hosts << { :name => "cruijff",:host_url => "cruijff.t.u-tokyo.ac.jp",:user => 'murase', :job_path => '~/job1', :submission_command => 'qsub', :show_status_command => 'qstatlist'}
-    File.open( "hosts.yml", 'w') do |io|
-      YAML.dump( @hosts, io)
-      io.flush
-    end
-    @rc = NERA::RemoteConnector.new( @db_folder, "hosts.yml")
-
-    puts @rc.show_status("cruijff")
-    
-    @rc.submit( [1,2], "cruijff")
-    sleep 3
-    a = @rc.check_completion("cruijff")
-    @rc.download( a, "cruijff")
-    
-  end
-=end
-
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nera
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors: 
 - Yohsuke Murase
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-05-25 00:00:00 +09:00
+date: 2009-08-05 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -40,7 +40,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.3.0
+        version: 1.4.1
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe