comana 0.1.0 → 0.1.1

data/lib/comana/gridengine.rb

@@ -0,0 +1,173 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+#
+#
+#
+class Comana::GridEngine
+
+  def self.write_qsub_script(q_name:, pe_name:, ppn:, ld_library_path: , command:, io:)
+    io.puts "#! /bin/sh"
+    io.puts "#$ -S /bin/sh"
+    io.puts "#$ -cwd"
+    io.puts "#$ -o stdout"
+    io.puts "#$ -e stderr"
+    io.puts "#$ -q #{q_name}"
+    io.puts "#$ -pe #{pe_name} #{ppn}"
+    io.puts "MACHINE_FILE='machines'"
+    io.puts "LD_LIBRARY_PATH=#{ld_library_path}" if ld_library_path
+    io.puts "export LD_LIBRARY_PATH" if ld_library_path
+    io.puts "cd $SGE_O_WORKDIR"
+    io.puts "printenv | sort > printenv.log"
+    io.puts "cut -d ' ' -f 1,2 $PE_HOSTFILE | sed 's/ / cpu=/' > $MACHINE_FILE"
+    io.puts "#{command}"
+    #{__FILE__} execute
+  end
+
+  def self.qstat_f(io = IO.popen("qstat -f -xml", "r+"))
+    qs = Nokogiri::XML(io)
+    results = qs.xpath("/job_info/queue_info/Queue-List").map do |queue|
+      hash = {}
+      queue.children.each do |j|
+        next if j.name == 'text'
+        key = j.name
+        val = j.children.to_s
+        val = val.to_i if val.integer?
+        hash[key] = val
+      end
+      hash
+    end
+    results
+  end
+
+  def self.queue_alive_hosts(io = IO.popen("qstat -f -xml", "r+"))
+    qs = self.qstat_f(io)
+    results = {}
+    qs.each do |q|
+      next if q['state'] == 'au'
+      /(.*)\@(.*)/ =~ q["name"]
+      q = $1
+      host = $2
+      results[q] ||= []
+      results[q] << host
+    end
+    results
+  end
+
+  def self.queue_alive_nums(io = IO.popen("qstat -f -xml", "r+"))
+    qs = self.queue_alive_hosts(io)
+    results = {}
+    qs.each do |key,val|
+      results[key] = val.size
+    end
+    results
+  end
+
+  def self.qstat_u(io = IO.popen("qstat -u '*' -xml", "r+"))
+    qs = Nokogiri::XML(io)
+    results = []
+    qs.xpath("/job_info/queue_info/job_list").each do |queue|
+      hash = {}
+      value = queue.attributes.values[0].to_s
+      hash['job_list_state'] = value
+      queue.children.each do |j|
+        next if j.name == 'text'
+        key = j.name
+        val = j.children.to_s
+        if val.integer?
+          val = val.to_i
+        elsif val.float?
+          val = val.to_f
+        end
+        hash[key] = val
+      end
+      results << hash
+    end
+
+    qs.xpath("/job_info/job_info/job_list").each do |queue|
+      hash = {}
+      value = queue.attributes.values[0].to_s
+      hash['job_list_state'] = value
+      queue.children.each do |j|
+        next if j.name == 'text'
+        key = j.name
+        val = j.children.to_s
+        if val.integer?
+          val = val.to_i
+        elsif val.float?
+          val = val.to_f
+        end
+        hash[key] = val
+      end
+      results << hash
+    end
+
+    results
+  end
+
+  #def self.qconf_sql(str = `qconf -sql`)
+  def self.queues(str = `qconf -sql`)
+    str.strip.split("\n")
+  end
+
+  
+  #def self.queue_jobs(qname, str = `qconf -sql`)
+  def self.queue_jobs(qname, io = nil)
+    io ||= IO.popen("qstat -q #{qname} -u '*' -xml", "r+") 
+    results = []
+    qs = Nokogiri::XML(io)
+    qs.xpath("/job_info/queue_info/job_list").each do |queue|
+      hash = {}
+      value = queue.attributes.values[0].to_s
+      hash['job_list_state'] = value
+      queue.children.each do |j|
+        next if j.name == 'text'
+        key = j.name
+        val = j.children.to_s
+        if val.integer?
+          val = val.to_i
+        elsif val.float?
+          val = val.to_f
+        end
+        hash[key] = val
+      end
+      results << hash
+    end
+
+    qs.xpath("/job_info/job_info/job_list").each do |queue|
+      hash = {}
+      value = queue.attributes.values[0].to_s
+      hash['job_list_state'] = value
+      queue.children.each do |j|
+        next if j.name == 'text'
+        key = j.name
+        val = j.children.to_s
+        if val.integer?
+          val = val.to_i
+        elsif val.float?
+          val = val.to_f
+        end
+        hash[key] = val
+      end
+      results << hash
+    end
+
+    results
+    #qs.xpath("/job_info/job_info/job_list").each do |queue|
+  end
+
+  #nj: number of jobs
+  #nh: number of hosts
+  #bench: benchmark time
+  def self.guess_end_time(nj:, nh:, bench:)
+    nj = nj.to_f
+    nh = nh.to_f
+    if nj < nh
+      start = 0
+    else
+      start = nj/nh
+    end
+    (start + 1) * bench
+  end
+
+end

data/lib/comana/hostinspector.rb

@@ -12,16 +12,15 @@ class Comana::HostInspector
   attr_reader :hostname
 
   #
-  def initialize(hostname)
+  def initialize(hostname, cache_dir)
     @hostname = hostname
-    @cache_dir = "#{CACHE_DIR}/#{@hostname}"
+    @cache_dir = "#{cache_dir}/#{@hostname}"
   end
 
 
-  ##ping
-  #Try ping three times.
-  #Return true if at least one time responds.
-  #def ping3
+  ##Try ping three times.
+  ##Return true if at least one time responds.
+  ##def ping3
   def update_ping
     result = false
     3.times do
@@ -39,11 +38,9 @@ class Comana::HostInspector
   ##cwd
   ##readlink コマンドが使えるかとも思ったが、シムリンク自体の名前が不明瞭になる。
   def update_cwd
-    #str = `ssh #{@hostname} 'ls -l /proc/*/cwd'`
     str = ssh_str('ls -l /proc/\*/cwd 2> /dev/null')
     results = {}
     str.split("\n").each do |line|
-      #pp line
       items = line.split
       pid = items[8].sub(/^\/proc\//, '').sub(/\/cwd$/, '')
       results[pid] = items[10]
@@ -69,26 +66,6 @@ class Comana::HostInspector
   #auxw だと、
   #ippei     2948  198 11.8 4495708 3884740 pts/3 Rl   Apr01 173494:26 /opt/bin/vasp5212openmpi で桁が崩れることがある。
   def update_ps
-    #str = ssh_str('ps -eo "user pid %cpu %mem command"')
-    #results = {}
-    #lines = str.split("\n")
-    #lines.shift  # titles of items
-    #lines.each do |line|
-    #  user     = line[0..7]
-    #  pid      = line[9..13]
-    #  cpu      = line[15..18]
-    #  mem      = line[20..23]
-    #  command  = line[25..-1]
-    #  results[pid] = {
-    #    "user"    => user,
-    #    "cpu"     => cpu,
-    #    "mem"     => mem,
-    #    "command" => command
-    #  }
-    #end
-    #write_cache('ps', results)
-
-    #str = `ssh #{@hostname} 'ps auxw'`
     str = ssh_str('ps auxw')
     results = {}
     lines = str.split("\n")
@@ -109,32 +86,6 @@ class Comana::HostInspector
       }
     end
     write_cache('ps', results)
-
-    ##str = `ssh #{@hostname} 'ps auxw'`
-    #str = ssh_str('ps auxw')
-    #results = {}
-    #lines = str.split("\n")
-    #lines.shift  # titles of items
-    #lines.each do |line|
-    #  user     = line[0..7]
-    #  pid      = line[9..13]
-    #  cpu      = line[15..18]
-    #  mem      = line[20..23]
-    #  #vsz      = line[25..30]
-    #  #rss      = line[32..36]
-    #  #tty      = line[38..45]
-    #  #stat     = line[47..50]
-    #  #start    = line[52..56]
-    #  #time     = line[58..63]
-    #  command  = line[65..-1]
-    #  results[pid] = {
-    #    "user"    => user,
-    #    "cpu"     => cpu,
-    #    "mem"     => mem,
-    #    "command" => command
-    #  }
-    #end
-    #write_cache('ps', results)
   end
 
   # dmesg ログ形式でつらい。
@@ -162,7 +113,6 @@ class Comana::HostInspector
   end
 
   def update_meminfo
-    #str = `ssh #{@hostname} 'cat /proc/meminfo'`
     str = ssh_str('cat /proc/meminfo')
     results = {}
     lines = str.split("\n")
@@ -173,8 +123,6 @@ class Comana::HostInspector
     write_cache('meminfo', results)
   end
 
-  ############################################################
-  ## common
   #Return from cached ping data.
   def fetch(name)
     load_cache(name)
@@ -193,7 +141,6 @@ class Comana::HostInspector
 
   # 先に ping を打ち、返事がなければ 空文字列 を返す。
   def ssh_str(command)
-    #pp command
     update_ping
     if fetch('ping')
       return `ssh #{@hostname} #{command}`
@@ -210,7 +157,6 @@ class Comana::HostInspector
   end
 
   def load_cache(name)
-    #return nil unless File.exist? "#{@cache_dir}/#{name}.yaml"
     cache_file = "#{@cache_dir}/#{name}.yaml"
     unless File.exist? cache_file
       raise NoUpdateFile, "#{cache_file} not found."

data/lib/comana.rb

@@ -1,13 +1,10 @@
 module Comana; end
 
-#require "rubygems"
-#gem     "nokogiri"
-#require "nokogiri"
-
+require 'nokogiri'
+require 'string/integer'
+require 'string/float'
 require "comana/computationmanager.rb"
 require "comana/clustersetting.rb"
 require "comana/hostselector.rb"
-require "comana/queuesubmitter.rb"
 require "comana/hostinspector.rb"
-require "comana/queuemanager.rb"
-require "comana/gridenginescript.rb"
+require "comana/gridengine.rb"

data/test/gridengine/qconfsql.dat

@@ -0,0 +1,13 @@
+Ag.q
+Cd.q
+Ga.q
+Ge.q
+Kr.q
+Nb.q
+Pd.q
+Rb.q
+Rh.q
+Ru.q
+Se.q
+Sr.q
+Tc.q