comana 0.0.10 → 0.1.0

data/lib/comana/hostinspector.rb

@@ -1,4 +1,223 @@
-#! /usr/bin/env ruby
+#s! /usr/bin/env ruby
 # coding: utf-8
+#
+require "yaml"
+
+class Comana::HostInspector
+  PING_MIN_INTERVAL = 1
+  CACHE_DIR = "#{ENV['HOME']}/var/comana"
+
+  class NoUpdateFile < Exception; end
+
+  attr_reader :hostname
+
+  #
+  def initialize(hostname)
+    @hostname = hostname
+    @cache_dir = "#{CACHE_DIR}/#{@hostname}"
+  end
+
+
+  ##ping
+  #Try ping three times.
+  #Return true if at least one time responds.
+  #def ping3
+  def update_ping
+    result = false
+    3.times do
+      command =
+        "ping -c 1 -W #{PING_MIN_INTERVAL} #{@hostname} 2> /dev/null 1> /dev/null"
+      if system(command)
+        result = true
+        break
+      end
+    end
+
+    write_cache("ping", result)
+  end
+
+  ##cwd
+  ##readlink コマンドが使えるかとも思ったが、シムリンク自体の名前が不明瞭になる。
+  def update_cwd
+    #str = `ssh #{@hostname} 'ls -l /proc/*/cwd'`
+    str = ssh_str('ls -l /proc/\*/cwd 2> /dev/null')
+    results = {}
+    str.split("\n").each do |line|
+      #pp line
+      items = line.split
+      pid = items[8].sub(/^\/proc\//, '').sub(/\/cwd$/, '')
+      results[pid] = items[10]
+    end
+
+    write_cache('cwd', results)
+  end
+
+  ##processes
+  #%ps auxw
+  #USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+  #ippei    28971  0.0  0.0 103684  3764 ?        S    15:19   0:00 sshd: ippei@pts/19
+  #root         1  0.0  0.0  33876  2280 ?        Ss    5月17   0:22 /sbin/init
+  #0---------1---------2---------3---------4---------5---------6---------7
+  #0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0
+  #
+  #% ps -o 'user pid %cpu %mem command'
+  #USER       PID %CPU %MEM COMMAND
+  #ippei    19991  0.0  0.2 zsh
+  #ippei    23217  0.0  0.0 ps -o user pid %cpu %mem command
+  #0---------1---------2---------3---------4---------5---------6---------7
+  #0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0-2-4-6-8-0
+  #auxw だと、
+  #ippei     2948  198 11.8 4495708 3884740 pts/3 Rl   Apr01 173494:26 /opt/bin/vasp5212openmpi で桁が崩れることがある。
+  def update_ps
+    #str = ssh_str('ps -eo "user pid %cpu %mem command"')
+    #results = {}
+    #lines = str.split("\n")
+    #lines.shift  # titles of items
+    #lines.each do |line|
+    #  user     = line[0..7]
+    #  pid      = line[9..13]
+    #  cpu      = line[15..18]
+    #  mem      = line[20..23]
+    #  command  = line[25..-1]
+    #  results[pid] = {
+    #    "user"    => user,
+    #    "cpu"     => cpu,
+    #    "mem"     => mem,
+    #    "command" => command
+    #  }
+    #end
+    #write_cache('ps', results)
+
+    #str = `ssh #{@hostname} 'ps auxw'`
+    str = ssh_str('ps auxw')
+    results = {}
+    lines = str.split("\n")
+    lines.shift  # titles of items
+    lines.each do |line|
+      items = line.split
+      user    = items[0]
+      pid     = items[1]
+      cpu     = items[2]
+      mem     = items[3]
+      command = items[10]
+
+      results[pid] = {
+        "user"    => user,
+        "cpu"     => cpu,
+        "mem"     => mem,
+        "command" => command
+      }
+    end
+    write_cache('ps', results)
+
+    ##str = `ssh #{@hostname} 'ps auxw'`
+    #str = ssh_str('ps auxw')
+    #results = {}
+    #lines = str.split("\n")
+    #lines.shift  # titles of items
+    #lines.each do |line|
+    #  user     = line[0..7]
+    #  pid      = line[9..13]
+    #  cpu      = line[15..18]
+    #  mem      = line[20..23]
+    #  #vsz      = line[25..30]
+    #  #rss      = line[32..36]
+    #  #tty      = line[38..45]
+    #  #stat     = line[47..50]
+    #  #start    = line[52..56]
+    #  #time     = line[58..63]
+    #  command  = line[65..-1]
+    #  results[pid] = {
+    #    "user"    => user,
+    #    "cpu"     => cpu,
+    #    "mem"     => mem,
+    #    "command" => command
+    #  }
+    #end
+    #write_cache('ps', results)
+  end
+
+  # dmesg ログ形式でつらい。
+  # /proc/cpuinfo コアごとにでるのでパースめんどう。
+  # lscpu これだと思ったら、CPU MHz がずれてる。ハードウェアで想定される値ではなく、
+  # 実際の速度で書かれるらしい。
+  # 負荷の有無で値がかわる。
+  def update_cpuinfo
+    #str = `ssh #{@hostname} 'cat /proc/cpuinfo'`
+    str = ssh_str('cat /proc/cpuinfo')
+    results = []
+    cur_index = 0
+    results[cur_index] = {}
+    lines = str.split("\n")
+    lines.each do |line|
+      if line =~ /^\s*$/
+        cur_index += 1
+        results[cur_index] = {}
+        next
+      end
+      key, value = line.split(/\s*:\s*/)
+      results[cur_index][key] = value
+    end
+    write_cache('cpuinfo', results)
+  end
+
+  def update_meminfo
+    #str = `ssh #{@hostname} 'cat /proc/meminfo'`
+    str = ssh_str('cat /proc/meminfo')
+    results = {}
+    lines = str.split("\n")
+    lines.each do |line|
+      key, value = line.split(/\s*:\s*/)
+      results[key] = value
+    end
+    write_cache('meminfo', results)
+  end
+
+  ############################################################
+  ## common
+  #Return from cached ping data.
+  def fetch(name)
+    load_cache(name)
+  end
+
+  def time_updated(name)
+    ping_file = "#{@cache_dir}/#{name}.yaml"
+    if File.exist?  ping_file
+      return File.mtime(ping_file)
+    else
+      return nil
+    end
+  end
+
+  private
+
+  # 先に ping を打ち、返事がなければ 空文字列 を返す。
+  def ssh_str(command)
+    #pp command
+    update_ping
+    if fetch('ping')
+      return `ssh #{@hostname} #{command}`
+    else
+      return ''
+    end
+  end
+
+  def write_cache(name, value)
+    FileUtils.mkdir_p @cache_dir
+    File.open("#{@cache_dir}/#{name}.yaml", "w") do |io|
+      YAML.dump(value, io)
+    end
+  end
+
+  def load_cache(name)
+    #return nil unless File.exist? "#{@cache_dir}/#{name}.yaml"
+    cache_file = "#{@cache_dir}/#{name}.yaml"
+    unless File.exist? cache_file
+      raise NoUpdateFile, "#{cache_file} not found."
+    end
+
+    YAML.load_file("#{@cache_dir}/#{name}.yaml")
+  end
+
+end
 
-module Comana::HostInspector; end

data/lib/comana/queuemanager.rb

@@ -0,0 +1,34 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+#
+#
+#
+class Comana::QueueManager
+  #
+  def initialize()
+  end
+
+  #
+  def queues
+    `qconf -sql`.split("\n")
+  end
+
+  # return a queue name with less tasks.
+  def light_queue
+    queues.each do |queue|
+      hostlist(queue)
+    end
+
+  end
+
+  def qhost
+    
+  end
+
+  private
+
+
+
+end
+

data/lib/comana/queuesubmitter.rb

@@ -4,6 +4,8 @@
 require "fileutils"
 #require "comana/computationmanager.rb"
 #require "comana/clustersetting.rb"
+#
+#For Torque. This class will be obsoleted.
 
 #
 #
@@ -53,7 +55,7 @@ class Comana::QueueSubmitter < Comana::ComputationManager
     script_path = "#{@dir}/#{SCRIPT}"
     File.open(script_path, "w") { |io| dump_script(io) }
 
-    # run
+    # exec
     system("cd #{@dir}; qsub -l prologue=#{prologue_path} -l epilogue=#{epilogue_path} #{script_path} > jobid.log")
   end
 

data/lib/comana.rb

@@ -9,5 +9,5 @@ require "comana/clustersetting.rb"
 require "comana/hostselector.rb"
 require "comana/queuesubmitter.rb"
 require "comana/hostinspector.rb"
-require "comana/hostinspector/ping.rb"
-require "comana/hostinspector/pbsnodes.rb"
+require "comana/queuemanager.rb"
+require "comana/gridenginescript.rb"

data/test/hostinspector/.gitignore

@@ -0,0 +1 @@
+var

data/test/test_gridenginescript.rb

@@ -0,0 +1,16 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+require "pp"
+require "helper"
+#require "test/unit"
+#require "pkg/klass.rb"
+
+
+class TC_GridEngineScript < Test::Unit::TestCase
+  def setup
+    @g00 = GridEngineScript.new
+  end
+
+end
+

data/test/test_hostinspector.rb

@@ -0,0 +1,112 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+require "helper"
+#require "test/unit"
+#require "pkg/klass.rb"
+
+
+class Comana::HostInspector
+  CACHE_DIR = "test/hostinspector/var"
+end
+
+class TC_HostInspector < Test::Unit::TestCase
+  CACHE_DIR = Comana::HostInspector::CACHE_DIR 
+
+  def setup
+      @h00 = Comana::HostInspector.new("localhost")
+      @h01 = Comana::HostInspector.new("__not_exist_host__")
+  end
+
+  def test_update_ping
+    ping_file = "#{CACHE_DIR}/#{@h00.hostname}/ping.yaml"
+    #FileUtils.rm_rf "#{CACHE_DIR}/#{@h00.hostname}"
+    FileUtils.rm ping_file if File.exist?(ping_file)
+    assert_equal(false, File.exist?(ping_file))
+    @h00.update_ping
+    assert_equal(true, File.exist?(ping_file))
+    assert_equal(true, YAML.load_file(ping_file))
+
+    ping_file = "#{CACHE_DIR}/#{@h01.hostname}/ping.yaml"
+    FileUtils.rm ping_file if File.exist?(ping_file)
+    assert_equal(false, File.exist?(ping_file))
+    @h01.update_ping
+    assert_equal(true, File.exist?(ping_file))
+    assert_equal(false, YAML.load_file(ping_file))
+  end
+
+  def test_update_cwd
+    cwd_file = "#{CACHE_DIR}/#{@h00.hostname}/cwd.yaml"
+    FileUtils.rm cwd_file if File.exist?(cwd_file)
+    assert_equal(false, File.exist?(cwd_file))
+    @h00.update_cwd
+    assert_equal(true, File.exist?(cwd_file))
+    assert_equal(Hash, YAML.load_file(cwd_file).class)
+
+    #cwd_file = "#{CACHE_DIR}/#{@h01.hostname}/cwd.yaml"
+    #FileUtils.rm cwd_file if File.exist?(cwd_file)
+    #assert_equal(false, File.exist?(cwd_file))
+    #@h01.update_cwd
+    #assert_equal(true, File.exist?(cwd_file))
+    #assert_equal(Hash, YAML.load_file(cwd_file).class)
+  end
+
+  def test_update_ps
+    ps_file = "#{CACHE_DIR}/#{@h00.hostname}/ps.yaml"
+    FileUtils.rm ps_file if File.exist?(ps_file)
+    assert_equal(false, File.exist?(ps_file))
+    @h00.update_ps
+    assert_equal(true, File.exist?(ps_file))
+    assert_equal(Hash, YAML.load_file(ps_file).class)
+  end
+
+  def test_update_cpuinfo
+    cpuinfo_file = "#{CACHE_DIR}/#{@h00.hostname}/cpuinfo.yaml"
+    FileUtils.rm cpuinfo_file if File.exist?(cpuinfo_file)
+    assert_equal(false, File.exist?(cpuinfo_file))
+    @h00.update_cpuinfo
+    assert_equal(true, File.exist?(cpuinfo_file))
+
+    results = YAML.load_file(cpuinfo_file)
+    assert_equal(Array, results.class)
+    assert_equal(Hash, results[0].class)
+  end
+
+  def test_update_meminfo
+    meminfo_file = "#{CACHE_DIR}/#{@h00.hostname}/meminfo.yaml"
+    FileUtils.rm meminfo_file if File.exist?(meminfo_file)
+    assert_equal(false, File.exist?(meminfo_file))
+    @h00.update_meminfo
+    assert_equal(true, File.exist?(meminfo_file))
+
+    results = YAML.load_file(meminfo_file)
+    assert_equal(Hash, results.class)
+  end
+
+  def test_fetch
+    @h00.update_ping
+    assert_equal(true, @h00.fetch('ping'))
+
+    @h01.update_ping
+    assert_equal(false, @h01.fetch('ping'))
+
+    ping_file = "#{CACHE_DIR}/#{@h00.hostname}/ping.yaml"
+    FileUtils.rm ping_file if File.exist?(ping_file)
+    assert_raise(Comana::HostInspector::NoUpdateFile) do
+      @h00.fetch('ping')
+    end
+    @h00.update_ping
+    assert_equal(true, @h00.fetch('ping'))
+  end
+
+  def test_time_ping
+    time = Time.local(2015, 5, 29, 22, 00, 17)
+    ping_file = "#{CACHE_DIR}/#{@h00.hostname}/ping.yaml"
+    #pp File.mtime(ping_file)
+    File::utime(time, time, ping_file)
+    #pp File.mtime(ping_file)
+    assert_equal(time, File.mtime(ping_file))
+  end
+
+end
+

data/test/test_queuemanager.rb

@@ -0,0 +1,19 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+require "helper"
+#require "test/unit"
+#require "pkg/klass.rb"
+
+class TC_QueueManager < Test::Unit::TestCase
+  def setup
+    @qm00 = Comana::QueueManager.new
+  end
+
+  def test_qhost
+    @qm00.qhost
+
+  end
+
+end
+

metadata

@@ -1,103 +1,109 @@
 --- !ruby/object:Gem::Specification
 name: comana
 version: !ruby/object:Gem::Version
-  version: 0.0.10
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - ippei94da
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-28 00:00:00.000000000 Z
+date: 2016-03-04 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: '1.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: tefil
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: '0.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.3
-description: ! "Comana, COmputation MANAger,\n    is a software to provide a framework
+        version: '0.1'
+description: "Comana, COmputation MANAger,\n    is a software to provide a framework
   of\n    managing scientific computing.\n    Researchers on computing have to check
   calculation and\n    generate new calculation and execute, repeatedly.\n    The
   abstract class that this gem provide would help the work.\n  "
 email: ippei94da@gmail.com
 executables:
+- genqsub
+- hostinfo
 - machinestatus
+- queueinfo
 - scpall
 - sshall
 extensions: []
@@ -105,28 +111,33 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
 files:
-- .document
-- .rspec
+- ".document"
+- ".rspec"
 - CHANGES
 - Gemfile
 - LICENSE.txt
 - README.rdoc
 - Rakefile
 - VERSION
+- bin/genqsub
+- bin/hostinfo
 - bin/machinestatus
+- bin/queueinfo
 - bin/scpall
 - bin/sshall
+- comana.gemspec
 - example/dot.clustersetting
 - lib/comana.rb
 - lib/comana/clustersetting.rb
 - lib/comana/computationmanager.rb
+- lib/comana/gridenginescript.rb
 - lib/comana/hostinspector.rb
-- lib/comana/hostinspector/pbsnodes.rb
-- lib/comana/hostinspector/ping.rb
 - lib/comana/hostselector.rb
+- lib/comana/queuemanager.rb
 - lib/comana/queuesubmitter.rb
 - memo.txt
 - test/helper.rb
+- test/hostinspector/.gitignore
 - test/locked/input_a
 - test/locked/input_b
 - test/locked/lock_comana/dummy
@@ -149,36 +160,33 @@ files:
 - test/queuesubmitter/unlocked/dummy
 - test/test_clustersetting.rb
 - test/test_computationmanager.rb
-- test/test_hostinspector_pbsnodes.rb
-- test/test_hostinspector_ping.rb
+- test/test_gridenginescript.rb
+- test/test_hostinspector.rb
 - test/test_hostselector.rb
+- test/test_queuemanager.rb
 - test/test_queuesubmitter.rb
 homepage: http://github.com/ippei94da/comana
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3582063889889649984
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Manager for scientific computing
 test_files: []