comana 0.1.0 → 0.1.1

data/test/test_computationmanager.rb

@@ -3,13 +3,11 @@
 
 require "fileutils"
 require "helper"
-
-#require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require "pp"
 
 NOW = Time.now
 
 class Comana::ComputationManager
-  #public :latest_modified_time, :started?
   public :started?
 end
 
@@ -22,12 +20,10 @@ class TC_ComputationManager < Test::Unit::TestCase
     def finished?         ; false     ; end
   end
 
-  #describe Comana::ComputationManager, "with output, without lock" do
   class CalcStarted < Comana::ComputationManager
     def finished?         ; false     ; end
   end
 
-  #describe Comana::ComputationManager, "terminated" do
   class CalcTerminated < Comana::ComputationManager
     def finished?         ; false     ; end
     def initialize(dir)
@@ -37,12 +33,10 @@ class TC_ComputationManager < Test::Unit::TestCase
     end
   end
 
-  #describe Comana::ComputationManager, "finished" do
   class CalcFinished    < Comana::ComputationManager
     def finished?         ; true      ; end
   end
 
-  #describe Comana::ComputationManager, "cannot execute" do
   class CalcNotExecutable    < Comana::ComputationManager
     def calculate
       end_status = system "" # notExistCommand
@@ -54,14 +48,10 @@ class TC_ComputationManager < Test::Unit::TestCase
     end
 
     def prepare_next
-      #return false
       raise
     end
   end
 
-
-
-
   def setup
     calc_dir = "test/not_started"
     @calc00 = CalcYet.new(calc_dir)
@@ -110,22 +100,66 @@ class TC_ComputationManager < Test::Unit::TestCase
   end
 
   def test_latest_modified_time
-    #it "should return latest modified time" do
     assert_equal(NOW - 1000, @calc00.latest_modified_time)
   end
 
   def test_started?
-    #it "should return false without lock." do
     assert_not_nil(@calc00.started?)
     assert_equal(false, @calc00.started?)
 
-    #it "should return true with lock." do
     assert_equal(false, @calc00.started?)
   end
 
   def test_start
-    #it "should raise error" do
     assert_raise(Comana::ComputationManager::ExecuteError){ @calc_not_exe.start}
   end
+
+  def test_effective_queue
+    ## 空きホストがあるときは benchmarks の短い方
+    queues = ['A.q', 'B.q']
+    jobs  = {'A.q' => 1, 'B.q' => 0}
+    hosts = {'A.q' => 2, 'B.q' => 2}
+    benchmarks = {'A.q' => 1.0, 'B.q' => 2.0}
+    r = Comana::ComputationManager.effective_queue(queues, jobs, hosts, benchmarks)
+    c = 'A.q'
+    assert_equal(c, r)
+
+    ## 空きホストがあるなかで選ぶ。
+    queues = ['A.q', 'B.q']
+    jobs  = {'A.q' => 2, 'B.q' => 0}
+    hosts = {'A.q' => 2, 'B.q' => 2}
+    benchmarks = {'A.q' => 1.0, 'B.q' => 2.0}
+    r = Comana::ComputationManager.effective_queue(queues, jobs, hosts, benchmarks)
+    c = 'B.q'
+    assert_equal(c, r)
+
+    ## 全てのホストが埋まっていたら、見込み時間の早いもので。
+    queues = ['A.q', 'B.q']
+    jobs  = {'A.q' => 2, 'B.q' => 2}
+    hosts = {'A.q' => 2, 'B.q' => 2}
+    benchmarks = {'A.q' => 1.0, 'B.q' => 2.0}
+    r = Comana::ComputationManager.effective_queue(queues, jobs, hosts, benchmarks)
+    c = 'A.q'
+    assert_equal(c, r)
+
+    ## 全てのホストが埋まっていたら、見込み時間の早いもので。
+    queues = ['A.q', 'B.q']
+    jobs  = {'A.q' => 99, 'B.q' => 2}
+    hosts = {'A.q' => 2, 'B.q' => 2}
+    benchmarks = {'A.q' => 1.0, 'B.q' => 2.0}
+    r = Comana::ComputationManager.effective_queue(queues, jobs, hosts, benchmarks)
+    c = 'B.q'
+    assert_equal(c, r)
+  end
+
+  def test_guess_end_time
+    assert_equal(1.0, Comana::ComputationManager.guess_end_time(0, 1, 1.0))
+    assert_equal(1.0, Comana::ComputationManager.guess_end_time(1, 2, 1.0))
+    assert_equal(2.0, Comana::ComputationManager.guess_end_time(1, 1, 1.0))
+    assert_equal(3.0, Comana::ComputationManager.guess_end_time(2, 2, 2.0))
+    assert_equal(4.5, Comana::ComputationManager.guess_end_time(8, 4, 2.0))
+  end
+
+
 end
 

data/test/test_gridengine.rb

@@ -0,0 +1,198 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+require "pp"
+require "helper"
+
+
+class TC_GridEngine < Test::Unit::TestCase
+  def test_write_qsub_script
+    io = StringIO.new
+
+    Comana::GridEngine.write_qsub_script(
+      q_name:  'Cd.q',
+      pe_name: 'Cd.openmpi',
+      ppn: '4',
+      ld_library_path: '/usr/lib:/usr/local/lib:/opt/intel/mkl/lib/intel64:/opt/intel/lib/intel64:/opt/intel/lib:/opt/openmpi-intel/lib',
+      command: '/opt/openmpi-intel/bin/mpiexec -machinefile machines -np $NSLOTS /opt/bin/vasp5212openmpi',
+      io: io
+    )
+    io.rewind
+    results = io.readlines
+    corrects = [
+      "#! /bin/sh\n",
+      "#$ -S /bin/sh\n",
+      "#$ -cwd\n",
+      "#$ -o stdout\n",
+      "#$ -e stderr\n",
+      "#$ -q Cd.q\n",
+      "#$ -pe Cd.openmpi 4\n",
+      "MACHINE_FILE='machines'\n",
+      "LD_LIBRARY_PATH=/usr/lib:/usr/local/lib:/opt/intel/mkl/lib/intel64:/opt/intel/lib/intel64:/opt/intel/lib:/opt/openmpi-intel/lib\n",
+      "export LD_LIBRARY_PATH\n",
+      "cd $SGE_O_WORKDIR\n",
+      "printenv | sort > printenv.log\n",
+      "cut -d ' ' -f 1,2 $PE_HOSTFILE | sed 's/ / cpu=/' > $MACHINE_FILE\n",
+      "/opt/openmpi-intel/bin/mpiexec -machinefile machines -np $NSLOTS /opt/bin/vasp5212openmpi\n",
+    ]
+    assert_equal(corrects, results)
+    
+  end
+
+  def test_qstat_f
+    results = Comana::GridEngine.qstat_f(File.open('test/gridengine/qstatf1.xml', 'r'))
+    corrects = [
+      {'name' =>'Ag.q@Ag00.calc.atom',
+       'qtype' =>'BIP',
+       'slots_used' =>4,
+       'slots_resv' =>0,
+       'slots_total' =>4,
+       'arch' =>'lx26-amd64'},
+
+      {'name' =>'Ag.q@Ag01.calc.atom',
+       'qtype' =>'BIP',
+       'slots_used' =>4,
+       'slots_resv' =>0,
+       'slots_total' =>4,
+       'arch' =>'lx26-amd64'},
+
+      {'name' =>'Cd.q@Cd00.calc.atom',
+       'qtype' =>'BIP',
+       'slots_used' =>0,
+       'slots_resv' =>0,
+       'slots_total' =>4,
+       'arch' =>'lx26-amd64'},
+
+      {'name' =>'Cd.q@Cd02.calc.atom',
+       'qtype' =>'BIP',
+       'slots_used' =>0,
+       'slots_resv' =>0,
+       'slots_total' =>4,
+       'state' => 'au',
+      },
+    ]
+    assert_equal(corrects, results)
+  end
+
+  def test_queue_alive_hosts
+    results = Comana::GridEngine.queue_alive_hosts(File.open('test/gridengine/qstatf1.xml', 'r'))
+    corrects = {
+      'Ag.q' => ["Ag00.calc.atom", "Ag01.calc.atom"],
+      'Cd.q' => ['Cd00.calc.atom']
+    }
+    assert_equal(corrects, results)
+  end
+
+  def test_queue_alive_num
+    results = Comana::GridEngine.queue_alive_nums(File.open('test/gridengine/qstatf1.xml', 'r'))
+    corrects = {
+      'Ag.q' => 2,
+      'Cd.q' => 1
+    }
+    assert_equal(corrects, results)
+  end
+
+  def test_qstat_u
+    results = Comana::GridEngine.qstat_u(File.open('test/gridengine/qstatu1.xml', 'r'))
+    corrects = [
+      { 'job_list_state' => "running",
+        'JB_job_number' =>  26,
+        'JAT_prio' => 0.75000,
+        'JB_name' => 'vasp-Ag.qsub',
+        'JB_owner' => 'koyama',
+        'state' => 'r',
+        'JAT_start_time'     => '2016-02-16T17:53:44',
+        'queue_name'=> 'Ag.q@Ag01.calc.atom',
+        'slots' => 4,
+      }, {'job_list_state' => "running",
+       'JB_job_number' =>  46,
+       'JAT_prio' => 0.73443,
+       'JB_name' => 'vasp-Tc.qsub',
+       'JB_owner' => 'koyama',
+       'state' => 'dr',
+       'JAT_start_time'     => '2016-02-17T09:31:36',
+       'queue_name'=> 'Tc.q@Tc04.calc.atom',
+       'slots' => 4,
+      }, {'job_list_state' => "running",
+       'JB_job_number' => 283,
+       'JAT_prio' => 0.51059,
+       'JB_name' => 'vasp-Ag.qsub',
+       'JB_owner' => 'koyama',
+       'state' => 'r',
+       'JAT_start_time'     => '2016-02-26T18:10:43',
+       'queue_name'=> 'Ag.q@Ag05.calc.atom',
+       'slots' => 4,
+      }, {'job_list_state' => "pending",
+       'JB_job_number' => 468,
+       'JAT_prio' => 0.25000,
+       'JB_name' => 'vasp-Ga.qsub',
+       'JB_owner' => 'ippei',
+       'state' => 'qw',
+       'JB_submission_time' => '2016-03-08T15:43:29',
+       'queue_name'=> '',
+       'slots' => 4,
+      },
+    ]
+    assert_equal(corrects, results)
+  end
+
+  def test_queues
+    str = "Ag.q\nGa.q"
+    results = Comana::GridEngine.queues(str)
+    corrects = ['Ag.q', 'Ga.q']
+    assert_equal(corrects, results)
+  end
+
+  def test_queue_jobs
+    io = File.open('test/gridengine/qstatq1.xml', 'r')
+    results = Comana::GridEngine.queue_jobs('Ga.q', io)
+    corrects = [
+      {"job_list_state"=>"running",
+      "JB_job_number"=>557,
+      "JAT_prio"=>0.25,
+      "JB_name"=>"vasp-Ga.qsub",
+      "JB_owner"=>"ippei",
+      "state"=>"r",
+      "JAT_start_time"=>"2016-03-08T16:59:02",
+      "queue_name"=>"Ga.q@Ga00.calc.atom",
+      "slots"=>4},
+      {"job_list_state"=>"running",
+      "JB_job_number"=>563,
+      "JAT_prio"=>0.25,
+      "JB_name"=>"vasp-Ga.qsub",
+      "JB_owner"=>"ippei",
+      "state"=>"r",
+      "JAT_start_time"=>"2016-03-08T16:59:02",
+      "queue_name"=>"Ga.q@Ga01.calc.atom",
+      "slots"=>4},
+      {"job_list_state"=>"pending",
+      "JB_job_number"=>564,
+      "JAT_prio"=>0.25,
+      "JB_name"=>"vasp-Ga.qsub",
+      "JB_owner"=>"ippei",
+      "state"=>"qw",
+      "JB_submission_time"=>"2016-03-08T16:58:13",
+      "queue_name"=>"",
+      "slots"=>4}
+    ]
+    assert_equal(corrects, results)
+  end
+
+  def test_guess_end_time
+    assert_equal(1.00, Comana::GridEngine.guess_end_time(nj: 0, nh: 4, bench: 1.0))
+    assert_equal(1.00, Comana::GridEngine.guess_end_time(nj: 1, nh: 4, bench: 1.0))
+    assert_equal(1.00, Comana::GridEngine.guess_end_time(nj: 2, nh: 4, bench: 1.0))
+    assert_equal(1.00, Comana::GridEngine.guess_end_time(nj: 3, nh: 4, bench: 1.0))
+    assert_equal(2.00, Comana::GridEngine.guess_end_time(nj: 4, nh: 4, bench: 1.0))
+    assert_equal(2.25, Comana::GridEngine.guess_end_time(nj: 5, nh: 4, bench: 1.0))
+    assert_equal(2.50, Comana::GridEngine.guess_end_time(nj: 6, nh: 4, bench: 1.0))
+
+    assert_equal(2.00, Comana::GridEngine.guess_end_time(nj: 0, nh: 4, bench: 2.0))
+    assert_equal(2.00, Comana::GridEngine.guess_end_time(nj: 1, nh: 4, bench: 2.0))
+    assert_equal(2.00, Comana::GridEngine.guess_end_time(nj: 2, nh: 4, bench: 2.0))
+    assert_equal(2.00, Comana::GridEngine.guess_end_time(nj: 3, nh: 4, bench: 2.0))
+    assert_equal(4.00, Comana::GridEngine.guess_end_time(nj: 4, nh: 4, bench: 2.0))
+    assert_equal(4.50, Comana::GridEngine.guess_end_time(nj: 5, nh: 4, bench: 2.0))
+    assert_equal(5.00, Comana::GridEngine.guess_end_time(nj: 6, nh: 4, bench: 2.0))
+  end
+end

data/test/test_hostinspector.rb

@@ -2,32 +2,23 @@
 # coding: utf-8
 
 require "helper"
-#require "test/unit"
-#require "pkg/klass.rb"
-
-
-class Comana::HostInspector
-  CACHE_DIR = "test/hostinspector/var"
-end
 
 class TC_HostInspector < Test::Unit::TestCase
-  CACHE_DIR = Comana::HostInspector::CACHE_DIR 
-
   def setup
-      @h00 = Comana::HostInspector.new("localhost")
-      @h01 = Comana::HostInspector.new("__not_exist_host__")
+    @cache_dir = "test/hostinspector/var"
+    @h00 = Comana::HostInspector.new("localhost", @cache_dir)
+    @h01 = Comana::HostInspector.new("__not_exist_host__", @cache_dir)
   end
 
   def test_update_ping
-    ping_file = "#{CACHE_DIR}/#{@h00.hostname}/ping.yaml"
-    #FileUtils.rm_rf "#{CACHE_DIR}/#{@h00.hostname}"
+    ping_file = "#{@cache_dir}/#{@h00.hostname}/ping.yaml"
     FileUtils.rm ping_file if File.exist?(ping_file)
     assert_equal(false, File.exist?(ping_file))
     @h00.update_ping
     assert_equal(true, File.exist?(ping_file))
     assert_equal(true, YAML.load_file(ping_file))
 
-    ping_file = "#{CACHE_DIR}/#{@h01.hostname}/ping.yaml"
+    ping_file = "#{@cache_dir}/#{@h01.hostname}/ping.yaml"
     FileUtils.rm ping_file if File.exist?(ping_file)
     assert_equal(false, File.exist?(ping_file))
     @h01.update_ping
@@ -36,23 +27,16 @@ class TC_HostInspector < Test::Unit::TestCase
   end
 
   def test_update_cwd
-    cwd_file = "#{CACHE_DIR}/#{@h00.hostname}/cwd.yaml"
+    cwd_file = "#{@cache_dir}/#{@h00.hostname}/cwd.yaml"
     FileUtils.rm cwd_file if File.exist?(cwd_file)
     assert_equal(false, File.exist?(cwd_file))
     @h00.update_cwd
     assert_equal(true, File.exist?(cwd_file))
     assert_equal(Hash, YAML.load_file(cwd_file).class)
-
-    #cwd_file = "#{CACHE_DIR}/#{@h01.hostname}/cwd.yaml"
-    #FileUtils.rm cwd_file if File.exist?(cwd_file)
-    #assert_equal(false, File.exist?(cwd_file))
-    #@h01.update_cwd
-    #assert_equal(true, File.exist?(cwd_file))
-    #assert_equal(Hash, YAML.load_file(cwd_file).class)
   end
 
   def test_update_ps
-    ps_file = "#{CACHE_DIR}/#{@h00.hostname}/ps.yaml"
+    ps_file = "#{@cache_dir}/#{@h00.hostname}/ps.yaml"
     FileUtils.rm ps_file if File.exist?(ps_file)
     assert_equal(false, File.exist?(ps_file))
     @h00.update_ps
@@ -61,7 +45,7 @@ class TC_HostInspector < Test::Unit::TestCase
   end
 
   def test_update_cpuinfo
-    cpuinfo_file = "#{CACHE_DIR}/#{@h00.hostname}/cpuinfo.yaml"
+    cpuinfo_file = "#{@cache_dir}/#{@h00.hostname}/cpuinfo.yaml"
     FileUtils.rm cpuinfo_file if File.exist?(cpuinfo_file)
     assert_equal(false, File.exist?(cpuinfo_file))
     @h00.update_cpuinfo
@@ -73,7 +57,7 @@ class TC_HostInspector < Test::Unit::TestCase
   end
 
   def test_update_meminfo
-    meminfo_file = "#{CACHE_DIR}/#{@h00.hostname}/meminfo.yaml"
+    meminfo_file = "#{@cache_dir}/#{@h00.hostname}/meminfo.yaml"
     FileUtils.rm meminfo_file if File.exist?(meminfo_file)
     assert_equal(false, File.exist?(meminfo_file))
     @h00.update_meminfo
@@ -90,7 +74,7 @@ class TC_HostInspector < Test::Unit::TestCase
     @h01.update_ping
     assert_equal(false, @h01.fetch('ping'))
 
-    ping_file = "#{CACHE_DIR}/#{@h00.hostname}/ping.yaml"
+    ping_file = "#{@cache_dir}/#{@h00.hostname}/ping.yaml"
     FileUtils.rm ping_file if File.exist?(ping_file)
     assert_raise(Comana::HostInspector::NoUpdateFile) do
       @h00.fetch('ping')
@@ -101,10 +85,8 @@ class TC_HostInspector < Test::Unit::TestCase
 
   def test_time_ping
     time = Time.local(2015, 5, 29, 22, 00, 17)
-    ping_file = "#{CACHE_DIR}/#{@h00.hostname}/ping.yaml"
-    #pp File.mtime(ping_file)
+    ping_file = "#{@cache_dir}/#{@h00.hostname}/ping.yaml"
     File::utime(time, time, ping_file)
-    #pp File.mtime(ping_file)
     assert_equal(time, File.mtime(ping_file))
   end
 

data/test/test_hostselector.rb

@@ -24,33 +24,22 @@ class TC_HostSelector < Test::Unit::TestCase
   end
 
   def test_load_file
-    hs = Comana::HostSelector.load_file("example/dot.clustersetting")
+    hs = Comana::HostSelector.load_file("test/hostselector/dot.clustersetting")
     assert_equal(Comana::HostSelector, hs.class)
-    #pp hs
-    #assert_equal(["A00", "A01", "B00", "B01", "B02"], hs.select_all)
     assert_equal({"A"=>["A00", "A01"], "B"=>["B00", "B01", "B02"]}, hs.groups_hosts)
   end
 
   def test_select_all
-    #it 'should return all hosts' do
     assert_equal(["A00", "A01", "B00", "B01", "B02"], @hs00.select_all)
-
-    #it 'should return all hosts without nil' do
     assert_equal(["A00", "A01", "B00", "B01", "B02"], @hs01.select_all)
   end
 
   def test_select_group
-    #it 'should return hosts in GroupA' do
     assert_equal(["A00", "A01"], @hs00.select_group("GroupA"))
-
-    #it 'should raise Comana::HostSelector::NoEntryError' do
-    assert_raise(Comana::HostSelector::NoEntryError){
-      @hs00.select_group("GroupNil")
-    }
+    assert_raise(Comana::HostSelector::NoEntryError){ @hs00.select_group("GroupNil") }
   end
 
   def test_groups
-    #it 'should return all groups' do
     assert_equal(["GroupA", "GroupB"], @hs00.groups)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: comana
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - ippei94da
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-04 00:00:00.000000000 Z
+date: 2016-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: builtinextension
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: "Comana, COmputation MANAger,\n    is a software to provide a framework
   of\n    managing scientific computing.\n    Researchers on computing have to check
   calculation and\n    generate new calculation and execute, repeatedly.\n    The
@@ -102,8 +116,6 @@ email: ippei94da@gmail.com
 executables:
 - genqsub
 - hostinfo
-- machinestatus
-- queueinfo
 - scpall
 - sshall
 extensions: []
@@ -121,8 +133,6 @@ files:
 - VERSION
 - bin/genqsub
 - bin/hostinfo
-- bin/machinestatus
-- bin/queueinfo
 - bin/scpall
 - bin/sshall
 - comana.gemspec
@@ -130,14 +140,19 @@ files:
 - lib/comana.rb
 - lib/comana/clustersetting.rb
 - lib/comana/computationmanager.rb
-- lib/comana/gridenginescript.rb
+- lib/comana/gridengine.rb
 - lib/comana/hostinspector.rb
 - lib/comana/hostselector.rb
-- lib/comana/queuemanager.rb
-- lib/comana/queuesubmitter.rb
-- memo.txt
+- test/gridengine/qconfsql.dat
+- test/gridengine/qstatf0.xml
+- test/gridengine/qstatf1.xml
+- test/gridengine/qstatq0.xml
+- test/gridengine/qstatq1.xml
+- test/gridengine/qstatu0.xml
+- test/gridengine/qstatu1.xml
 - test/helper.rb
 - test/hostinspector/.gitignore
+- test/hostselector/dot.clustersetting
 - test/locked/input_a
 - test/locked/input_b
 - test/locked/lock_comana/dummy
@@ -160,11 +175,9 @@ files:
 - test/queuesubmitter/unlocked/dummy
 - test/test_clustersetting.rb
 - test/test_computationmanager.rb
-- test/test_gridenginescript.rb
+- test/test_gridengine.rb
 - test/test_hostinspector.rb
 - test/test_hostselector.rb
-- test/test_queuemanager.rb
-- test/test_queuesubmitter.rb
 homepage: http://github.com/ippei94da/comana
 licenses:
 - MIT
@@ -185,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Manager for scientific computing