comana 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8203f79238f204d50f1e79f7e29bea90f20f4613
-  data.tar.gz: 8b3fe89b37f61a5f8f5bf0df6a346d2a0614c072
+  metadata.gz: 26543f302d5fb0a59138a6542e84caef0b2491ce
+  data.tar.gz: 2f0061d8200ae6ac68a18c0616a82100c96e8cae
 SHA512:
-  metadata.gz: 291a9ab5f01fa4641a40d2c917f3a6d63861777dba484a1a065f8342b5f9d2416c66f0d87d273163d2b2a02f3040346f8333210a61813e7a605d1c765e549416
-  data.tar.gz: 0794a0a6e80a3556e3fb7b7cb4b49a46d78746c59597e6ea194a8fe852bfaa4588431383edacc5c2a5a3225fc87385af8874d585313e11b910993f8431b9219b
+  metadata.gz: 3a797bd56b438b3142b1836f76ff94d432164b9d5d5d3e3929490460299e5a102b9365a4c9a79b6eea3ed29b1f17ede8ebd125da99b81d404faf661c7c954253
+  data.tar.gz: c4029c86b02d2ea9774351f8988b7cd48896355ff4c7fa08c9a0268b0308cdf814ef4c7ea8a65c7a3e31962883f93148c66f69111b3e6073b1fed50be0eff124

data/CHANGES

@@ -1,27 +1,60 @@
 = comana changelog
 
-== Master (for 0.1.1)
+== Master (for 0.1.2)
+
+== Version 0.1.1 [2016-03-11] released
+- Comana::ComputationManager#start() rename to ...
+- Change argument specification of bin/genqsub
+- Delete GridEngineScript
+- Add GridEngine
+- Add ComputationManager::qsub
+- Delete bin/machinestatus. Use hostinfo
+- Delete QueueSubmitter
 
 == Version 0.1.0 [2016-03-04] released
-  - Comana::ComputationManager#start() rename to exec().
-  - Comana::ComputationManager.exec is added.
-  - Add Comana::HostInspector
-  - Add bin/hostinfo
-  - Add bin/genqsub
+- Comana::ComputationManager#start() rename to exec().
+- Comana::ComputationManager.exec is added.
+- Add Comana::HostInspector
+- Add bin/hostinfo
+- Add bin/genqsub
 
 == Version 0.0.10 [2014-08-28] released
-  - Provide namespace 'Comana' for classes.
-  - Add HostnameSelector
-  - Add HostnameSelector::Ping
-  - Add HostnameSelector::Pbsnodes
-  - MachineInfo class in machineinfo.rb is renamed to be
-    ClusterSetting class in clusterinfo.rb,
-    And change interface.
-  - Change Rspec to TestUnit
-  - Add bin/sshall
-  - Add bin/scpall
-  - Add priority option to opts in QueueSubmitter::initialize,
-    affecting PBS -p option in script.sh.
-  - ComputationManager.latest_modified_time is changed to public method.
+- Provide namespace 'Comana' for classes.
+- Add HostnameSelector
+- Add HostnameSelector::Ping
+- Add HostnameSelector::Pbsnodes
+- MachineInfo class in machineinfo.rb is renamed to be
+  ClusterSetting class in clusterinfo.rb,
+  And change interface.
+- Change Rspec to TestUnit
+- Add bin/sshall
+- Add bin/scpall
+- Add priority option to opts in QueueSubmitter::initialize,
+  affecting PBS -p option in script.sh.
+- ComputationManager.latest_modified_time is changed to public method.
 
 == Version 0.0.9 [2012-05-29] released
+  Change opts specification in QueueSubmitter; symbol to string
+
+== Version 0.0.8
+  Changed queuesubmitter option. (long name symbol)
+  Add prologue and epilogue to QueueSubmitter.
+
+== Version 0.0.7
+  Bugfix of danger to rsync large directory:
+    - bin/submitqueue is removed.
+    - lib/queuesubmitter.rb is modified.
+  lib/queuesubmitter.rb is modified.
+    - finished? returns bool.
+    - content of qsub script to use "rsync" and "mv dir ~/.trash".
+
+== Version 0.0.6
+  bin/submitqueue is add.
+  lib/queuesubmitter.rb is add.
+
+== Version 0.0.5
+  Directory structure and file names are changed.
+  Method names are hanged.
+  MachineInfo is added.
+
+== Version 0.0.4

data/Gemfile

@@ -12,4 +12,5 @@ group :development do
   gem "jeweler", "~> 2.0"
   gem "simplecov", "~> 0.11"
   gem "tefil", ">= 0.1"
+  gem "builtinextension", ">= 0.1"
 end

data/README.rdoc

@@ -1,6 +1,11 @@
 = comana
 
-Description goes here.
+Comana: COmputation MANAger
+
+Comana don't have information of output files.
+Because it is difficult to define final output and to deal systematically.
+e.g., repeated calculation till convergence.
+
 
 == Contributing to comana
  

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/bin/genqsub

@@ -5,41 +5,49 @@ require "pp"
 require "comana"
 require "optparse"
 
-## option analysis
-OPTIONS = {}
+options = {}
 op = OptionParser.new
-op.banner = [
-  "Usage: #{File.basename("#{__FILE__}")} [options] host_series command",
-].join("\n")
-op.on("-s"   , "--submit"    , "Submit at the same time."){OPTIONS[:submit] = true}
-op.on("-o file", "--outfile=file", "Output in file."){|v| OPTIONS[:outfile] = v}
+op.banner = <<HERE
+genqsub [options] [dirs]
+Submit queue to grid engine. Empty 'dirs' in argument indicates current directory.
+HERE
+op.on('--q-name=queue'         , "Queue name for '#$ -q'"){
+  |v| options[:q_name] = v}
+op.on('--pe-name=pe'     , "Parallel environment name for '\#$ -pe'"){
+  |v| options[:pe_name] = v}
+op.on('--ppn=ppn'            , "Process per node, 2nd argument for'#$ -pe'"){
+  |v| options[:ppn] = v}
+op.on('--ld_library_path=path', "Set 'LD_LIBRARY_PATH'"){
+  |v| options[:ld_library_path] = v}
+op.on('--load-group=group'     , "Load setting from group in setting file."){
+  |v| options[:load_group] = v}
+#op.on('-no_submit'      , "Write script', ' but not submit."){
+#|v| options[:no_submit] = v}
+op.on('--auto'           , "Select low load group and automatic setting."){ options[:auto] = true}
 op.parse!(ARGV)
 
-io = STDOUT
-io = File.open( OPTIONS[:outfile], "w") if OPTIONS[:outfile]
-
-series = ARGV.shift
-
-cluster_setting = Comana::ClusterSetting.load_file
-settings =  cluster_setting.groups[series]
+if options[:load_group]
+  settings = Comana::ClusterSetting.load_file
+  gs = settings.groups[options[:load_group]]
+  q_name          ||= gs['queue']
+  pe_name         ||= gs['pe']         
+  ppn             ||= gs['ppn']         
+  ld_library_path ||= gs['ld_library_path'] 
+end
 
-unless settings
-  puts "Not found '#{series}' in #{cluster_setting.data_file}"
-  puts op.banner
+if ARGV.empty?
+  puts "Need command in argument. Exit."
   exit
+else
+  command = ARGV.join(" ")
 end
-ppn = settings["ppn"]
 
-command = ARGV.join(' ')
+GridEngine.write_qsub_script(
+  q_name:          q_name          ,
+  pe_name:         pe_name         ,
+  ppn:             ppn             ,
+  command:         command         ,
+  ld_library_path: ld_library_path ,
+  io: $stdout
+)
 
-GridEngineScript.generate(io, series, ppn, command)
-io.close
-
-if OPTIONS[:submit]
-    unless OPTIONS[:outfile]
-        puts "'--submit' option must use with '--outfile' option."
-        exit
-    end
-
-    GridEngineScript.write_submit(series, ppn, command, OPTIONS[:outfile])
-end

data/bin/scpall

@@ -26,7 +26,7 @@ require "comana"
 OPTIONS = {}
 op = OptionParser.new
 op.on("-f from_file", "--from-file=str", "Copied file in a localhost."){|v| OPTIONS[:from_file] = v}
-op.on("-t from_file", "--to-file=str", "File path in remotehosts."){|v| OPTIONS[:to_file] = v}
+op.on("-t to_file", "--to-file=str", "File path in remotehosts."){|v| OPTIONS[:to_file] = v}
 op.on("-g", "--group", "Interpret arguments as group name."){OPTIONS[:group] = true}
 op.on("-u user", "--user=str"    , "User on remote host."){|v| OPTIONS[:user] = v}
 op.on("-T", "--test"    , "Test mode; not execute."){OPTIONS[:test] = true}

data/comana.gemspec

@@ -2,19 +2,19 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: comana 0.1.0 ruby lib
+# stub: comana 0.1.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "comana"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["ippei94da"]
-  s.date = "2016-03-04"
+  s.date = "2016-03-10"
   s.description = "Comana, COmputation MANAger,\n    is a software to provide a framework of\n    managing scientific computing.\n    Researchers on computing have to check calculation and\n    generate new calculation and execute, repeatedly.\n    The abstract class that this gem provide would help the work.\n  "
   s.email = "ippei94da@gmail.com"
-  s.executables = ["genqsub", "hostinfo", "machinestatus", "queueinfo", "scpall", "sshall"]
+  s.executables = ["genqsub", "hostinfo", "scpall", "sshall"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
@@ -30,8 +30,6 @@ Gem::Specification.new do |s|
     "VERSION",
     "bin/genqsub",
     "bin/hostinfo",
-    "bin/machinestatus",
-    "bin/queueinfo",
     "bin/scpall",
     "bin/sshall",
     "comana.gemspec",
@@ -39,14 +37,19 @@ Gem::Specification.new do |s|
     "lib/comana.rb",
     "lib/comana/clustersetting.rb",
     "lib/comana/computationmanager.rb",
-    "lib/comana/gridenginescript.rb",
+    "lib/comana/gridengine.rb",
     "lib/comana/hostinspector.rb",
     "lib/comana/hostselector.rb",
-    "lib/comana/queuemanager.rb",
-    "lib/comana/queuesubmitter.rb",
-    "memo.txt",
+    "test/gridengine/qconfsql.dat",
+    "test/gridengine/qstatf0.xml",
+    "test/gridengine/qstatf1.xml",
+    "test/gridengine/qstatq0.xml",
+    "test/gridengine/qstatq1.xml",
+    "test/gridengine/qstatu0.xml",
+    "test/gridengine/qstatu1.xml",
     "test/helper.rb",
     "test/hostinspector/.gitignore",
+    "test/hostselector/dot.clustersetting",
     "test/locked/input_a",
     "test/locked/input_b",
     "test/locked/lock_comana/dummy",
@@ -69,15 +72,13 @@ Gem::Specification.new do |s|
     "test/queuesubmitter/unlocked/dummy",
     "test/test_clustersetting.rb",
     "test/test_computationmanager.rb",
-    "test/test_gridenginescript.rb",
+    "test/test_gridengine.rb",
     "test/test_hostinspector.rb",
-    "test/test_hostselector.rb",
-    "test/test_queuemanager.rb",
-    "test/test_queuesubmitter.rb"
+    "test/test_hostselector.rb"
   ]
   s.homepage = "http://github.com/ippei94da/comana"
   s.licenses = ["MIT"]
-  s.rubygems_version = "2.2.3"
+  s.rubygems_version = "2.5.1"
   s.summary = "Manager for scientific computing"
 
   if s.respond_to? :specification_version then
@@ -90,6 +91,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<jeweler>, ["~> 2.0"])
       s.add_development_dependency(%q<simplecov>, ["~> 0.11"])
       s.add_development_dependency(%q<tefil>, [">= 0.1"])
+      s.add_development_dependency(%q<builtinextension>, [">= 0.1"])
     else
       s.add_dependency(%q<test-unit>, ["~> 3.1"])
       s.add_dependency(%q<rdoc>, ["~> 4.2"])
@@ -97,6 +99,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<jeweler>, ["~> 2.0"])
       s.add_dependency(%q<simplecov>, ["~> 0.11"])
       s.add_dependency(%q<tefil>, [">= 0.1"])
+      s.add_dependency(%q<builtinextension>, [">= 0.1"])
     end
   else
     s.add_dependency(%q<test-unit>, ["~> 3.1"])
@@ -105,6 +108,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<jeweler>, ["~> 2.0"])
     s.add_dependency(%q<simplecov>, ["~> 0.11"])
     s.add_dependency(%q<tefil>, [">= 0.1"])
+    s.add_dependency(%q<builtinextension>, [">= 0.1"])
   end
 end
 

data/example/dot.clustersetting

@@ -4,12 +4,27 @@
 pbs_server: "P00"
 
 groups:
-  A:
-    data1:  'A-1'
-    data2:  'A-2'
-    members: ["A00", "A01"]
-
-  B:
-    data1:  'B-1'
-    data2:  'B-2'
-    members: ["B00", "B01", "B02"]
+  Ag:
+    mpi: '/opt/openmpi-intel/bin/mpiexec'
+    vasp: '/opt/bin/vasp5212openmpi'
+    ppn: 4
+    members: [
+      'Ag00','Ag01','Ag02','Ag03','Ag04','Ag05','Ag06','Ag07'
+      ]
+    ld_library_path: '/usr/lib:/usr/local/lib:/opt/intel/mkl/lib/intel64:/opt/intel/lib/intel64:/opt/intel/lib:/opt/openmpi-intel/lib'
+    queue: 'Ag.q'
+    pe: 'Ag.openmpi'
+    benchmark: 11.6302
+
+  Cd:
+    mpi: '/opt/openmpi-intel/bin/mpiexec'
+    vasp: '/opt/bin/vasp5212openmpi'
+    ppn: 4
+    members: [
+      'Cd00','Cd01','Cd02','Cd03','Cd04','Cd05','Cd06','Cd07'
+      ]
+    ld_library_path: '/usr/lib:/usr/local/lib:/opt/intel/mkl/lib/intel64:/opt/intel/lib/intel64:/opt/intel/lib:/opt/openmpi-intel/lib'
+    queue: 'Cd.q'
+    pe: 'Cd.openmpi'
+    benchmark: 12.0019
+

data/lib/comana/clustersetting.rb

@@ -41,6 +41,16 @@ class Comana::ClusterSetting
     @groups[clustername]
   end
 
+  #Return settings as a hash for a cluster, the 'queue' key has a value of q_name.
+  def settings_queue(q_name)
+    result = nil
+    #pp @groups
+    @groups.each do |name, items|
+      result = items if items['queue'] == q_name
+    end
+    result
+  end
+
   #Return settings as a hash for a host belonged to cluster.
   def settings_host(hostname)
     settings_group(belonged_cluster(hostname))

data/lib/comana/computationmanager.rb

@@ -5,9 +5,14 @@
 # Users have to redefine some methods in subclasses for various computation.
 # 
 class Comana::ComputationManager
-  class NotImplementedError < Exception; end
-  class AlreadyStartedError < Exception; end
-  class ExecuteError < Exception; end
+  class InitializeError < StandardError; end
+  class NotImplementedError < StandardError; end
+  class AlreadyStartedError < StandardError; end
+  class AlreadySubmittedError < StandardError; end
+  class ExecuteError < StandardError; end
+
+  QSUB_SCRIPT_NAME = 'qsub.sh'
+  QSUB_LOG_NAME    = 'qsub.log'
 
   attr_reader :dir
 
@@ -18,28 +23,121 @@ class Comana::ComputationManager
     @alive_time = 3600
   end
 
-  def self.exec(args)
+  def self.execute(args)
     targets = args
-    targets = [ENV['PWD']] if targets.size == 0
+    targets = [ENV['PWD']] if targets.empty?
 
     targets.each do |dir|
       print "#{dir}..."
       begin
         calc_dir = self.new(dir)
       rescue => exc
-        puts "Not suitable directory, due of an exception: #{exc}"
+        puts "Not #{self}: #{dir}"
         next
       end
 
       begin
         calc_dir.start
-      rescue Comana::ComputationManager::AlreadyStartedError
-        puts "Already started."
+      rescue self::AlreadyStartedError
+        puts "Already started: #{dir}"
         next
       end
     end
   end
 
+  def self.qsub(args, options)
+    if options[:auto] || options[:load_group]
+      # OK
+    elsif !(options[:q_name] && options[:pe_name] && options[:ppn] && options[:command])
+      puts "Lack of required options."
+      puts "Need (--auto) or (--load-group) or (--q-name && --pe-name && --ppn && --command )"
+      puts "E.g., OK: #{File.basename($0)} --auto"
+      puts "      OK: #{File.basename($0)} --load-group=cluster_name"
+      puts "      OK: #{File.basename($0)} --q-name=a --pe-name=b --ppn=1 --command=c"
+      puts "      NG: #{File.basename($0)} --q-name=a --pe-name=b --ppn=1"
+      puts "Exit."
+      exit
+    end
+
+    tgts = args
+    tgts = [ENV['PWD']] if tgts.empty?
+
+    tgts.each do |dir|
+      cs = Comana::ClusterSetting.load_file
+      if options[:load_group]
+        q_name = options[:load_group]
+      elsif options[:auto]
+        queues = Comana::GridEngine.queues
+        jobs = {}
+        hosts = {}
+        benchmarks = {}
+        queues.each do |q|
+          jobs[q] = Comana::GridEngine.queue_jobs(q).size
+          hosts[q] = Comana::GridEngine.queue_alive_nums[q]
+          benchmarks[q] = Comana::ClusterSetting.load_file.settings_queue(q)['benchmark']
+        end
+        q_name =  self.effective_queue(queues, jobs, hosts, benchmarks)
+      end
+
+      if options[:load_group] || options[:auto]
+        gs = cs.settings_queue(q_name)
+        q_name          ||= gs['queue']
+        pe_name         ||= gs['pe']
+        ppn             ||= gs['ppn']
+        ld_library_path ||= gs['ld_library_path']
+      end
+
+      q_name          = options[:q_name]          if options[:q_name]
+      pe_name         = options[:pe_name]         if options[:pe_name]
+      ppn             = options[:ppn]             if options[:ppn]
+      ld_library_path = options[:ld_library_path] if options[:ld_library_path]
+      command = options[:command] || "#{`which #{__FILE__}`.chomp} execute"
+
+      begin
+        calc_dir = self.new(dir)
+        calc_dir.queue_submit(
+          q_name:           q_name,
+          pe_name:          pe_name,
+          ppn:              ppn,
+          ld_library_path:  ld_library_path,
+          command:          command
+        )
+      rescue self::InitializeError
+        puts "Not #{self} : #{dir}"
+      rescue Comana::ComputationManager::AlreadySubmittedError
+        puts "Already started: #{dir}"
+      end
+    end
+  end
+
+  ## jobs < hosts のキューがあれば(空きホストがあれば)、その中で bench が最小のもの
+  ## なければ、self.guess_end_time の値が最小のもの。
+  def self.effective_queue(queues, jobs, hosts, benchmarks)
+    candidates = queues.select do |q|
+      jobs[q] < hosts[q] 
+    end
+    if candidates.empty?
+      result = queues.min_by{|q| self.guess_end_time(jobs[q], hosts[q], benchmarks[q]) }
+    else
+      result = candidates.min_by {|q| benchmarks[q]}
+    end
+    result
+  end
+
+  # 新しく1個ジョブを追加した場合の終了見込み時間
+  # ジョブの終了時刻がランダムであり、ジョブの実行時間は等しいと仮定して算出。
+  # 空きホストがあれば benchmark 通りの見込み。
+  # 空きホストがなければ、ホストがあくまでの見込み時間を加算。
+  def self.guess_end_time(num_jobs , num_hosts, benchmark)
+    if num_jobs < num_hosts
+      unit = 1.0 #unit 
+    else
+      unit = (num_jobs.to_f + 1.0) / (num_hosts.to_f)
+    end
+    result = unit * benchmark
+    result
+  end
+
   # Return a symbol which indicate state of calculation.
   #   :yet           not started
   #   :started       started, but not ended, including short time from last output
@@ -55,7 +153,8 @@ class Comana::ComputationManager
   # Execute calculation.
   # If log of ComputationManager exist, raise ComputationManager::AlreadyStartedError,
   # because the calculation has been done by other process already.
-  def exec
+  # This method is aliased to 'start'.
+  def execute
     begin
       Dir.mkdir "#{@dir}/#{@lockdir}"
     rescue Errno::EEXIST
@@ -68,7 +167,7 @@ class Comana::ComputationManager
       prepare_next
     end
   end
-  alias start exec
+  alias start execute
 
   # Return latest modified time of files in calc dir recursively.
   # require "find"
@@ -79,6 +178,31 @@ class Comana::ComputationManager
     File.mtime(tmp)
   end
 
+  def queue_submit(q_name:, pe_name:, ppn:, ld_library_path: , command:, submit: true)
+
+    qsub_path = "#{@dir}/#{QSUB_SCRIPT_NAME}"
+    if FileTest.exist? qsub_path
+      raise AlreadySubmittedError,
+        "Already exist #{qsub_path}."
+    end
+    File.open(qsub_path, "w") do |io|
+      Comana::GridEngine.write_qsub_script(
+        q_name:          q_name,
+        pe_name:         pe_name,
+        ppn:             ppn,
+        ld_library_path: ld_library_path,
+        command:         command,
+        io:              io
+      )
+    end
+    cur_dir = Dir.pwd
+    Dir.chdir @dir
+    print   "Submitting #{qsub_path}..."
+    system("qsub #{QSUB_SCRIPT_NAME} > #{QSUB_LOG_NAME}") if submit
+    puts   "Done."
+    Dir.chdir cur_dir
+  end
+
   private
 
   # Redefine in subclass, e.g.,