comana 0.0.10 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8203f79238f204d50f1e79f7e29bea90f20f4613
+  data.tar.gz: 8b3fe89b37f61a5f8f5bf0df6a346d2a0614c072
+SHA512:
+  metadata.gz: 291a9ab5f01fa4641a40d2c917f3a6d63861777dba484a1a065f8342b5f9d2416c66f0d87d273163d2b2a02f3040346f8333210a61813e7a605d1c765e549416
+  data.tar.gz: 0794a0a6e80a3556e3fb7b7cb4b49a46d78746c59597e6ea194a8fe852bfaa4588431383edacc5c2a5a3225fc87385af8874d585313e11b910993f8431b9219b

data/CHANGES

@@ -1,6 +1,13 @@
 = comana changelog
 
-== Master (for 0.0.11)
+== Master (for 0.1.1)
+
+== Version 0.1.0 [2016-03-04] released
+  - Comana::ComputationManager#start() rename to exec().
+  - Comana::ComputationManager.exec is added.
+  - Add Comana::HostInspector
+  - Add bin/hostinfo
+  - Add bin/genqsub
 
 == Version 0.0.10 [2014-08-28] released
   - Provide namespace 'Comana' for classes.

data/Gemfile

@@ -4,17 +4,12 @@ source "http://rubygems.org"
 #   gem "activesupport", ">= 2.3.5"
 
 # Add dependencies to develop your gem here.
-# Include everything needed to run rake, tests, features, etc.
+# Include everything needed to exec rake, tests, features, etc.
 group :development do
-  #gem "rspec", ">= 2.10.0"
-  gem "rdoc", "~> 4.0.1"
-  gem "bundler", "~> 1.7.2"
-  gem "jeweler", ">= 2.0.1"
-  gem "simplecov", ">= 0"
-  #gem "builder", "~> 3.2.2"
-  #gem "nokogiri", ">= 1.5.9"
-  #
-  gem "tefil", ">= 0.0.3"
-  #gem "psych", ">= 1.3.4"
-
+  gem "test-unit", "~> 3.1"
+  gem "rdoc", "~> 4.2"
+  gem "bundler", "~> 1.11"
+  gem "jeweler", "~> 2.0"
+  gem "simplecov", "~> 0.11"
+  gem "tefil", ">= 0.1"
 end

data/VERSION

@@ -1 +1 @@
-0.0.10
+0.1.0

data/bin/genqsub

@@ -0,0 +1,45 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+require "pp"
+require "comana"
+require "optparse"
+
+## option analysis
+OPTIONS = {}
+op = OptionParser.new
+op.banner = [
+  "Usage: #{File.basename("#{__FILE__}")} [options] host_series command",
+].join("\n")
+op.on("-s"   , "--submit"    , "Submit at the same time."){OPTIONS[:submit] = true}
+op.on("-o file", "--outfile=file", "Output in file."){|v| OPTIONS[:outfile] = v}
+op.parse!(ARGV)
+
+io = STDOUT
+io = File.open( OPTIONS[:outfile], "w") if OPTIONS[:outfile]
+
+series = ARGV.shift
+
+cluster_setting = Comana::ClusterSetting.load_file
+settings =  cluster_setting.groups[series]
+
+unless settings
+  puts "Not found '#{series}' in #{cluster_setting.data_file}"
+  puts op.banner
+  exit
+end
+ppn = settings["ppn"]
+
+command = ARGV.join(' ')
+
+GridEngineScript.generate(io, series, ppn, command)
+io.close
+
+if OPTIONS[:submit]
+    unless OPTIONS[:outfile]
+        puts "'--submit' option must use with '--outfile' option."
+        exit
+    end
+
+    GridEngineScript.write_submit(series, ppn, command, OPTIONS[:outfile])
+end

data/bin/hostinfo

@@ -0,0 +1,182 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+require "comana"
+require "optparse"
+require "pp"
+
+##Analyze options
+COMMAND_NAME = File.basename("#{__FILE__}")
+OPTIONS = {}
+op = OptionParser.new
+op.banner = <<HERE
+Usage: #{COMMAND_NAME} subcommand [options] [hosts]
+Subcommands:
+  ping    # Ping: o and x indicate OK and N/A, respectively.
+  alive   # Ping only alive: show alive hostname
+  cwd     # cwd of processes owned by ssh login user.
+  ps      # Info from ps.
+  cpuinfo # CPU information from /proc/cpuinfo.
+  meminfo # memory information from /proc/meminfo.
+  load    # the highest process
+For example,
+  #{COMMAND_NAME} ping                # for all hosts 
+  #{COMMAND_NAME} ping host01 host02
+  #{COMMAND_NAME} ping -g host_series
+  #{COMMAND_NAME} ping -u             # update cached data
+HERE
+
+op.on("-u", "--update" , "Update cache"){OPTIONS[:update] = true}
+op.on("-g", "--group"  , "Indicate group of hosts"    ){OPTIONS[:group] = true}
+op.on("-q", "--quiet"  , "No output 'Wainting for...'"){OPTIONS[:quiet] = true}
+op.parse!(ARGV)
+
+## Analyze subcommand
+subcommand = ARGV.shift
+requirement = {}
+case subcommand
+when "ping"
+  requirement[:ping] = true
+when "alive"
+  requirement[:ping] = true
+when "cwd"
+  requirement[:cwd] = true
+when "ps"
+  requirement[:ps] = true
+when "cpuinfo"
+  requirement[:cpuinfo] = true
+when "meminfo"
+  requirement[:meminfo] = true
+when "load"
+  requirement[:ps] = true
+  requirement[:cwd] = true
+else
+  puts op.help
+  exit
+end
+
+## Return selected hosts
+def target_hosts
+  cs = Comana::ClusterSetting.load_file
+  groups = {}
+  cs.groups.each do |key, val|
+    cluster = key
+    groups[key] = val["members"]
+  end
+  hs = Comana::HostSelector.new groups
+  if OPTIONS[:group]
+    hosts = []
+    ARGV.each do |group|
+      begin
+        hosts << hs.select_group(group)
+      rescue Comana::HostSelector::NoEntryError
+        $stderr.puts "Unknown group: #{group}"
+      end
+    end
+    hosts.flatten!
+  else
+    hosts = ARGV
+  end
+  hosts = hs.select_all if ARGV.empty?
+  hosts
+end
+
+def time_ago(uptime)
+  if uptime
+    second = (Time.now - uptime).to_i
+    if second >= 86400
+      result = sprintf("%2s days ago", second / 86400)
+    elsif second >= 3600
+      result = sprintf("%2s hours ago", second / 3600)
+    elsif second >= 60
+      result = sprintf("%2s min  ago", second / 60)
+    else
+      result = sprintf("%2s sec  ago", second)
+    end
+  else #e.g., not exist cache file
+    result = "need_update"
+  end
+  return result
+end
+
+## Collect information
+hostinspectors = target_hosts.map { |host| Comana::HostInspector.new(host) }
+if OPTIONS[:update]
+  Thread.abort_on_exception = true
+  threads = {}
+  hostinspectors.each do |hi|
+    threads[hi] = Thread.start do
+      hi.update_ping if requirement[:ping]
+      hi.update_cwd  if requirement[:cwd]
+      hi.update_ps   if requirement[:ps]
+      hi.update_cpuinfo   if requirement[:cpuinfo]
+      hi.update_meminfo   if requirement[:meminfo]
+    end
+  end
+  threads.each do |hi, thread|
+    print "Waiting for #{hi.hostname}...\n" unless OPTIONS[:quiet]
+    thread.join              # wait until all processes are completed
+  end
+end
+
+## Output
+hostinspectors.each do |hi|
+  # oldest update
+  oldest_key, val = requirement.min_by {|key,val| hi.time_updated(key.to_s)}
+  hostname = hi.hostname
+  timeago  = time_ago( hi.time_updated(oldest_key.to_s))
+  hosttime = sprintf("%-10s (%12s) ",
+         hi.hostname, time_ago( hi.time_updated(oldest_key.to_s)))
+  begin
+    case subcommand
+    when "ping"
+      if hi.fetch('ping')
+        hosttime += "o"
+      else
+        hosttime += "x"
+      end
+      puts hosttime
+    when "alive"
+      puts hostname if hi.fetch('ping')
+    when "cwd"
+      puts hosttime
+      hi.fetch('cwd').each do |pid, cwd|
+        printf("  %5s %s\n", pid, cwd)
+      end
+    when "ps"
+      puts hosttime
+      hi.fetch('ps').each do |pid, ps|
+        printf("  %-10s %5s\n", hi.hostname, pid)
+      end
+    when "cpuinfo"
+      result = hosttime
+      begin
+        cpuinfo = hi.fetch('cpuinfo')
+        result += sprintf("%2d_cores  %s\n",
+               cpuinfo.size, cpuinfo[0]["model name"].gsub(/ +/, '_'))
+      rescue
+        result += sprintf("not_obtained\n")
+      end
+      puts result
+    when "meminfo"
+      result = hosttime
+      meminfo = hi.fetch('meminfo')
+      result += sprintf("%s\n", meminfo['MemTotal'])
+      puts result
+    when "load"
+      result = hosttime
+      ps      = hi.fetch("ps")
+      cwd     = hi.fetch("cwd")
+      pid, process = ps.max_by {|key, val| val['cpu'].to_f}
+      command = File.basename(process['command'].split[0])
+      result += sprintf("%5s %8s %5.1f %5.1f %20s %s\n",
+             pid, process['user'], process['cpu'], process['mem'],
+            command, cwd[pid])
+      puts result
+    end
+  rescue Comana::HostInspector::NoUpdateFile
+    puts hosttime + "------------"
+  rescue NoMethodError # 主にデータを取れなかった時用
+    puts hosttime + "------------"
+  end
+end

data/bin/machinestatus

@@ -1,6 +1,10 @@
 #! /usr/bin/env ruby
 # coding: utf-8
 
+
+puts "machinestatus is obsoleted. use 'hostinfo'"
+exit
+
 USAGE = <<HERE
 machinestatus HostA HostB
 machinestatus -g GroupA GroupB #-g option interprets as group name
@@ -57,8 +61,6 @@ else
 end
 hosts = hs.select_all if ARGV.empty?
 
-#pp hosts;exit
-
 ##Collect information
 Thread.abort_on_exception = true
 results = {}
@@ -186,4 +188,5 @@ end
 
 data = data.unshift(titles)
 #pp data
-Tefil::ColumnFormer.form(data, $stdout, "|")
+t = Tefil::ColumnFormer.new
+t.form(data, $stdout, "|")

data/bin/queueinfo

@@ -0,0 +1,28 @@
+#! /usr/bin/env ruby
+
+
+require 'thor'
+require 'comana'
+require 'pp'
+
+### Command template
+class Queueinfo < Thor
+  desc "light", "Detect the lightest queue."
+
+  def lightqueue
+    qinfo = Comana::QueueManager.new
+    qinfo.light_queue
+  end
+
+  #def job
+  #end
+
+  def show
+
+  end
+
+end
+
+Queueinfo.start(ARGV)
+
+

data/comana.gemspec

@@ -0,0 +1,110 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: comana 0.1.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "comana"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["ippei94da"]
+  s.date = "2016-03-04"
+  s.description = "Comana, COmputation MANAger,\n    is a software to provide a framework of\n    managing scientific computing.\n    Researchers on computing have to check calculation and\n    generate new calculation and execute, repeatedly.\n    The abstract class that this gem provide would help the work.\n  "
+  s.email = "ippei94da@gmail.com"
+  s.executables = ["genqsub", "hostinfo", "machinestatus", "queueinfo", "scpall", "sshall"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "CHANGES",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/genqsub",
+    "bin/hostinfo",
+    "bin/machinestatus",
+    "bin/queueinfo",
+    "bin/scpall",
+    "bin/sshall",
+    "comana.gemspec",
+    "example/dot.clustersetting",
+    "lib/comana.rb",
+    "lib/comana/clustersetting.rb",
+    "lib/comana/computationmanager.rb",
+    "lib/comana/gridenginescript.rb",
+    "lib/comana/hostinspector.rb",
+    "lib/comana/hostselector.rb",
+    "lib/comana/queuemanager.rb",
+    "lib/comana/queuesubmitter.rb",
+    "memo.txt",
+    "test/helper.rb",
+    "test/hostinspector/.gitignore",
+    "test/locked/input_a",
+    "test/locked/input_b",
+    "test/locked/lock_comana/dummy",
+    "test/locked_outputted/input_a",
+    "test/locked_outputted/input_b",
+    "test/locked_outputted/lock_comana/dummy",
+    "test/locked_outputted/output",
+    "test/not_executable/dummy",
+    "test/not_started/input_a",
+    "test/not_started/input_b",
+    "test/outputted/input_a",
+    "test/outputted/input_b",
+    "test/outputted/output",
+    "test/pbsnodes/Br09.xml",
+    "test/pbsnodes/Br10.xml",
+    "test/pbsnodes/Ge00.xml",
+    "test/pbsnodes/Ge08.xml",
+    "test/pbsnodes/all.xml",
+    "test/queuesubmitter/locked/lock_queuesubmitter/dummy",
+    "test/queuesubmitter/unlocked/dummy",
+    "test/test_clustersetting.rb",
+    "test/test_computationmanager.rb",
+    "test/test_gridenginescript.rb",
+    "test/test_hostinspector.rb",
+    "test/test_hostselector.rb",
+    "test/test_queuemanager.rb",
+    "test/test_queuesubmitter.rb"
+  ]
+  s.homepage = "http://github.com/ippei94da/comana"
+  s.licenses = ["MIT"]
+  s.rubygems_version = "2.2.3"
+  s.summary = "Manager for scientific computing"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<test-unit>, ["~> 3.1"])
+      s.add_development_dependency(%q<rdoc>, ["~> 4.2"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.11"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0"])
+      s.add_development_dependency(%q<simplecov>, ["~> 0.11"])
+      s.add_development_dependency(%q<tefil>, [">= 0.1"])
+    else
+      s.add_dependency(%q<test-unit>, ["~> 3.1"])
+      s.add_dependency(%q<rdoc>, ["~> 4.2"])
+      s.add_dependency(%q<bundler>, ["~> 1.11"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0"])
+      s.add_dependency(%q<simplecov>, ["~> 0.11"])
+      s.add_dependency(%q<tefil>, [">= 0.1"])
+    end
+  else
+    s.add_dependency(%q<test-unit>, ["~> 3.1"])
+    s.add_dependency(%q<rdoc>, ["~> 4.2"])
+    s.add_dependency(%q<bundler>, ["~> 1.11"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0"])
+    s.add_dependency(%q<simplecov>, ["~> 0.11"])
+    s.add_dependency(%q<tefil>, [">= 0.1"])
+  end
+end
+

data/lib/comana/clustersetting.rb

@@ -8,21 +8,22 @@ require "yaml"
 # E.g.,
 #   "Fe", "Fe00", "Fe01" are of series "Fe" and not "F"
 class Comana::ClusterSetting
-  attr_reader :groups, :pbs_server
+  attr_reader :data_file, :groups, :pbs_server
 
   DEFAULT_DATA_FILE = ENV["HOME"] + "/.clustersetting"
 
   class NoEntryError < Exception; end
 
   #
-  def initialize(settings)
+  def initialize(settings, data_file = nil)
     @pbs_server = settings["pbs_server"]
     @groups = settings["groups"]
+    @data_file = data_file
   end
 
   def self.load_file(data_file = DEFAULT_DATA_FILE)
     settings = YAML.load_file(data_file)
-    self.new settings
+    self.new(settings, data_file)
   end
 
   #Return belonged cluster of the host.

data/lib/comana/computationmanager.rb

@@ -1,7 +1,7 @@
 #! /usr/bin/env ruby
 # coding: utf-8
 
-# This class profides a framework of scientific computation.
+# This class provides a framework of scientific computation.
 # Users have to redefine some methods in subclasses for various computation.
 # 
 class Comana::ComputationManager
@@ -18,6 +18,28 @@ class Comana::ComputationManager
     @alive_time = 3600
   end
 
+  def self.exec(args)
+    targets = args
+    targets = [ENV['PWD']] if targets.size == 0
+
+    targets.each do |dir|
+      print "#{dir}..."
+      begin
+        calc_dir = self.new(dir)
+      rescue => exc
+        puts "Not suitable directory, due of an exception: #{exc}"
+        next
+      end
+
+      begin
+        calc_dir.start
+      rescue Comana::ComputationManager::AlreadyStartedError
+        puts "Already started."
+        next
+      end
+    end
+  end
+
   # Return a symbol which indicate state of calculation.
   #   :yet           not started
   #   :started       started, but not ended, including short time from last output
@@ -33,7 +55,7 @@ class Comana::ComputationManager
   # Execute calculation.
   # If log of ComputationManager exist, raise ComputationManager::AlreadyStartedError,
   # because the calculation has been done by other process already.
-  def start
+  def exec
     begin
       Dir.mkdir "#{@dir}/#{@lockdir}"
     rescue Errno::EEXIST
@@ -46,6 +68,7 @@ class Comana::ComputationManager
       prepare_next
     end
   end
+  alias start exec
 
   # Return latest modified time of files in calc dir recursively.
   # require "find"

data/lib/comana/gridenginescript.rb

@@ -0,0 +1,68 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+#
+#
+#
+class GridEngineScript
+  #DEFAULT_QSUB_FILE = 'qsub.sh'
+  QSUB_PATH = '/usr/bin/qsub'
+
+  #
+  def initialize()
+  end
+
+  # series: name of computers' series
+  # ppn:    number of using cores
+  def self.generate(io, series, ppn, command)
+    io.puts self.string(series, ppn, command)
+  end
+
+  def self.string(series, ppn, command)
+    string = <<HERE
+\#! /bin/sh
+\#$ -S /bin/sh
+\#$ -cwd
+\#$ -o stdout
+\#$ -e stderr
+\#$ -q #{series}.q
+\#$ -pe #{series}.openmpi #{ppn}
+
+MACHINE_FILE="machines"
+ENV_FILE="printenv.log"
+
+LD_LIBRARY_PATH=/usr/lib:/usr/local/lib:/opt/intel/mkl/lib/intel64:/opt/intel/lib/intel64:/opt/intel/lib:/opt/openmpi-intel/lib
+export LD_LIBRARY_PATH
+
+cd $SGE_O_WORKDIR
+if [ -e $ENV_FILE ]; then
+    echo "$ENV_FILE already exist. Exit."
+    exit
+fi
+printenv | sort > printenv.log
+cut -d " " -f 1,2 $PE_HOSTFILE | sed 's/ / cpu=/' > $MACHINE_FILE
+
+#{command}
+HERE
+    string
+  end
+
+  def self.write(series, ppn, command, io)
+    io.puts self.string(series, ppn, command)
+  end
+
+  def self.write_submit(series, ppn, command, filename)
+    #qsub_file = DEFAULT_QSUB_FILE
+    io = File.open(filename, 'w')
+    self.write(series, ppn, command, io)
+    io.close
+
+    logfile  = filename.sub(/#{File.extname filename}$/, '.log')
+    command = "#{QSUB_PATH} #{filename} > #{logfile}"
+    #command = "qsub #{filename}"
+    puts command
+    system command
+  end
+
+end
+