comana 0.0.9 → 0.0.10

data/CHANGES

@@ -0,0 +1,20 @@
+= comana changelog
+
+== Master (for 0.0.11)
+
+== Version 0.0.10 [2014-08-28] released
+  - Provide namespace 'Comana' for classes.
+  - Add HostnameSelector
+  - Add HostnameSelector::Ping
+  - Add HostnameSelector::Pbsnodes
+  - MachineInfo class in machineinfo.rb is renamed to be
+    ClusterSetting class in clusterinfo.rb,
+    And change interface.
+  - Change Rspec to TestUnit
+  - Add bin/sshall
+  - Add bin/scpall
+  - Add priority option to opts in QueueSubmitter::initialize,
+    affecting PBS -p option in script.sh.
+  - ComputationManager.latest_modified_time is changed to public method.
+
+== Version 0.0.9 [2012-05-29] released

data/Gemfile

@@ -6,9 +6,15 @@ source "http://rubygems.org"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem "rspec", ">= 2.10.0"
-  gem "rdoc", ">= 3.12"
-  gem "bundler", ">= 1.1.3"
-  gem "jeweler", ">= 1.8.3"
+  #gem "rspec", ">= 2.10.0"
+  gem "rdoc", "~> 4.0.1"
+  gem "bundler", "~> 1.7.2"
+  gem "jeweler", ">= 2.0.1"
   gem "simplecov", ">= 0"
+  #gem "builder", "~> 3.2.2"
+  #gem "nokogiri", ">= 1.5.9"
+  #
+  gem "tefil", ">= 0.0.3"
+  #gem "psych", ">= 1.3.4"
+
 end

data/Rakefile

@@ -31,18 +31,35 @@ Jeweler::Tasks.new do |gem|
 end
 Jeweler::RubygemsDotOrgTasks.new
 
-require 'rspec/core'
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
-end
+#require 'rspec/core'
+#require 'rspec/core/rake_task'
+#RSpec::Core::RakeTask.new(:spec) do |spec|
+#  spec.pattern = FileList['spec/**/*_spec.rb']
+#end
+#
+#RSpec::Core::RakeTask.new(:rcov) do |spec|
+#  spec.pattern = 'spec/**/*_spec.rb'
+#  spec.rcov = true
+#end
+#
+#task :default => :spec
 
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
 end
 
-task :default => :spec
+#require 'rcov/rcovtask'
+#Rcov::RcovTask.new do |test|
+#  test.libs << 'test'
+#  test.pattern = 'test/**/test_*.rb'
+#  test.verbose = true
+#  test.rcov_opts << '--exclude "gems/*"'
+#end
+
+task :default => :test
 
 require 'rdoc/task'
 Rake::RDocTask.new do |rdoc|

data/VERSION

@@ -1 +1 @@
-0.0.9
+0.0.10

data/bin/machinestatus

@@ -0,0 +1,189 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+USAGE = <<HERE
+machinestatus HostA HostB
+machinestatus -g GroupA GroupB #-g option interprets as group name
+HERE
+
+require "pp"
+require "optparse"
+require "rubygems"
+require "comana"
+require "highline"
+require "tefil"
+
+HIGHLINE = HighLine.new
+
+##Analyze options
+OPTIONS = {}
+op = OptionParser.new
+op.on("-a", "--ping-alive", "Check only alive using ping"){OPTIONS[:ping] = true}
+#op.on("-t", "--top"       , "Info from top.     "){OPTIONS[:top     ] = true}
+#op.on("-p", "--ps"        , "Info from ps.      "){OPTIONS[:ps      ] = true}
+op.on("-n", "--pbsnodes"  , "Info from pbsnodes."){OPTIONS[:pbsnodes] = true}
+op.on("-l", "--load"      , "Pbsnodes; state, loadave."){
+  OPTIONS[:pbsnodes] = true
+  OPTIONS[:load] = true
+}
+#op.on("-d", "--dmesg"     , "Info from dmesg.   "){OPTIONS[:dmesg   ] = true}
+
+op.on("-g", "--group"  , "Indicate group of hosts"    ){OPTIONS[:group] = true}
+op.on("-q", "--quiet"  , "No output 'Wainting for...'"){OPTIONS[:quiet] = true}
+op.parse!(ARGV)
+
+
+##Select hosts
+cs = Comana::ClusterSetting.load_file
+groups = {}
+cs.groups.each do |key, val|
+  cluster = key
+  groups[key] = val["members"]
+end
+
+hs = Comana::HostSelector.new groups
+if OPTIONS[:group]
+  hosts = []
+  ARGV.each do |group|
+    begin
+      hosts << hs.select_group(group)
+    rescue Comana::HostSelector::NoEntryError
+      $stderr.puts "Unknown group: #{group}"
+    end
+  end
+  hosts.flatten!
+else
+  hosts = ARGV
+end
+hosts = hs.select_all if ARGV.empty?
+
+#pp hosts;exit
+
+##Collect information
+Thread.abort_on_exception = true
+results = {}
+threads = {}
+hosts.each do |host|
+  threads[host] = Thread.start do
+    results[host] = {}
+    if OPTIONS[:ping]
+      results[host][:ping] = Comana::HostInspector::Ping.new host
+    end
+    if OPTIONS[:pbsnodes]
+      begin
+        results[host][:pbsnodes] = Comana::HostInspector::Pbsnodes.new(host, cs.pbs_server)
+      rescue Comana::HostInspector::Pbsnodes::UnknownNodeError
+        #results[host][:pbsnodes] = Comana::HostInspector::Pbsnodes.new(host, cs.pbs_server)
+        results[host][:pbsnodes] = nil
+        #results[host][:pbsnodes]["status"] = {}
+      end
+    end
+  end
+end
+
+threads.each do |host, thread|
+  print "Waiting for #{host}...\n" unless OPTIONS[:quiet]
+  thread.join              # wait until all processes are completed
+end
+
+##Titles
+titles = []
+titles << "host"
+if OPTIONS[:ping]
+  titles << "ping"
+end
+if OPTIONS[:pbsnodes]
+  titles << "state"
+  titles << "np"          unless OPTIONS[:load]
+  titles << "properties"  unless OPTIONS[:load]
+  titles << "ntype"       unless OPTIONS[:load]
+  titles << "gpus"        unless OPTIONS[:load]
+  titles << "rectime"     unless OPTIONS[:load]
+  titles << "varattr"     unless OPTIONS[:load]
+  titles << "jobs"
+  titles << "state"       unless OPTIONS[:load]
+  titles << "netload"     unless OPTIONS[:load]
+  titles << "gres"        unless OPTIONS[:load]
+  titles << "loadave"
+  titles << "ncpus"       unless OPTIONS[:load]
+  titles << "physmem"     unless OPTIONS[:load]
+  titles << "availmem"    unless OPTIONS[:load]
+  titles << "totmem"      unless OPTIONS[:load]
+  titles << "idletime"    unless OPTIONS[:load]
+  titles << "nusers"      unless OPTIONS[:load]
+  titles << "nsessions"   unless OPTIONS[:load]
+  titles << "sessions"    unless OPTIONS[:load]
+  titles << "uname"       unless OPTIONS[:load]
+  titles << "opsys"       unless OPTIONS[:load]
+end
+
+##Data
+data = []
+results.keys.sort.each do |host|
+  items = [host]
+  #printf("%9s: ", )
+  #
+  if OPTIONS[:ping]
+    #state = HIGHLINE.color("dead", :red)
+    state = "dead"
+    state = "alive" if results[host][:ping].alive?
+    items << state
+    #sprintf("%5s", state)
+  end
+
+  if OPTIONS[:pbsnodes]
+    if results[host][:pbsnodes]
+      items << results[host][:pbsnodes].state
+      items << results[host][:pbsnodes].np                  unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].properties          unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].ntype               unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].gpus                unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["rectime"  ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["varattr"  ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["jobs"     ]                       
+      items << results[host][:pbsnodes].status["state"    ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["netload"  ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["gres"     ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["loadave"  ]                       
+      items << results[host][:pbsnodes].status["ncpus"    ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["physmem"  ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["availmem" ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["totmem"   ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["idletime" ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["nusers"   ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["nsessions"] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["sessions" ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["uname"    ] unless OPTIONS[:load] 
+      items << results[host][:pbsnodes].status["opsys"    ] unless OPTIONS[:load] 
+    else
+      items << "pbs_unknown_host"
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << ""                       
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << ""                       
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+      items << "" unless OPTIONS[:load] 
+
+    end
+  end
+  data << items
+end
+
+data = data.unshift(titles)
+#pp data
+Tefil::ColumnFormer.form(data, $stdout, "|")

data/bin/scpall

@@ -0,0 +1,64 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+USAGE = <<HERE
+  scpall
+    scp from from on localhost to remotehost.
+
+    E.g.,
+      scpall -f from_file -t to_file HostA HostB
+        Indicate each host
+      scpall -f from_file -t to_file GroupA GroupB
+        Indicate Hosts with group names
+      scpall -f from_file -t to_file
+        Execute on all hosts when empty targets
+
+      scpall -u USER -f from_file -t to_file
+        Login as USER
+HERE
+
+require "optparse"
+require "pp"
+require "rubygems"
+require "comana"
+
+## option analysis
+OPTIONS = {}
+op = OptionParser.new
+op.on("-f from_file", "--from-file=str", "Copied file in a localhost."){|v| OPTIONS[:from_file] = v}
+op.on("-t from_file", "--to-file=str", "File path in remotehosts."){|v| OPTIONS[:to_file] = v}
+op.on("-g", "--group", "Interpret arguments as group name."){OPTIONS[:group] = true}
+op.on("-u user", "--user=str"    , "User on remote host."){|v| OPTIONS[:user] = v}
+op.on("-T", "--test"    , "Test mode; not execute."){OPTIONS[:test] = true}
+op.parse!(ARGV)
+
+unless OPTIONS[:from_file]
+  puts "-f option is not set."
+  puts USAGE; exit
+end
+unless OPTIONS[:to_file]
+  puts "-t option is not set."
+  puts USAGE; exit
+end
+
+hs = Comana::HostSelector.load_file
+hosts = []
+if ARGV.empty?
+  hosts = hs.select_all
+elsif OPTIONS[:group]
+  ARGV.each do |tgt|
+    hosts = hs.select_group(tgt)
+  end
+else
+  hosts = ARGV
+end
+
+hosts.each do |host|
+  host = "#{OPTIONS[:user]}@#{host}" if OPTIONS[:user]
+
+  command = "scp #{OPTIONS[:from_file]} #{host}:#{OPTIONS[:to_file]}"
+  print "#####{command}####"
+  puts
+  system command unless OPTIONS[:test]
+end
+

data/bin/sshall

@@ -0,0 +1,58 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+USAGE = <<HERE
+  sshall
+    -g groups
+
+    E.g.,
+      sshall -c "ls -l /" HostA HostB
+        Indicate each host
+      sshall -c "ls -l /" -g GroupA GroupB
+        Indicate Hosts with group names
+      sshall -c "ls -l /"
+        Execute on all hosts when empty targets
+
+      sshall HostA HostB
+        Login each hosts when -c option is omitted.
+
+      sshall -u USER -c "ls -l /"
+        Login as USER
+HERE
+
+require "optparse"
+require "pp"
+require "rubygems"
+require "comana"
+
+## option analysis
+OPTIONS = {}
+op = OptionParser.new
+op.on("-c command"    , "--command=str"    , "Command to be sent."){|v| OPTIONS[:command] = v}
+op.on("-g", "--group", "Interpret arguments as group name."){OPTIONS[:group] = true}
+op.on("-u user", "--user=str"    , "User on remote host."){|v| OPTIONS[:user] = v}
+op.on("-T", "--test"    , "Test mode; not execute."){OPTIONS[:test] = true}
+op.parse!(ARGV)
+
+hs = Comana::HostSelector.load_file
+
+hosts = []
+if ARGV.empty?
+  hosts = hs.select_all
+elsif OPTIONS[:group]
+  ARGV.each do |tgt|
+    hosts = hs.select_group(tgt)
+  end
+else
+  hosts = ARGV
+end
+
+hosts.each do |host|
+  host = "#{OPTIONS[:user]}@#{host}" if OPTIONS[:user]
+
+  command = "ssh #{host} #{OPTIONS[:command]}"
+  print "#####{command}####"
+  puts
+  system command unless OPTIONS[:test]
+end
+

data/example/dot.clustersetting

@@ -0,0 +1,15 @@
+--- # vim: syntax=yaml
+# cluster_name: [ num_economy, num_speed ]
+
+pbs_server: "P00"
+
+groups:
+  A:
+    data1:  'A-1'
+    data2:  'A-2'
+    members: ["A00", "A01"]
+
+  B:
+    data1:  'B-1'
+    data2:  'B-2'
+    members: ["B00", "B01", "B02"]

data/lib/comana.rb

@@ -1,2 +1,13 @@
+module Comana; end
+
+#require "rubygems"
+#gem     "nokogiri"
+#require "nokogiri"
+
 require "comana/computationmanager.rb"
-require "comana/machineinfo.rb"
+require "comana/clustersetting.rb"
+require "comana/hostselector.rb"
+require "comana/queuesubmitter.rb"
+require "comana/hostinspector.rb"
+require "comana/hostinspector/ping.rb"
+require "comana/hostinspector/pbsnodes.rb"

data/lib/comana/clustersetting.rb

@@ -0,0 +1,55 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+require "yaml"
+
+# Series name is composed only of alphabets.
+# Host name is started by the series name and followed by integers.
+# E.g.,
+#   "Fe", "Fe00", "Fe01" are of series "Fe" and not "F"
+class Comana::ClusterSetting
+  attr_reader :groups, :pbs_server
+
+  DEFAULT_DATA_FILE = ENV["HOME"] + "/.clustersetting"
+
+  class NoEntryError < Exception; end
+
+  #
+  def initialize(settings)
+    @pbs_server = settings["pbs_server"]
+    @groups = settings["groups"]
+  end
+
+  def self.load_file(data_file = DEFAULT_DATA_FILE)
+    settings = YAML.load_file(data_file)
+    self.new settings
+  end
+
+  #Return belonged cluster of the host.
+  #Raise NoEntryError if not match.
+  def belonged_cluster(hostname)
+    @groups.each do |group, settings|
+      next unless settings["members"]
+      return group if settings["members"].include? hostname
+    end
+    raise NoEntryError, "#{hostname} is not in `@groups': #{@groups.inspect}"
+  end
+
+  #Return settings as a hash for a cluster.
+  def settings_group(clustername)
+    @groups[clustername]
+  end
+
+  #Return settings as a hash for a host belonged to cluster.
+  def settings_host(hostname)
+    settings_group(belonged_cluster(hostname))
+  end
+
+  #Return an array of cluster names.
+  def clusters
+    @groups.keys
+  end
+
+
+end
+