drbqs 0.0.13 → 0.0.14

data/lib/drbqs/utility/command_line/command_ssh.rb

@@ -0,0 +1,126 @@
+module DRbQS
+  class CommandSSH < CommandBase
+    HELP_MESSAGE =<<HELP
+Usage: #{@@command_name} <command> [arguments ...]
+  Execute command over SSH.
+  <command> is 'list', 'show', 'execute', or 'environment'
+
+  #{@@command_name} list
+  #{@@command_name} show <configuration>
+  #{@@command_name} environment <destination>
+  #{@@command_name} execute <destination>
+
+HELP
+
+    def parse_option(argv)
+      options = {}
+      argv, command_args = split_arguments(argv)
+
+      begin
+        OptionParser.new(HELP_MESSAGE) do |opt|
+          opt.on('--debug', 'Set $DEBUG true.') do |v|
+            $DEBUG = true
+          end
+          opt.on('--dir DIR', String, 'Set the base directory over ssh.') do |v|
+            options[:dir] = v
+          end
+          opt.on('--shell STR', String, 'Set the shell over ssh') do |v|
+            options[:shell] = v
+          end
+          opt.on('--rvm STR', String, 'Ruby version to use on RVM over ssh.') do |v|
+            options[:rvm] = v
+          end
+          opt.on('--rvm-init PATH', String, 'Path of script to initialize RVM over ssh.') do |v|
+            options[:rvm_init] = v
+          end
+          opt.on('--output PATH', String, 'File path that stdout and stderr are output to over ssh.') do |v|
+            options[:output] = v
+          end
+          opt.on('--nice NUM', Integer, 'Set the value for nice command.') do |v|
+            options[:nice] = v
+          end
+          opt.on('--nohup', 'Use nohup command.') do |v|
+            options[:nohup] = true
+          end
+          opt.parse!(argv)
+        end
+      rescue OptionParser::InvalidOption
+        $stderr.print "error: Invalid Option\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      rescue OptionParser::InvalidArgument
+        $stderr.print "error: Invalid Argument\n\n" << HELP_MESSAGE
+        exit_invalid_option
+      end
+      @options = options
+      @command = argv.shift
+      @argv = argv
+      @command_args = command_args
+    end
+
+    def command_list
+      ssh_host = DRbQS::Config.new.ssh_host
+      $stdout.puts ssh_host.config_names.join("\n")
+      exit_normally
+    end
+    private :command_list
+
+    def command_show
+      check_argument_size(@argv, :==, 1)
+      name = @argv[0]
+      ssh_host = DRbQS::Config.new.ssh_host
+      if path = ssh_host.get_path(name)
+        $stdout.puts File.read(path)
+        exit_normally
+      else
+        $stderr.print "Can not find configuration file '#{name}'."
+        exit_unusually
+      end
+    end
+    private :command_show
+
+    def manage_ssh(dest)
+      DRbQS::Manage::SSHExecute.new(dest, @options)
+    end
+    private :manage_ssh
+
+    def command_environment
+      check_argument_size(@argv, :==, 1)
+      dest = @argv[0]
+      manage_ssh(dest).get_environment
+      exit_normally
+    end
+    private :command_environment
+
+    def command_execute
+      check_argument_size(@argv, :==, 1)
+      dest = @argv[0]
+      mng_ssh = manage_ssh(dest)
+      if @command_args.size > 0
+        mng_ssh.execute(@command_args)
+        exit_normally
+      else
+        $stderr.print "error: There is no command for ssh.\n\n" << HELP_MESSAGE
+        exit_unusually
+      end
+    end
+    private :command_execute
+
+    def exec
+      case @command
+      when 'list'
+        command_list
+      when 'show'
+        command_show
+      when 'environment'
+        command_environment
+      when 'execute'
+        command_execute
+      end
+      $stderr.print "error: Invalid command '#{@command}'.\n\n" << HELP_MESSAGE
+      exit_invalid_option
+    rescue => err
+      $stderr.print "error: #{err.to_s}\n" << err.backtrace.join("\n")
+      exit_unusually
+    end
+  end
+end

data/lib/drbqs/utility/command_line.rb

@@ -0,0 +1,15 @@
+require 'drbqs'
+require 'drbqs/utility/argument'
+require 'drbqs/manage/manage'
+require 'drbqs/manage/execute_node'
+require 'drbqs/utility/command_line/command_base'
+require 'optparse'
+
+Version = DRbQS::VERSION
+
+module DRbQS
+  autoload :CommandManage, 'drbqs/utility/command_line/command_manage'
+  autoload :CommandServer, 'drbqs/utility/command_line/command_server'
+  autoload :CommandNode, 'drbqs/utility/command_line/command_node'
+  autoload :CommandSSH, 'drbqs/utility/command_line/command_ssh'
+end

data/lib/drbqs/utility/misc.rb

@@ -0,0 +1,72 @@
+require 'sys/proctable'
+
+module DRbQS
+  class LoggerDummy
+    def info(*args)
+    end
+
+    def warn(*args)
+    end
+
+    def error(*args)
+    end
+
+    def debug(*args)
+    end
+  end
+
+  module Misc
+    def create_uri(opts = {})
+      if opts[:port] || !opts[:unix]
+        port = opts[:port] || ROOT_DEFAULT_PORT
+        "druby://:#{port}"
+      else
+        path = File.expand_path(opts[:unix])
+        if !File.directory?(File.dirname(path))
+          raise ArgumentError, "Directory #{File.dirname(path)} does not exist."
+        elsif File.exist?(path)
+          raise ArgumentError, "File #{path} already exists."
+        end
+        "drbunix:#{path}"
+      end
+    end
+    module_function :create_uri
+
+    def create_logger(log_file, log_level)
+      if IO === log_file
+        log_output = log_file
+      elsif log_file
+        log_output = FileName.create(log_file, :position => :middle, :directory => :parent, :type => :number)
+      else
+        log_output = STDOUT
+      end
+      logger = Logger.new(log_output)
+      logger.level = log_level || Logger::ERROR
+      logger
+    end
+    module_function :create_logger
+
+    def time_to_history_string(t)
+      t.strftime("%Y-%m-%d %H:%M:%S")
+    end
+    module_function :time_to_history_string
+
+    STRINGS_FOR_KEY = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+
+    def random_key(size = 20)
+      n = STRINGS_FOR_KEY.size
+      Array.new(size) do
+        STRINGS_FOR_KEY[rand(n)]
+      end.join
+    end
+    module_function :random_key
+
+    # If process of +pid+ does not exist or its state is zombie then the method return false.
+    # If +pid+ is invalid then the method also returns false.
+    def process_running_normally?(pid)
+      Integer === pid && (ps_table = Sys::ProcTable.ps(pid)) && (ps_table.state != 'Z')
+    end
+    module_function :process_running_normally?
+  end
+
+end

data/lib/drbqs/{server_define.rb → utility/server_define.rb}

@@ -2,17 +2,13 @@ module DRbQS
 
   class ServerDefinition
     HELP_MESSAGE =<<HELP
-[Server specific options]
+* Server specific options
   These options are separated by '--' from command options.
 
 HELP
 
     def initialize
-      @server_create = nil
-      @option_parse = nil
-      @opts = {}
-      @argv = nil
-      @default_server_opts = nil
+      clear_definition
     end
 
     def define_server(default_opts = {}, &block)
@@ -40,6 +36,25 @@ HELP
       @argv = opt_argv  
     end
 
+    def option_help_message
+      if @option_parse
+        OptionParser.new(HELP_MESSAGE) do |opt|
+          @option_parse.call(opt, @opts)
+          return opt.to_s
+        end
+      else
+        nil
+      end
+    end
+
+    def clear_definition
+      @server_create = nil
+      @option_parse = nil
+      @opts = {}
+      @argv = nil
+      @default_server_opts = nil
+    end
+
     def create_server(options)
       server = DRbQS::Server.new(@default_server_opts.merge(options))
       @server_create.call(server, @argv, @opts)
@@ -71,10 +86,10 @@ HELP
     end
   end
 
-  @@server_def = ServerDefinition.new
+  @@server_def = DRbQS::ServerDefinition.new
 
   class << self
-    [:define_server, :option_parser, :parse_option,
+    [:define_server, :option_parser, :parse_option, :option_help_message, :clear_definition,
      :start_server, :test_server].each do |m|
       define_method(m, &@@server_def.method(m))
     end

data/lib/drbqs/utility/temporary.rb

@@ -0,0 +1,49 @@
+module DRbQS
+  module Temporary
+    @@root = nil
+    @@filename = nil
+
+    # Return FileName object to generate names of temporary files on DRbQS nodes.
+    def self.filename
+      unless @@filename
+        pid = Process.pid
+        @@root = sprintf("/tmp/drbqs_%d_%d", pid, rand(10000))
+        FileUtils.mkdir_p(@@root)
+        @@filename = FileName.new(File.join(@@root, sprintf("temp_%d_%d", pid, rand(10000))))
+      end
+      @@filename
+    end
+
+    # Create new temporary directory and return the path of directory.
+    def self.directory
+      filename.create(:add => :always, :directory => :self)
+    end
+
+    # Return new path of temporary file.
+    def self.file
+      filename.create(:add => :always)
+    end
+
+    # Make root of temporary directory empty.
+    def self.delete
+      if @@root
+        FileUtils.rm_r(@@root)
+        FileUtils.mkdir_p(@@root)
+      end
+    end
+
+    # Delete all temporary directory.
+    def self.delete_all
+      if @@root
+        FileUtils.rm_r(@@root)
+        @@root = nil
+        @@filename = nil
+      end
+    end
+
+    # Return root of temporary directory.
+    def self.root
+      @@root
+    end
+  end
+end

data/lib/drbqs/{ssh/transfer.rb → utility/transfer/file_transfer.rb}

@@ -1,66 +1,17 @@
 module DRbQS
-
-  # Transfer files to directory on DRbQS server.
-  # In this class we use scp command.
-  # Note that after we transfer files we delete the files.
-  class Transfer
-    attr_reader :user, :host, :directory
-
-    # options
-    #   :mkdir    true or nil
-    def initialize(user, host, directory)
-      @user = user
-      @host = host
-      @directory = File.expand_path(directory)
-      FileUtils.mkdir_p(@directory)
-    end
-
-    def transfer_file(path, name)
-      system("scp -r #{path} #{@user}@#{@host}:#{File.join(@directory, name)} > /dev/null 2>&1")
-    end
-    private :transfer_file
-
-    def scp(path)
-      name = File.basename(path)
-      unless File.exist?(path)
-        raise ArgumentError, "File #{path} does not exist."
-      end
-      if transfer_file(path, name)
-        FileUtils.rm_r(path)
-        return true
-      end
-      return false
-    end
-
-    def information
-      "#{@user}@#{@host} #{@directory}"
-    end
-  end
-
-  class TransferTest < Transfer
-    def initialize(directory)
-      @directory = File.expand_path(directory)
-      FileUtils.mkdir_p(@directory)
-    end
-
-    def transfer_file(path, name)
-      FileUtils.cp(path, File.join(@directory, name))
-      true
-    end
-    private :transfer_file
-
-    def information
-      @directory
-    end
-  end
-
   # To compress files, we use gzip and tar command.
   # Note that if we compress files then we delete the source files.
   module FileTransfer
     @@files = Queue.new
 
-    def self.enqueue(path, compress = false)
-      if compress
+    # If opts[:compress] is true then the file of +path+ is compressed before tranfering.
+    def self.enqueue(path, opts = {})
+      if opts[:rename]
+        new_path = FileName.create(File.join(File.dirname(path), opts[:rename]), :directory => :parent)
+        FileUtils.mv(path, new_path)
+        path = new_path
+      end
+      if opts[:compress]
         if File.directory?(path)
           gz_path = "#{path.sub(/\/$/, '')}.tar.gz"
           cmd = "tar czf #{gz_path} -C #{File.dirname(path)} #{File.basename(path)} > /dev/null 2>&1"
@@ -83,7 +34,7 @@ module DRbQS
     end
 
     def self.compress_enqueue(path)
-      self.enqueue(path, true)
+      self.enqueue(path, :compress => true)
     end
 
     def self.dequeue
@@ -94,6 +45,14 @@ module DRbQS
       @@files.empty?
     end
 
+    def self.dequeue_all
+      files = []
+      until self.empty?
+        files << self.dequeue
+      end
+      files.empty? ? nil : files
+    end
+
     def self.decompress(server, filename)
       dir = server.transfer_directory
       path = File.join(dir, filename)
@@ -110,4 +69,5 @@ module DRbQS
       end
     end
   end
+
 end

data/lib/drbqs/utility/transfer/transfer_client.rb

@@ -0,0 +1,90 @@
+require 'net/sftp'
+
+module DRbQS
+
+  class TransferClient
+
+    class ClientBase
+      def initialize(directory)
+        @directory = File.expand_path(directory)
+      end
+
+      def upload_name(path)
+        File.join(@directory, File.basename(path))
+      end
+      private :upload_name
+    end
+
+    # Transfer files to directory on DRbQS server over sftp.
+    # Note that after we transfer files we delete the original files.
+    class SFTP < DRbQS::TransferClient::ClientBase
+      attr_reader :user, :host, :directory
+
+      def initialize(user, host, directory)
+        super(directory)
+        @user = user
+        @host = host
+      end
+
+      # Transfer and delete +files+.
+      def transfer(files)
+        Net::SFTP.start(@host, @user) do |sftp|
+          files.each do |path|
+            sftp.upload(path, upload_name(path))
+            FileUtils.rm_r(path)
+          end
+        end
+      rescue => err
+        raise err.class, "user=#{@user}, host=#{@host}, directory=#{@directory}; #{err.to_s}", err.backtrace 
+      end
+    end
+
+    class Local < DRbQS::TransferClient::ClientBase
+      def transfer(files)
+        files.each do |path|
+          FileUtils.mv(path, upload_name(path))
+        end
+      end
+    end
+
+    attr_reader :directory, :local, :sftp
+
+    def initialize(dir)
+      @directory = File.expand_path(dir)
+      @local = DRbQS::TransferClient::Local.new(@directory)
+      @sftp = nil
+    end
+
+    def make_directory
+      FileUtils.mkdir_p(@directory)
+    end
+
+    def set_sftp(user, host)
+      @sftp = DRbQS::TransferClient::SFTP.new(user, host, @directory)
+    end
+
+    def information
+      info = "directory: #{@directory}"
+      info << ", sftp: #{@sftp.user}@#{@sftp.host}" if @sftp
+      info
+    end
+
+    def transfer(files, on_same_host = nil)
+      transfered = false
+      if on_same_host
+        begin
+          @local.transfer(files)
+          transfered = true
+        rescue
+        end
+      end
+      if !transfered
+        unless @sftp
+          raise "Can not transfer files."
+        end
+        @sftp.transfer(files)
+      end
+    end
+  end
+
+end

data/lib/drbqs.rb

@@ -1,35 +1,23 @@
 require 'thread'
 require 'drb'
 require 'drb/acl'
+require 'socket'
 require 'rinda/tuplespace'
 require 'rinda/rinda'
+require 'logger'
+require 'fileutils'
 
-require 'drbqs/server_define'
+require 'filename'
 
-autoload :Logger, 'logger'
-autoload :FileUtils, 'fileutils'
-
-gem 'filename'
-autoload :FileName, 'filename'
+require 'drbqs/utility/server_define'
+require 'drbqs/utility/misc'
 
 module DRbQS
-  autoload :Server, 'drbqs/server'
-  autoload :Client, 'drbqs/client'
-  autoload :Task, 'drbqs/task'
-  autoload :TaskGenerator, 'drbqs/task_generator'
-  autoload :Manage, 'drbqs/manage'
-  autoload :Config, 'drbqs/config'
-  autoload :SSHShell, 'drbqs/ssh/shell'
-  autoload :SSHHost, 'drbqs/ssh/host'
-  autoload :CommandTask, 'drbqs/task'
-  autoload :CommandExecute, 'drbqs/task'
-  autoload :TaskSet, 'drbqs/task'
-  autoload :Transfer, 'drbqs/ssh/transfer'
-  autoload :FileTransfer, 'drbqs/ssh/transfer'
-  autoload :Utils, 'drbqs/utils'
-  autoload :ExecuteNode, 'drbqs/execute_node'
+  autoload :Server, 'drbqs/server/server'
+  autoload :Node, 'drbqs/node/node'
+  autoload :Config, 'drbqs/config/config'
 
   ROOT_DEFAULT_PORT = 13500
 
-  VERSION = '0.0.13'
+  VERSION = '0.0.14'
 end

data/spec/config/config_spec.rb

@@ -0,0 +1,84 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe DRbQS::Config do
+  before(:all) do
+    FileUtils.mkdir_p(HOME_FOR_SPEC)
+  end
+
+  subject do
+    DRbQS::Config.new
+  end
+
+  context "when setting new homedirectory" do
+    before(:all) do
+      @old_home_directory = DRbQS::Config.get_home_directory
+    end
+
+    it "should change home directory." do
+      new_home_directory = '/tmp/drbqs_new_home'
+      DRbQS::Config.set_home_directory(new_home_directory)
+      DRbQS::Config.get_home_directory.should == new_home_directory
+    end
+
+    after(:all) do
+      DRbQS::Config.set_home_directory(@old_home_directory)
+    end
+  end
+
+  context "when creating configuration directory" do
+    before(:all) do
+      @dir = File.join(HOME_FOR_SPEC, '.drbqs')
+      FileUtils.rm_r(@dir) if File.exist?(@dir)
+    end
+
+    it "should get directory existing." do
+      lambda do
+        subject
+      end.should change { File.exist?(@dir) }.from(false).to(true)
+    end
+  end
+
+  context "when saving samples" do
+    before(:all) do
+      subject.save_sample
+    end
+
+    it "should get sample of ACL file." do
+      subject.directory.list_in_directory('.').should include('acl.txt.sample')
+    end
+
+    it "should get sample of host file." do
+      subject.directory.list_in_directory('host').should include('host.yaml.sample')
+    end
+
+    it "should get bashrc for SSH of drbqs." do
+      subject.directory.list_in_directory('shell').should include('bashrc')
+    end
+  end
+
+  context "when managing ACL file" do
+    before(:all) do
+      @path = subject.directory.file_path('acl.txt')
+      FileUtils.rm(@path) if File.exist?(@path)
+    end
+
+    it "should not have ACL file by default." do
+      subject.get_acl_file.should_not be_true
+    end
+
+    it "should return path of ACL file" do
+      open(@path, 'w') do |f|
+      end
+      subject.get_acl_file.should == File.join(@path)
+      FileUtils.rm(@path) if File.exist?(@path)
+    end
+  end
+
+  it "should return DRbQS::Config::SSHHost object." do
+    subject.ssh_host.should be_an_instance_of DRbQS::Config::SSHHost
+  end
+
+  after(:all) do
+    FileUtils.rm_r(HOME_FOR_SPEC)
+  end
+end