drbqs 0.0.8 → 0.0.9

data/lib/drbqs/client.rb

@@ -18,9 +18,21 @@ module DRbQS
       @process_continue = opts[:continue]
     end
 
+    def transfer_file
+      until FileTransfer.empty?
+        path = FileTransfer.dequeue
+        unless @transfer.scp(path)
+          raise "Can not send file: #{path}"
+        end
+      end
+    end
+    private :transfer_file
+
     def execute_task(marshal_obj, method_sym, args)
       obj = Marshal.load(marshal_obj)
-      obj.__send__(method_sym, *args)
+      result = obj.__send__(method_sym, *args)
+      transfer_file
+      result
     end
     private :execute_task
 
@@ -29,6 +41,7 @@ module DRbQS
       @connection = ConnectionClient.new(obj[:message], @logger)
       node_id = @connection.get_id
       @task_client = TaskClient.new(node_id, obj[:queue], obj[:result], @logger)
+      @transfer = obj[:transfer]
       if ary = @connection.get_initialization
         execute_task(*ary)
       end

data/lib/drbqs/config.rb

@@ -3,10 +3,14 @@ require 'singleton'
 
 module DRbQS
 
+  ACL_DEFAULT_PATH = 'acl.txt'
+  ACL_SAMPLE_PATH = 'acl.txt.sample'
+  HOST_FILE_DIRECTORY = 'host'
+
   class Config
 
     @@data = {
-      :dir => ENV['HOME'] + '/.drbqs/'
+      :dir => ENV['HOME'] + '/.drbqs/',
     }
 
     ACL_SAMPLE =<<SAMPLE
@@ -25,22 +29,30 @@ SAMPLE
         @@data[:dir] = dir
       end
 
+      def get_host_file_directory
+        get_path(HOST_FILE_DIRECTORY)
+      end
+
       def check_directory_create
         unless File.exist?(@@data[:dir])
           FileUtils.mkdir_p(@@data[:dir])
           FileUtils.chmod(0700, @@data[:dir])
         end
+        host = get_host_file_directory
+        unless File.exist?(host)
+          FileUtils.mkdir_p(host)
+        end
       end
 
       def save_sample
-        path = get_path('acl.txt.sample')
+        path = get_path(ACL_SAMPLE_PATH)
         unless File.exist?(path)
           open(path, 'w') { |f| f.print ACL_SAMPLE }
         end
       end
 
       def get_acl_file
-        path = File.join(@@data[:dir], 'acl.txt')
+        path = File.join(@@data[:dir], ACL_DEFAULT_PATH)
         if File.exist?(path)
           return path
         end

data/lib/drbqs/manage.rb

@@ -2,6 +2,40 @@ require 'socket'
 
 module DRbQS
 
+  class SendCommand
+    MAX_WAIT_TIME = 10
+
+    def initialize(message)
+      @message = message
+    end
+
+    def get_hostname
+      "Command of #{Socket.gethostname}"
+    end
+    private :get_hostname
+
+    def send_exit_signal
+      @message.write([:exit_server, get_hostname])
+    end
+
+    def get_status
+      @message.write([:request_status, get_hostname])
+      i = 0
+      loop do
+        begin
+          mes = @message.take([:status, String], 0)
+          return mes[1]
+        rescue Rinda::RequestExpiredError
+          i += 1
+          if i > MAX_WAIT_TIME
+            return nil
+          end
+          sleep(1)
+        end
+      end
+    end
+  end
+
   class Manage
     def self.split_arguments(argv, split = '--')
       if n = argv.index(split)
@@ -16,15 +50,28 @@ module DRbQS
       Config.save_sample
     end
 
-    def send_exit_signal(access_uri)
+    def command_client(access_uri)
       obj = DRbObject.new_with_uri(access_uri)
-      obj[:message].write([:exit_server, "Command of #{Socket.gethostname}"])
+      DRbQS::SendCommand.new(obj[:message])
+    end
+    private :command_client
+
+    def send_exit_signal(access_uri)
+      command_client(access_uri).send_exit_signal
+    end
+
+    def get_status(access_uri)
+      command_client(access_uri).get_status
     end
 
     def execute_over_ssh(dest, opts, command)
       ssh = DRbQS::SSHShell.new(dest, opts)
-      ssh.get_environment
       ssh.start(command)
     end
+
+    def get_ssh_environment(dest, opts)
+      ssh = DRbQS::SSHShell.new(dest, opts)
+      ssh.get_environment
+    end
   end
 end

data/lib/drbqs/message.rb

@@ -28,11 +28,13 @@ module DRbQS
         @node_list.set_alive(arg)
       when :exit_server
         @logger.info("Get exit message from #{arg.to_s}") if @logger
-        return :exit_server
+      when :request_status
+        @logger.info("Get status request from #{arg.to_s}") if @logger
       else
         @logger.error("Invalid message from #{arg.to_s}") if @logger
         return nil
       end
+      return mes
     end
     private :manage_message
 
@@ -52,6 +54,29 @@ module DRbQS
       end
     end
 
+    def time_to_string(t)
+      t.strftime("%Y-%m-%d %H:%M:%S")
+    end
+    private :time_to_string
+
+    def send_status(calculating_task_id)
+      s = ''
+      @node_list.history.each do |node_id, hist|
+        s << sprintf("%4d %s\t", node_id, hist[0])
+        if hist.size == 3
+          s << "disconnected: (#{time_to_string(hist[1])} - #{time_to_string(hist[1])})\n"
+        else
+          task_ids = calculating_task_id[node_id]
+          s << "task: #{task_ids.map { |num| num.to_s }.join(', ')} (#{time_to_string(hist[1])})\n"
+        end
+      end
+      begin
+        @message.take([:status, nil], 0)
+      rescue Rinda::RequestExpiredError
+      end
+      @message.write([:status, s])
+    end
+
     def node_not_exist?
       @node_list.empty?
     end

data/lib/drbqs/node_list.rb

@@ -1,14 +1,38 @@
 module DRbQS
+  class NodeHistory
+    def initialize
+      @data = {}
+    end
+
+    def add(id, id_str)
+      @data[id] = [id_str, Time.now]
+    end
+
+    def disconnect(id)
+      if @data[id]
+        @data[id] << Time.now
+      end
+    end
+
+    def each(&block)
+      @data.each(&block)
+    end
+  end
+
   class NodeList
+    attr_reader :history
+
     def initialize
       @id = 0
       @list = {}
       @check = []
+      @history = NodeHistory.new
     end
 
     def get_new_id(id_str)
       @id += 1
       @list[@id] = id_str
+      @history.add(@id, id_str)
       @id
     end
 
@@ -23,6 +47,7 @@ module DRbQS
     def delete_not_alive
       @check.each do |id|
         @list.delete(id)
+        @history.disconnect(id)
       end
       deleted = @check
       @check = []

data/lib/drbqs/queue.rb

@@ -1,6 +1,7 @@
 module DRbQS
 
   class QueueServer
+    attr_reader :calculating
 
     def initialize(queue, result, logger = nil)
       @queue = queue

data/lib/drbqs/server.rb

@@ -49,7 +49,8 @@ module DRbQS
       @ts = {
         :message => Rinda::TupleSpace.new,
         :queue => Rinda::TupleSpace.new,
-        :result => Rinda::TupleSpace.new
+        :result => Rinda::TupleSpace.new,
+        :transfer => nil
       }
       if opts[:log_file]
         @logger = Logger.new(opts[:log_file])
@@ -162,9 +163,16 @@ module DRbQS
       end
     end
 
+    def set_file_transfer(user, host, directory)
+      @ts[:transfer] = DRbQS::Transfer.new(user, host, directory)
+    end
+
     def check_message
-      if @message.get_message == :exit_server
+      case @message.get_message
+      when :exit_server
         self.exit
+      when :request_status
+        @message.send_status(@queue.calculating)
       end
     end
     private :check_message

data/lib/drbqs/ssh/host.rb

@@ -0,0 +1,26 @@
+require 'yaml'
+require 'drbqs/config'
+
+module DRbQS
+  class SSHHost
+    def initialize
+      @dir = DRbQS::Config.get_host_file_directory
+      @host_files = (Dir.glob("#{@dir}/*.yaml") + Dir.glob("#{@dir}/*.yml")).map { |s| File.basename(s) }
+    end
+
+    def get(name)
+      if (name.size > 0) && (host = @host_files.find { |s| /^#{name}/ =~ s })
+        return File.join(@dir, host)
+      end
+      return nil
+    end
+    private :get
+
+    def get_options(name)
+      if path = get(name)
+        return [path, YAML.load_file(path)]
+      end
+      return [nil, {}]
+    end
+  end
+end

data/lib/drbqs/{ssh_shell.rb → ssh/shell.rb}

@@ -31,8 +31,11 @@ module DRbQS
       if (@rvm || @rvm_init) && !(String === @rvm_init)
         @rvm_init = DEFAULT_RVM_SCRIPT
       end
-      @out = opts[:output] || DEFAULT_OUTPUT_FILE
+      @nohup_output = opts[:output] || DEFAULT_OUTPUT_FILE
       @directory = opts[:dir]
+      @out = $stdout
+      @nice = opts[:nice]
+      @nohup = opts[:nohup]
     end
 
     def split_destination(dest)
@@ -55,38 +58,82 @@ module DRbQS
     end
     private :split_destination
 
+    def output_command(cmd, result)
+      @out.puts "#{@user}@#{@host}$ #{cmd}" if @out
+      @out.print result
+    end
+    private :output_command
+
+    def shell_exec_get_output(sh, cmd)
+      result = ''
+      pr_cmd = sh.execute!(cmd) do |sh_proc|
+        sh_proc.on_output do |pr, data|
+          result << data
+        end
+        sh_proc.on_error_output do |pr, data|
+          result << data
+        end
+      end
+      [pr_cmd, result]
+    end
+    private :shell_exec_get_output
+
     def shell_exec(sh, cmd)
-      pr = sh.execute!(cmd)
-      if pr.exit_status != 0
+      ary = shell_exec_get_output(sh, cmd)
+      output_command(cmd, ary[1])
+      ary
+    end
+    private :shell_exec
+
+    def shell_exec_check(sh, cmd)
+      ary = shell_exec(sh, cmd)
+      if ary[0].exit_status != 0
         raise GetInvalidExitStatus, "Can not execute '#{cmd}' on #{@host} properly."
       end
+      ary
     end
-    private :shell_exec
+    private :shell_exec_check
 
-    def execute_command(*cmds)
+    def execute_command(&block)
       Net::SSH.start(@host, @user, :port => @port) do |ssh|
         ssh.shell(@shell) do |sh|
-          shell_exec(sh, "cd #{@directory}") if @directory
-          shell_exec(sh, "source #{@rvm_init}") if @rvm_init
-          shell_exec(sh, "rvm use #{@rvm}") if @rvm
-          cmds.each do |c|
-            sh.execute c
-          end
-          sh.execute "exit"
+          shell_exec_check(sh, "cd #{@directory}") if @directory
+          shell_exec_check(sh, "source #{@rvm_init}") if @rvm_init
+          shell_exec_check(sh, "rvm use #{@rvm}") if @rvm
+          yield(sh)
+          shell_exec(sh, "exit")
         end
       end
     end
     private :execute_command
 
     def get_environment
-      execute_command('echo "directory: " `pwd`',
-                      'echo "files:"',
-                      'ls',
-                      'if which rvm > /dev/null; then rvm info; else ruby -v; fi')
+      execute_command do |sh|
+        ['echo "directory: " `pwd`',
+         'echo "files:"',
+         'ls',
+         'if which rvm > /dev/null; then rvm info; else ruby -v; fi'].each do |cmd|
+          shell_exec(sh, cmd)
+        end
+      end
     end
 
     def start(*args)
-      execute_command("nohup #{args.join(' ')} > #{@out} 2>&1 &")
+      cmd = args.join(' ')
+      if @nice
+        if Integer === @nice
+          cmd = "nice -n #{@nice.to_s} " + cmd
+        else
+          cmd = "nice " + cmd
+        end
+      end
+      execute_command do |sh|
+        if @nohup
+          pr, path = shell_exec_check(sh, "drbqs-manage new-filename #{@nohup_output}")
+          cmd = "nohup #{cmd} > #{path.strip} 2>&1 &"
+        end
+        shell_exec(sh, cmd)
+      end
     end
   end
 end

data/lib/drbqs/ssh/transfer.rb

@@ -0,0 +1,70 @@
+require 'fileutils'
+
+module DRbQS
+
+  # Transfer files to directory on DRbQS server.
+  # In this class we use scp command.
+  # Note that after we transfer files we delete the files.
+  class Transfer
+
+    # options
+    #   :mkdir    true or nil
+    def initialize(user, host, directory)
+      @user = user
+      @host = host
+      @directory = File.expand_path(directory)
+      FileUtils.mkdir_p(@directory)
+    end
+
+    def scp(path)
+      name = File.basename(path)
+      unless File.exist?(path)
+        raise ArgumentError, "File #{path} does not exist."
+      end
+      if system("scp -r #{path} #{@user}@#{@host}:#{File.join(@directory, name)} > /dev/null 2>&1")
+        FileUtils.rm_r(path)
+        return true
+      end
+      return false
+    end
+  end
+
+  # To compress files, we use gzip and tar command.
+  # Note that if we compress files then we delete the source files.
+  module FileTransfer
+    @@files = Queue.new
+
+    def self.enqueue(path, compress = false)
+      if compress
+        if File.directory?(path)
+          gz_path = "#{path.sub(/\/$/, '')}.tar.gz"
+          cmd = "tar czf #{gz_path} -C #{File.dirname(path)} #{File.basename(path)} > /dev/null 2>&1"
+        else
+          gz_path = path + '.gz'
+          cmd = "gzip --best #{path} > /dev/null 2>&1"
+        end
+        if File.exist?(gz_path)
+          raise "File has already existed: #{gz_path}"
+        elsif !system(cmd)
+          raise "Can not compress: #{path}"
+        end
+        FileUtils.rm_r(path) if File.exist?(path)
+        @@files.enq(gz_path)
+      else
+        @@files.enq(path)
+      end
+    end
+
+    def self.compress_enqueue(path)
+      self.enqueue(path, true)
+    end
+
+    def self.dequeue
+      @@files.deq
+    end
+
+    def self.empty?
+      @@files.empty?
+    end
+  end
+end