drbqs 0.0.10 → 0.0.11

data/Rakefile

@@ -23,6 +23,7 @@ Jeweler::Tasks.new do |gem|
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
   gem.add_runtime_dependency 'net-ssh', '>= 2.1.3'
   gem.add_runtime_dependency 'net-ssh-shell', '>= 0.1.0'
+  gem.add_runtime_dependency 'filename', '>= 0.0.5'
   gem.add_development_dependency 'rspec', '>= 2.5.0'
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-0.0.10
+0.0.11

data/bin/drbqs-manage

@@ -82,17 +82,15 @@ when 'exit-signal'
 when 'status'
   check_argument_size(argv, :==, 2)
   uri = argv[1]
-  $stdout.puts manage.get_status(uri)
+  if status = manage.get_status(uri)
+    $stdout.puts status
+  end
 when 'initialize'
   check_argument_size(argv, :==, 1)
   manage.create_config
 when 'ssh'
   dest = argv[1]
   manage.execute_over_ssh(dest, options, command_args)
-when 'new-filename'
-  check_argument_size(argv, :==, 2)
-  fname = DRbQS::FileName.new(argv[1], :add => :auto, :type => :time, :position => :suffix)
-  $stdout.puts fname.create(:directory => true)
 else
   raise "Invalid command: #{command}"
 end

data/bin/drbqs-node

@@ -82,18 +82,17 @@ puts "Connect to #{uri}"
 puts "Execute #{process_num} processes"
 
 if options[:log_prefix]
-  require 'fileutils'
   if /\/$/ =~ options[:log_prefix]
     options[:log_prefix] += 'out'
   end
-  options[:log_prefix] = File.expand_path(options[:log_prefix])
-  FileUtils.mkdir_p(File.dirname(options[:log_prefix]))
 end
 
 process_num.times do |i|
   fork do
     if options[:log_prefix]
-      logfile = "#{options[:log_prefix]}_#{Time.now.strftime("%Y%m%d%H%M")}_#{Process.pid}.log"
+      logfile = FileName.create(options[:log_prefix], :position => :suffix, :type => :time,
+                                :add => :always, :directory => true,
+                                :format => lambda { |t| t.strftime("%Y%m%d_%H%M_#{Process.pid}.log") })
     else
       logfile = STDOUT
     end

data/bin/drbqs-server

@@ -18,6 +18,8 @@ options = {
   :acl => DRbQS::Config.get_acl_file
 }
 
+test_opts = {}
+
 command_argv, server_argv = DRbQS::Manage.split_arguments(ARGV)
 
 begin
@@ -39,6 +41,18 @@ begin
         raise "Invalid log level."
       end
     end
+    opt.on('--file-directory DIR', String, 'Set the file archive directory.') do |v|
+      options[:file_directory] = v
+    end
+    opt.on('--scp-user USER', String, 'Set the user of scp destination.') do |v|
+      options[:scp_user] = v
+    end
+    opt.on('--scp-host HOST', String, 'Set the host of scp destination.') do |v|
+      options[:scp_host] = v
+    end
+    opt.on('--profile', 'Use profile for test exec.') do |v|
+      test_opts[:profile] = true
+    end
     opt.on('--debug', 'Set $DEBUG true.') do |v|
       $DEBUG = true
     end
@@ -75,7 +89,7 @@ case options[:command_type]
 when /^test/
   s = options[:command_type].split('_')[1].split(',')
   type = s[0].intern
-  DRbQS.test_server(options, type, s[1..-1])
+  DRbQS.test_server(options, type, s[1..-1], test_opts)
 else
   DRbQS.start_server(options)
 end

data/drbqs.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{drbqs}
-  s.version = "0.0.10"
+  s.version = "0.0.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Takayuki YAMAGUCHI"]
-  s.date = %q{2011-03-25}
+  s.date = %q{2011-04-09}
   s.description = %q{Task queuing system over network that is implemented by dRuby.}
   s.email = %q{d@ytak.info}
   s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server"]
@@ -34,6 +34,8 @@ Gem::Specification.new do |s|
     "example/drbqs-manage-test.rb",
     "example/drbqs-node-test.rb",
     "example/drbqs-server-test.rb",
+    "example/error/error.rb",
+    "example/error/server_def.rb",
     "example/server/server.rb",
     "example/sum/server_def.rb",
     "example/sum/sum.rb",
@@ -60,12 +62,11 @@ Gem::Specification.new do |s|
     "lib/drbqs/task.rb",
     "lib/drbqs/task_client.rb",
     "lib/drbqs/task_generator.rb",
-    "lib/drbqs/utils/filename.rb",
+    "lib/drbqs/utils.rb",
     "spec/acl_file_spec.rb",
     "spec/config_spec.rb",
     "spec/connection_spec.rb",
     "spec/data/acl.txt",
-    "spec/filename_spec.rb",
     "spec/history_spec.rb",
     "spec/manage_spec.rb",
     "spec/message_spec.rb",
@@ -88,13 +89,12 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/ytaka/drbqs}
   s.licenses = ["GPL3"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.5.2}
+  s.rubygems_version = %q{1.7.2}
   s.summary = %q{dRuby Queueing System}
   s.test_files = [
     "spec/acl_file_spec.rb",
     "spec/config_spec.rb",
     "spec/connection_spec.rb",
-    "spec/filename_spec.rb",
     "spec/history_spec.rb",
     "spec/manage_spec.rb",
     "spec/message_spec.rb",
@@ -126,6 +126,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_runtime_dependency(%q<net-ssh>, [">= 2.1.3"])
       s.add_runtime_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+      s.add_runtime_dependency(%q<filename>, [">= 0.0.5"])
       s.add_development_dependency(%q<rspec>, [">= 2.5.0"])
     else
       s.add_dependency(%q<rspec>, [">= 2.5.0"])
@@ -135,6 +136,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
       s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+      s.add_dependency(%q<filename>, [">= 0.0.5"])
       s.add_dependency(%q<rspec>, [">= 2.5.0"])
     end
   else
@@ -145,6 +147,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
     s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+    s.add_dependency(%q<filename>, [">= 0.0.5"])
     s.add_dependency(%q<rspec>, [">= 2.5.0"])
   end
 end

data/example/error/error.rb

@@ -0,0 +1,5 @@
+class TaskError
+  def exec
+    raise "Raise error"
+  end
+end

data/example/error/server_def.rb

@@ -0,0 +1,8 @@
+require_relative 'error.rb'
+
+DRbQS.define_server(:check_alive => 300) do |server, argv, opts|
+  server.queue.add(DRbQS::Task.new(TaskError, :exec))
+  server.add_hook(:finish) do |serv|
+    serv.exit
+  end
+end

data/example/server/server.rb

@@ -7,6 +7,8 @@ end
 
 server = DRbQS::Server.new(:port => 13501, :finish_exit => true)
 server.add_task_generator(task_generator)
+server.set_initialization_task(DRbQS::Task.new(Kernel, :puts, ['hook: initialize']))
+server.set_finalization_task(DRbQS::Task.new(Kernel, :puts, ['hook: finalize']))
 server.set_signal_trap
 server.start
 server.wait

data/example/transfer/file.rb

@@ -1,5 +1,3 @@
-require 'fileutils'
-
 class CreateFile
   def initialize(n)
     @n = n

data/example/transfer/server_def.rb

@@ -15,5 +15,5 @@ DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
   end
   server.add_task_generator(tgen)
 
-  server.set_file_transfer(ENV['USER'], 'localhost', '/tmp/drbqs_transfer_test/')
+  server.set_file_transfer('/tmp/drbqs_transfer_test/')
 end

data/lib/drbqs/client.rb

@@ -9,15 +9,16 @@ module DRbQS
     PRIORITY_RESPOND = 10
     PRIORITY_CALCULATE = 0
     OUTPUT_NOT_SEND_RESULT = 'not_send_result'
+    DEFAULT_LOG_FILE = 'drbqs_client.log'
 
     # :continue
     def initialize(access_uri, opts = {})
       @access_uri = access_uri
-      @logger = Logger.new(opts[:log_file] || 'drbqs_client.log')
-      @logger.level = opts[:log_level] || Logger::ERROR
+      @logger = DRbQS::Utils.create_logger(opts[:log_file] || DEFAULT_LOG_FILE, opts[:log_level])
       @connection = nil
       @task_client = nil
       @process_continue = opts[:continue]
+      @signal_queue = Queue.new
     end
 
     def transfer_file
@@ -72,12 +73,39 @@ module DRbQS
     end
     private :process_exit
 
+    def execute_finalization
+      if ary = @connection.get_finalization
+        execute_task(*ary)
+      end
+    rescue => err
+      output_error(err)
+    end
+    private :execute_finalization
+
+    def send_error(err)
+      output_error(err)
+      @connection.send_node_error("#{err.to_s}\n#{err.backtrace.join("\n")}")
+    end
+    private :send_error
+
     def communicate_with_server
       @task_client.add_new_task
-      if @connection.respond_signal == :exit
+      case @connection.respond_signal
+      when :exit
+        return nil
+      when :finalize
+        execute_finalization
         return nil
       end
       @task_client.send_result
+      until @signal_queue.empty?
+        signal, obj = @signal_queue.pop
+        case signal
+        when :node_error
+          send_error(obj)
+          process_exit
+        end
+      end
       return true
     end
     private :communicate_with_server
@@ -98,7 +126,7 @@ module DRbQS
             sleep(WAIT_NEW_TASK)
           end
         rescue => err
-          output_error(err)
+          send_error(err)
         ensure
           process_exit
         end
@@ -113,14 +141,20 @@ module DRbQS
             calculate_task
           end
         rescue => err
-          output_error(err)
-          process_exit
+          @signal_queue.push([:node_error, err])
         end
       end
     end
     private :thread_calculate
 
+    def set_signal_trap
+      Signal.trap(:TERM) do
+        process_exit
+      end
+    end
+
     def calculate(opts = {})
+      set_signal_trap
       cn = thread_communicate
       exec = thread_calculate
       cn.priority = PRIORITY_RESPOND

data/lib/drbqs/config.rb

@@ -1,4 +1,3 @@
-require 'fileutils'
 require 'singleton'
 
 module DRbQS
@@ -6,6 +5,9 @@ module DRbQS
   ACL_DEFAULT_PATH = 'acl.txt'
   ACL_SAMPLE_PATH = 'acl.txt.sample'
   HOST_FILE_DIRECTORY = 'host'
+  HOST_FILE_SAMPLE_PATH = 'host.yaml.sample'
+  SHELL_FILE_DIRECTORY = 'shell'
+  SHELL_BASHRC = 'bashrc'
 
   class Config
 
@@ -17,6 +19,22 @@ module DRbQS
 deny all
 allow localhost
 allow 127.0.0.1
+SAMPLE
+
+    HOST_YAML_SAMPLE =<<SAMPLE
+--- 
+:dest: user@example.com
+:dir:
+:shell: bash --noprofile --init-file ~/.drbqs/shell/bashrc
+:rvm:
+:rvm_init: ~/.rvm/scripts/rvm
+:output: 
+SAMPLE
+
+    BASHRC_SAMPLE = <<SAMPLE
+HISTFILE=$HOME/.drbqs/shell/bash_history
+HISTSIZE=10000
+HISTFILESIZE=20000
 SAMPLE
 
     class << self
@@ -33,23 +51,39 @@ SAMPLE
         get_path(HOST_FILE_DIRECTORY)
       end
 
+      def get_shell_file_directory
+        get_path(SHELL_FILE_DIRECTORY)
+      end
+
+      def make_directory(dir)
+        unless File.exist?(dir)
+          FileUtils.mkdir_p(dir)
+        end
+      end
+      private :make_directory
+
       def check_directory_create
         unless File.exist?(@@data[:dir])
           FileUtils.mkdir_p(@@data[:dir])
           FileUtils.chmod(0700, @@data[:dir])
         end
-        host = get_host_file_directory
-        unless File.exist?(host)
-          FileUtils.mkdir_p(host)
+        [get_host_file_directory, get_shell_file_directory].each do |dir|
+          make_directory(dir)
         end
       end
 
-      def save_sample
-        path = get_path(ACL_SAMPLE_PATH)
+      def output_to_file(path, content)
         unless File.exist?(path)
-          open(path, 'w') { |f| f.print ACL_SAMPLE }
+          open(path, 'w') { |f| f.print content }
         end
       end
+      private :output_to_file
+
+      def save_sample
+        output_to_file(get_path(ACL_SAMPLE_PATH), ACL_SAMPLE)
+        output_to_file("#{get_host_file_directory}/#{HOST_FILE_SAMPLE_PATH}", HOST_YAML_SAMPLE)
+        output_to_file("#{get_shell_file_directory}/#{SHELL_BASHRC}", BASHRC_SAMPLE)
+      end
 
       def get_acl_file
         path = File.join(@@data[:dir], ACL_DEFAULT_PATH)

data/lib/drbqs/connection.rb

@@ -25,14 +25,23 @@ module DRbQS
       @id_number
     end
 
-    def get_initialization
+    def get_special_task(label)
       begin
-        ary = @message.read([:initialize, nil, Symbol, nil], 0)
+        ary = @message.read([label, nil, Symbol, nil], 0)
         ary[1..-1]
       rescue Rinda::RequestExpiredError
         nil
       end
     end
+    private :get_special_task
+
+    def get_initialization
+      get_special_task(:initialize)
+    end
+
+    def get_finalization
+      get_special_task(:finalize)
+    end
 
     def respond_signal
       begin
@@ -42,11 +51,17 @@ module DRbQS
         when :alive_p
           @message.write([:server, :alive, @id_number])
           @logger.info("Send alive signal of node id #{@id_number}") if @logger
-        when :exit
-          return :exit
+        when :exit, :finalize
+          return sym
+        else
+          raise "Get invalid signal: #{sym.inspect}"
         end
       rescue Rinda::RequestExpiredError
       end
     end
+
+    def send_node_error(error_message)
+      @message.write([:server, :node_error, [@id_number, error_message]])
+    end
   end
 end

data/lib/drbqs/manage.rb

@@ -53,15 +53,22 @@ module DRbQS
     def command_client(access_uri)
       obj = DRbObject.new_with_uri(access_uri)
       DRbQS::SendCommand.new(obj[:message])
+    rescue DRb::DRbConnError
+      $stderr.puts "Can not access #{access_uri}"
+      nil
     end
     private :command_client
 
     def send_exit_signal(access_uri)
-      command_client(access_uri).send_exit_signal
+      if client = command_client(access_uri)
+        client.send_exit_signal
+      end
     end
 
     def get_status(access_uri)
-      command_client(access_uri).get_status
+      if client = command_client(access_uri)
+        client.get_status
+      end
     end
 
     def execute_over_ssh(dest, opts, command)

data/lib/drbqs/message.rb

@@ -32,11 +32,15 @@ module DRbQS
         @logger.info("Get exit message from #{arg.to_s}") if @logger
       when :request_status
         @logger.info("Get status request from #{arg.to_s}") if @logger
+      when :node_error
+        @node_list.delete(arg[0])
+        @logger.info("Node Error (#{arg[0]})") { arg[1] } if @logger
+        return [mes, arg[0]]
       else
         @logger.error("Invalid message from #{arg.to_s}") if @logger
         return nil
       end
-      return mes
+      return [mes]
     end
     private :manage_message
 
@@ -50,11 +54,21 @@ module DRbQS
       deleted
     end
 
-    def send_exit
+    def send_signal_to_all_nodes(signal)
       @node_list.each do |node_id, id_str|
-        @message.write([node_id, :exit])
+        @message.write([node_id, signal])
       end
     end
+    private :send_signal_to_all_nodes
+
+    def send_exit
+      send_signal_to_all_nodes(:exit)
+    end
+
+    def send_finalization(task)
+      set_finalization(task)
+      send_signal_to_all_nodes(:finalize)
+    end
 
     def send_status(calculating_task_id)
       s = ''
@@ -82,16 +96,24 @@ module DRbQS
       @node_list.empty?
     end
 
-    # If the task has already set,
-    # the method overwrite old task of initialization by new task.
-    def set_initialization(task)
+    def set_special_task(label, task)
       begin
-        @message.take([:initialize, nil, Symbol, nil], 0)
+        @message.take([label, nil, Symbol, nil], 0)
       rescue Rinda::RequestExpiredError
       end
-      @message.write(task.drb_args(:initialize))
+      @message.write(task.drb_args(label))
     end
+    private :set_special_task
 
+    # If the task has already set,
+    # the method overwrite old task of initialization by new task.
+    def set_initialization(task)
+      set_special_task(:initialize, task)
+    end
+
+    def set_finalization(task)
+      set_special_task(:finalization, task)
+    end
   end
 
 end

data/lib/drbqs/node_list.rb

@@ -26,10 +26,14 @@ module DRbQS
       @check = @list.keys
     end
 
+    def delete(id)
+      @list.delete(id)
+      @history.set(id, :disconnect)
+    end
+
     def delete_not_alive
       @check.each do |id|
-        @list.delete(id)
-        @history.set(id, :disconnect)
+        delete(id)
       end
       deleted = @check
       @check = []

data/lib/drbqs/queue.rb

@@ -72,19 +72,18 @@ module DRbQS
     end
     private :delete_task_id
 
-    def exec_task_hook(task_id, result)
+    def exec_task_hook(main_server, task_id, result)
       if task = @cache.delete(task_id)
         if hook = task.hook
           @history.set(task_id, :hook)
-          hook.call(self, result)
+          hook.call(main_server, result)
         end
       else
         @logger.error("Task #{task_id} is not cached.") if @logger
       end
     end
-    private :exec_task_hook
 
-    def get_result
+    def get_result(main_server)
       count = 0
       begin
         loop do
@@ -94,7 +93,7 @@ module DRbQS
           @history.set(task_id, :result, node_id)
           @logger.info("Get: result of #{task_id} from node #{node_id}.") if @logger
           delete_task_id(node_id, task_id)
-          exec_task_hook(task_id, result)
+          exec_task_hook(main_server, task_id, result)
         end
       rescue Rinda::RequestExpiredError
         @logger.debug("Get: #{count} results.") if @logger

data/lib/drbqs/server.rb

@@ -28,6 +28,7 @@ module DRbQS
   # empty_queue_hook is prior to task_generator.
   class Server
     WAIT_TIME_NODE_EXIT = 3
+    WAIT_TIME_NODE_FINALIZE = 10
     WAIT_TIME_NEW_RESULT = 1
 
     attr_reader :queue
@@ -46,6 +47,12 @@ module DRbQS
     #   Exit programs in finish_hook.
     # :signal_trap
     #   Set trapping signal.
+    # :scp_user
+    #   Set user of scp.
+    # :scp_host
+    #   Set host of scp.
+    # :file_directory
+    #   Set the setting of file directory.
     def initialize(opts = {})
       @port = opts[:port] || ROOT_DEFAULT_PORT
       @acl = acl_init(opts[:acl])
@@ -55,20 +62,30 @@ module DRbQS
         :result => Rinda::TupleSpace.new,
         :transfer => nil
       }
-      if opts[:log_file]
-        @logger = Logger.new(opts[:log_file])
-        @logger.level = opts[:log_level] || Logger::ERROR
-      else
-        @logger = nil
-      end
+      @logger = DRbQS::Utils.create_logger(opts[:log_file], opts[:log_level])
       @message = MessageServer.new(@ts[:message], @logger)
       @queue= QueueServer.new(@ts[:queue], @ts[:result], @logger)
       @check_alive = CheckAlive.new(opts[:check_alive])
       @task_generator = []
       hook_init(opts[:finish_exit])
       set_signal_trap if opts[:signal_trap]
+      @finalization_task = nil
+      @transfer_setting = get_transfer_setting(opts[:scp_host], opts[:scp_user], opts[:file_directory])
+    end
+
+    def transfer_directory
+      @ts[:transfer] && @ts[:transfer].directory
     end
 
+    def get_transfer_setting(host, user, directory)
+      setting = { :directory => directory, :user => user, :host => host, :set => true }
+      if host || user || directory
+        setting[:set] = true
+      end
+      setting
+    end
+    private :get_transfer_setting
+
     def acl_init(acl_arg)
       case acl_arg
       when Array
@@ -88,6 +105,9 @@ module DRbQS
     private :hook_init
 
     def start
+      if @transfer_setting[:set] && @transfer_setting[:directory] && !@ts[:transfer]
+        set_file_transfer(@transfer_setting[:directory])
+      end
       DRb.install_acl(@acl) if @acl
       uri = "druby://:#{@port}"
       DRb.start_service(uri, @ts)
@@ -128,6 +148,10 @@ module DRbQS
       @message.set_initialization(task)
     end
 
+    def set_finalization_task(task)
+      @finalization_task = task
+    end
+
     # +key+ is :empty_queue or :finish_exit.
     # &block takes self as an argument.
     def add_hook(key, name = nil, &block)
@@ -138,6 +162,12 @@ module DRbQS
       @hook.delete(key, name)
     end
 
+    def exec_finish_hook
+      @logger.info("Execute finish hook.") if @logger
+      @hook.exec(:finish, self)
+    end
+    private :exec_finish_hook
+    
     def exec_hook
       if @queue.empty?
         @logger.info("Execute empty queue hook.") if @logger
@@ -145,16 +175,21 @@ module DRbQS
       end
       add_tasks_from_generator
       if @queue.finished?
-        @logger.info("Execute finish hook.") if @logger
-        @hook.exec(:finish, self)
+        exec_finish_hook
       end
     end
     private :exec_hook
 
     def exit
-      @message.send_exit
+      if @finalization_task
+        @message.send_finalization(@finalization_task)
+        wait_time = WAIT_TIME_NODE_FINALIZE
+      else
+        @message.send_exit
+        wait_time = WAIT_TIME_NODE_EXIT
+      end
       until @message.node_not_exist?
-        sleep(WAIT_TIME_NODE_EXIT)
+        sleep(wait_time)
         check_connection(true)
       end
       @logger.info("History of tasks") { "\n" + @queue.all_logs } if @logger
@@ -167,17 +202,24 @@ module DRbQS
       end
     end
 
-    def set_file_transfer(user, host, directory)
+    def set_file_transfer(directory, opts = {})
+      user = opts[:user] || @transfer_setting[:user] || ENV['USER']
+      host = opts[:host] || @transfer_setting[:host] || 'localhost'
       @ts[:transfer] = DRbQS::Transfer.new(user, host, directory)
+      @logger.info("File transfer") { @ts[:transfer].information } if @logger
     end
 
     def check_message
-      while mes = @message.get_message
+      while mes_arg = @message.get_message
+        mes, arg = mes_arg
         case mes
         when :exit_server
           self.exit
         when :request_status
           @message.send_status(@queue.calculating)
+        when :node_error
+          @queue.get_accept_signal
+          @queue.requeue_for_deleted_node_id([arg])
         end
       end
     end
@@ -193,7 +235,7 @@ module DRbQS
       loop do
         check_message
         check_connection
-        count_results = @queue.get_result
+        count_results = @queue.get_result(self)
         exec_hook
         @logger.debug("Calculating tasks: #{@queue.calculating_task_number}") if @logger
         if count_results <= 1
@@ -202,25 +244,48 @@ module DRbQS
       end
     end
 
+    def start_profile
+      require 'ruby-prof'
+      RubyProf.start
+    end
+    private :start_profile
+
+    def finish_profile
+      result = RubyProf.stop
+      printer = RubyProf::FlatPrinter.new(result)
+      # printer = RubyProf::GraphPrinter.new(result)
+      # printer = RubyProf::CallTreePrinter.new(result)
+      printer.print(STDOUT)
+    end
+    private :finish_profile
+
     def test_exec(opts = {})
       task_generator_init
       dummy_client = DRbQS::Client.new(nil, :log_file => $stdout, :log_level => opts[:log_level])
       dummy_task_client = DRbQS::TaskClient.new(nil, @ts[:queue], nil)
+      if @ts[:transfer]
+        dummy_client.instance_variable_set(:@transfer, DRbQS::TransferTest.new(@ts[:transfer].directory))
+      end
       num = 0
+      start_profile if opts[:profile]
       loop do
-        num += 1
         exec_hook
         if ary = dummy_task_client.get_task
           task_id, marshal_obj, method_sym, args = ary
           result = dummy_client.instance_eval { execute_task(marshal_obj, method_sym, args) }
-          @queue.instance_eval do
-            exec_task_hook(task_id, result)
-          end
+          @queue.exec_task_hook(self, task_id, result)
         end
-        if opts[:limit] && num > opts[:limit]
+        num += 1
+        if opts[:limit] && num >= opts[:limit]
           break
         end
       end
+      finish_profile if opts[:profile]
+      if @finalization_task
+        args = @finalization_task.drb_args(nil)[1..-1]
+        dummy_client.instance_eval { execute_task(*args) }
+      end
+      exec_finish_hook
     end
 
     def test_task_generator(opts = {})

data/lib/drbqs/server_define.rb

@@ -57,14 +57,14 @@ HELP
       server.wait
     end
 
-    def test_server(options, type, arg = [])
+    def test_server(options, type, arg = [], test_opts = {})
       server = create_server(options)
       case type
       when :task
         puts "*** Test of Task Generators ***"
         server.test_task_generator(:limit => arg[0] ? arg[0].to_i : nil, :progress => true)
       when :exec
-        server.test_exec(:limit => arg[0] ? arg[0].to_i : nil)
+        server.test_exec(:limit => arg[0] ? arg[0].to_i : nil, :profile => test_opts[:profile])
       else
         puts "*** Not be yet implemented ***"
       end

data/lib/drbqs/ssh/shell.rb

@@ -129,7 +129,7 @@ module DRbQS
       end
       execute_command do |sh|
         if @nohup
-          pr, path = shell_exec_check(sh, "drbqs-manage new-filename #{@nohup_output}")
+          pr, path = shell_exec_check(sh, "filename-create new -p middle -D parent -t time #{@nohup_output}")
           cmd = "nohup #{cmd} > #{path.strip} 2>&1 &"
         end
         shell_exec(sh, cmd)

data/lib/drbqs/ssh/transfer.rb

@@ -1,11 +1,10 @@
-require 'fileutils'
-
 module DRbQS
 
   # Transfer files to directory on DRbQS server.
   # In this class we use scp command.
   # Note that after we transfer files we delete the files.
   class Transfer
+    attr_reader :user, :host, :directory
 
     # options
     #   :mkdir    true or nil
@@ -16,17 +15,43 @@ module DRbQS
       FileUtils.mkdir_p(@directory)
     end
 
+    def transfer_file(path, name)
+      system("scp -r #{path} #{@user}@#{@host}:#{File.join(@directory, name)} > /dev/null 2>&1")
+    end
+    private :transfer_file
+
     def scp(path)
       name = File.basename(path)
       unless File.exist?(path)
         raise ArgumentError, "File #{path} does not exist."
       end
-      if system("scp -r #{path} #{@user}@#{@host}:#{File.join(@directory, name)} > /dev/null 2>&1")
+      if transfer_file(path, name)
         FileUtils.rm_r(path)
         return true
       end
       return false
     end
+
+    def information
+      "#{@user}@#{@host} #{@directory}"
+    end
+  end
+
+  class TransferTest < Transfer
+    def initialize(directory)
+      @directory = File.expand_path(directory)
+      FileUtils.mkdir_p(@directory)
+    end
+
+    def transfer_file(path, name)
+      FileUtils.cp(path, File.join(@directory, name))
+      true
+    end
+    private :transfer_file
+
+    def information
+      @directory
+    end
   end
 
   # To compress files, we use gzip and tar command.
@@ -49,10 +74,12 @@ module DRbQS
           raise "Can not compress: #{path}"
         end
         FileUtils.rm_r(path) if File.exist?(path)
-        @@files.enq(gz_path)
+        path_to_send = gz_path
       else
-        @@files.enq(path)
+        path_to_send = path
       end
+      @@files.enq(path_to_send)
+      File.basename(path_to_send)
     end
 
     def self.compress_enqueue(path)
@@ -66,5 +93,21 @@ module DRbQS
     def self.empty?
       @@files.empty?
     end
+
+    def self.decompress(server, filename)
+      dir = server.transfer_directory
+      path = File.join(dir, filename)
+      if File.exist?(path)
+        case path
+        when /\.tar\.gz$/
+          cmd = "tar xvzf #{path} -C #{dir} > /dev/null 2>&1"
+        when /\.gz$/
+          cmd = "gunzip #{path} > /dev/null 2>&1"
+        else
+          cmd = nil
+        end
+        system(cmd) if cmd
+      end
+    end
   end
 end

data/lib/drbqs/task_client.rb

@@ -30,8 +30,8 @@ module DRbQS
     private :dequeue_result
 
     def queue_task(task_id, ary)
-      @task_queue.enq(ary)
       @calculating_task = task_id
+      @task_queue.enq(ary)
     end
 
     def dequeue_task
@@ -50,9 +50,9 @@ module DRbQS
       unless @calculating_task
         if ary = get_task
           task_id, obj, method_sym, args = ary
+          @logger.info("Send accept signal: node #{@node_id} caluclating #{task_id}") if @logger
+          @result.write([:accept, task_id, @node_id])
           queue_task(task_id, [obj, method_sym, args])
-          @logger.info("Send accept signal: node #{@node_id} caluclating #{@calculating_task}") if @logger
-          @result.write([:accept, @calculating_task, @node_id])
         end
       end
     end

data/lib/drbqs/utils.rb

@@ -0,0 +1,19 @@
+module DRbQS
+  module Utils
+    def create_logger(log_file, log_level)
+      if log_file
+        if IO === log_file
+          log_output = log_file
+        else
+          log_output = FileName.create(log_file, :position => :middle, :directory => :parent, :type => :number)
+        end
+        logger = Logger.new(log_output)
+        logger.level = log_level || Logger::ERROR
+        return logger
+      end
+      return nil
+    end
+    module_function :create_logger
+  end
+
+end

data/lib/drbqs.rb

@@ -1,5 +1,4 @@
 require 'thread'
-require 'logger'
 require 'drb'
 require 'drb/acl'
 require 'rinda/tuplespace'
@@ -7,6 +6,12 @@ require 'rinda/rinda'
 
 require 'drbqs/server_define'
 
+autoload :Logger, 'logger'
+autoload :FileUtils, 'fileutils'
+
+gem 'filename'
+autoload :FileName, 'filename'
+
 module DRbQS
   autoload :Server, 'drbqs/server'
   autoload :Client, 'drbqs/client'
@@ -18,9 +23,9 @@ module DRbQS
   autoload :SSHHost, 'drbqs/ssh/host'
   autoload :CommandTask, 'drbqs/task'
   autoload :CommandExecute, 'drbqs/task'
-  autoload :FileName, 'drbqs/utils/filename'
   autoload :Transfer, 'drbqs/ssh/transfer'
   autoload :FileTransfer, 'drbqs/ssh/transfer'
+  autoload :Utils, 'drbqs/utils'
 
   ROOT_DEFAULT_PORT = 13500
 end

data/spec/connection_spec.rb

@@ -26,6 +26,16 @@ describe DRbQS::ConnectionClient do
     @connection.get_initialization.should == ary[1..-1]
   end
 
+  it "should get no finalization method." do
+    @connection.get_finalization.should be_nil
+  end
+
+  it "should get finalization" do
+    ary = [:finalize, [1, 2], :size, []]
+    @message.write(ary)
+    @connection.get_finalization.should == ary[1..-1]
+  end
+
   it "should respond :alive_p signal" do
     @message.write([@node_id, :alive_p])
     @connection.respond_signal

data/spec/server_spec.rb

@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe DRbQS::Server do
-  context "when we setup ACL objects" do
+  context "when we initialize DRbQS::Server" do
     it "should initialize an ACL object by ACLFile.load" do
       path = File.dirname(__FILE__) + '/data/acl.txt'
       DRbQS::ACLFile.should_receive(:load).with(path)
@@ -14,4 +14,43 @@ describe DRbQS::Server do
       DRbQS::Server.new(:acl => ary, :log_file => nil)
     end
   end
+
+  context "when we start DRbQS::Server" do
+    it "should not set DRbQS::FileTransfer" do
+      server = DRbQS::Server.new
+      DRbQS::Transfer.should_not_receive(:new)
+      DRb.should_receive(:start_service).once
+      server.start
+    end
+
+    it "should set defalt settings of DRbQS::FileTransfer" do
+      server = DRbQS::Server.new(:file_directory => '/tmp')
+      DRbQS::Transfer.should_receive(:new).with(ENV['USER'], 'localhost', '/tmp')
+      DRb.should_receive(:start_service).once
+      server.start
+    end
+
+    it "should set DRbQS::FileTransfer" do
+      server = DRbQS::Server.new(:file_directory => '/tmp', :scp_user => 'hello', :scp_host => 'example.com')
+      DRbQS::Transfer.should_receive(:new).with('hello', 'example.com', '/tmp')
+      DRb.should_receive(:start_service).once
+      server.start
+    end
+
+    it "should set DRbQS::FileTransfer by DRbQS::Server#set_file_transfer" do
+      server = DRbQS::Server.new
+      DRbQS::Transfer.should_receive(:new).with(ENV['USER'], 'localhost', '/tmp')
+      DRb.should_receive(:start_service).once
+      server.set_file_transfer('/tmp')
+      server.start
+    end
+
+    it "should set DRbQS::FileTransfer by DRbQS::Server#set_file_transfer with optional arguments" do
+      server = DRbQS::Server.new
+      DRbQS::Transfer.should_receive(:new).with('hello', 'example.com', '/tmp')
+      DRb.should_receive(:start_service).once
+      server.set_file_transfer('/tmp', :user => 'hello', :host => 'example.com')
+      server.start
+    end
+  end
 end

metadata

@@ -2,7 +2,7 @@
 name: drbqs
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors: 
 - Takayuki YAMAGUCHI
@@ -10,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-25 00:00:00 +09:00
-default_executable: 
+date: 2011-04-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
@@ -91,8 +90,19 @@ dependencies:
   prerelease: false
   version_requirements: *id007
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: filename
   requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -100,7 +110,7 @@ dependencies:
         version: 2.5.0
   type: :development
   prerelease: false
-  version_requirements: *id008
+  version_requirements: *id009
 description: Task queuing system over network that is implemented by dRuby.
 email: d@ytak.info
 executables: 
@@ -129,6 +139,8 @@ files:
 - example/drbqs-manage-test.rb
 - example/drbqs-node-test.rb
 - example/drbqs-server-test.rb
+- example/error/error.rb
+- example/error/server_def.rb
 - example/server/server.rb
 - example/sum/server_def.rb
 - example/sum/sum.rb
@@ -155,12 +167,11 @@ files:
 - lib/drbqs/task.rb
 - lib/drbqs/task_client.rb
 - lib/drbqs/task_generator.rb
-- lib/drbqs/utils/filename.rb
+- lib/drbqs/utils.rb
 - spec/acl_file_spec.rb
 - spec/config_spec.rb
 - spec/connection_spec.rb
 - spec/data/acl.txt
-- spec/filename_spec.rb
 - spec/history_spec.rb
 - spec/manage_spec.rb
 - spec/message_spec.rb
@@ -179,7 +190,6 @@ files:
 - spec/test1_spec.rb
 - spec/test2_spec.rb
 - spec/transfer_spec.rb
-has_rdoc: true
 homepage: http://github.com/ytaka/drbqs
 licenses: 
 - GPL3
@@ -193,7 +203,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 4286900576171591504
+      hash: -783083245369623093
       segments: 
       - 0
       version: "0"
@@ -206,7 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.5.2
+rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
 summary: dRuby Queueing System
@@ -214,7 +224,6 @@ test_files:
 - spec/acl_file_spec.rb
 - spec/config_spec.rb
 - spec/connection_spec.rb
-- spec/filename_spec.rb
 - spec/history_spec.rb
 - spec/manage_spec.rb
 - spec/message_spec.rb

data/lib/drbqs/utils/filename.rb

@@ -1,118 +0,0 @@
-require 'fileutils'
-
-module DRbQS
-  class FileName
-
-    # The options are following:
-    # * :start
-    #     Fixnum
-    #     If ID string type is number, the ID starts from the specified number.
-    # 
-    # * :digit
-    #     Fixnum
-    #     ID number converted to a string with specified digit.
-    # 
-    # * :delimiter
-    #     String
-    # 
-    # * :type
-    #     :number or :time
-    #     If the value is :number, use a number for ID string.
-    #     If the value is :time, use current time.
-    # 
-    # * :position
-    #     :prefix, :suffix, or :middle
-    #     Set the position of an ID string.
-    def initialize(basepath, opts = {})
-      @basepath = File.expand_path(basepath)
-      @number = opts[:start] || 0
-      @digit = opts[:digit] || 2
-      @type = opts[:type] || :number
-      @position = opts[:position] || :suffix
-      @delimiter = opts[:delimiter] || (@position == :suffix ? '.' : '_')
-    end
-
-    def get_basepath(extension = nil)
-      if extension
-        extension = '.' + extension unless extension[0] == '.'
-        oldext = File.extname(@basepath)
-        if oldext.size > 0
-          @basepath.sub(Regexp.new("\\#{oldext}$"), extension)
-        else
-          @basepath + extension
-        end
-      else
-        @basepath
-      end
-    end
-    private :get_basepath
-
-    def get_addition(add, filename)
-      if add != :prohibit && (add == :always || File.exist?(filename))
-        case @type
-        when :time
-          t = Time.now
-          return t.strftime("%Y%m%d_%H%M%S_") + sprintf("%06d", t.usec)
-        when :number
-          s = sprintf("%0#{@digit}d", @number)
-          @number += 1
-          return s
-        else
-          raise "Invalid type of addition."
-        end
-      end
-      nil
-    end
-    private :get_addition
-
-    def add_addition(filename, addition)
-      case @position
-      when :prefix
-        dir, base = File.split(filename)
-        dir + '/' + addition + @delimiter + base
-      when :middle
-        dir, base = File.split(filename)
-        ext = File.extname(base)
-        if ext.size > 0
-          filename.sub(Regexp.new("\\#{ext}$"), @delimiter + addition + ext)
-        else
-          filename + @delimiter + addition
-        end
-      else # :suffix
-        filename + @delimiter + addition
-      end
-    end
-    private :add_addition
-
-    # The options are following:
-    # * :extension
-    #     String of extension
-    #     Use the extension if the value is specified.
-    # 
-    # * :add
-    #     :always    Always add an ID string.
-    #     :auto      If some file exists, add an ID string.
-    #     :prohibit  Even if some file exists, add no ID string.
-    # 
-    # * :directory
-    #     If the value is true and the parent directory does not exist,
-    #     create the directory.
-    def create(opts = {})
-      base = get_basepath(opts[:extension])
-      FileUtils.mkdir_p(File.dirname(base)) if opts[:directory]
-      if addition = get_addition(opts[:add], base)
-        path = add_addition(base, addition)
-        while File.exist?(path)
-          if addition = get_addition(opts[:add], base)
-            path = add_addition(base, addition)
-          else
-            raise "Can not create new filename."
-          end
-        end
-        path
-      else
-        base
-      end
-    end
-  end
-end

data/spec/filename_spec.rb

@@ -1,44 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-
-describe DRbQS::FileName do
-  it "should return unchanged filename" do
-    filename = DRbQS::FileName.new("abc.txt")
-    filename.create.should == File.expand_path(File.dirname('.') + '/abc.txt')
-  end
-
-  it "should return new filename with number" do
-    filename = DRbQS::FileName.new(__FILE__)
-    path = filename.create
-    File.exist?(path).should_not be_true
-    path.should match(/\.\d+$/)
-  end
-
-  it "should return new filename with time" do
-    filename = DRbQS::FileName.new(__FILE__, :type => :time)
-    path = filename.create
-    File.exist?(path).should_not be_true
-    path.should match(/\.[\d_]+$/)
-  end
-
-  it "should return same filename" do
-    filename = DRbQS::FileName.new(__FILE__)
-    path = filename.create(:add => :prohibit)
-    path.should == File.expand_path(__FILE__)
-  end
-
-  it "should return filename with prefix" do
-    filename = DRbQS::FileName.new(__FILE__, :position => :prefix)
-    path = filename.create
-    dir, name = File.split(path)
-    name.should match(/^\d+_/)
-  end
-
-  it "should return filename with addition before extension" do
-    filename = DRbQS::FileName.new(__FILE__, :position => :middle)
-    path = filename.create
-    dir, name = File.split(path)
-    ext = File.extname(name)
-    name.should match(Regexp.new("_\\d+\\#{ext}"))
-  end
-
-end