drbqs 0.0.13 → 0.0.14

data/lib/drbqs/message.rb

@@ -1,119 +0,0 @@
-require 'drbqs/node_list'
-
-module DRbQS
-  class MessageServer
-    include HistoryUtils
-
-    def initialize(message, logger = nil)
-      @message = message
-      @node_list = NodeList.new
-      @logger = logger
-    end
-
-    def get_message
-      begin
-        mes = @message.take([:server, Symbol, nil], 0)
-        manage_message(*mes[1..2])
-      rescue Rinda::RequestExpiredError
-        nil
-      end
-    end
-
-    def manage_message(mes, arg)
-      @logger.info("Get message") { [mes, arg] } if @logger
-      case mes
-      when :connect
-        a = [arg, @node_list.get_new_id(arg)]
-        @logger.info("New node") { a } if @logger
-        @message.write(a)
-      when :alive
-        @node_list.set_alive(arg)
-      when :exit_server
-        @logger.info("Get exit message from #{arg.to_s}") if @logger
-      when :request_status
-        @logger.info("Get status request from #{arg.to_s}") if @logger
-      when :node_error
-        @node_list.delete(arg[0])
-        @logger.info("Node Error (#{arg[0]})") { arg[1] } if @logger
-        return [mes, arg[0]]
-      else
-        @logger.error("Invalid message from #{arg.to_s}") if @logger
-        return nil
-      end
-      return [mes]
-    end
-    private :manage_message
-
-    def check_connection
-      deleted = @node_list.delete_not_alive
-      @logger.info("IDs of deleted nodes") { deleted } if deleted.size > 0 && @logger
-      @node_list.each do |id, str|
-        @message.write([id, :alive_p])
-      end
-      @node_list.set_check_connection
-      deleted
-    end
-
-    def send_signal_to_all_nodes(signal)
-      @node_list.each do |node_id, id_str|
-        @message.write([node_id, signal])
-      end
-    end
-    private :send_signal_to_all_nodes
-
-    def send_exit
-      send_signal_to_all_nodes(:exit)
-    end
-
-    def send_finalization(task)
-      set_finalization(task)
-      send_signal_to_all_nodes(:finalize)
-    end
-
-    def send_status(calculating_task_id)
-      s = ''
-      @node_list.history.each do |node_id, events|
-        if events.size == 0 || events.size > 2
-          raise "Invalid history of nodes: #{events.inspect}"
-        end
-        connect = events[0]
-        s << sprintf("%4d %s\t", node_id, connect[2])
-        if disconnect = events[1]
-          s << "disconnected: (#{time_to_string(connect[0])} - #{time_to_string(disconnect[0])})\n"
-        else
-          task_ids = calculating_task_id[node_id]
-          s << "task: #{task_ids.map { |num| num.to_s }.join(', ')} (#{time_to_string(connect[0])})\n"
-        end
-      end
-      begin
-        @message.take([:status, nil], 0)
-      rescue Rinda::RequestExpiredError
-      end
-      @message.write([:status, s])
-    end
-
-    def node_not_exist?
-      @node_list.empty?
-    end
-
-    def set_special_task(label, task)
-      begin
-        @message.take([label, nil, Symbol, nil], 0)
-      rescue Rinda::RequestExpiredError
-      end
-      @message.write(task.drb_args(label))
-    end
-    private :set_special_task
-
-    # If the task has already set,
-    # the method overwrite old task of initialization by new task.
-    def set_initialization(task)
-      set_special_task(:initialize, task)
-    end
-
-    def set_finalization(task)
-      set_special_task(:finalization, task)
-    end
-  end
-
-end

data/lib/drbqs/node_list.rb

@@ -1,52 +0,0 @@
-require 'drbqs/history'
-
-module DRbQS
-  class NodeList
-    attr_reader :history
-
-    def initialize
-      @id = 0
-      @list = {}
-      @check = []
-      @history = History.new
-    end
-
-    def get_new_id(id_str)
-      @id += 1
-      @list[@id] = id_str
-      @history.set(@id, :connect, @list[@id])
-      @id
-    end
-
-    def each(&block)
-      @list.each(&block)
-    end
-
-    def set_check_connection
-      @check = @list.keys
-    end
-
-    def delete(id)
-      @list.delete(id)
-      @history.set(id, :disconnect)
-    end
-
-    def delete_not_alive
-      @check.each do |id|
-        delete(id)
-      end
-      deleted = @check
-      @check = []
-      deleted
-    end
-
-    def set_alive(id)
-      @check.delete(id)
-    end
-
-    def empty?
-      @list.size == 0
-    end
-  end
-
-end

data/lib/drbqs/queue.rb

@@ -1,138 +0,0 @@
-require 'drbqs/history'
-
-module DRbQS
-
-  class QueueServer
-    include HistoryUtils
-
-    attr_reader :calculating, :history
-
-    def initialize(queue, result, logger = nil)
-      @queue = queue
-      @result = result
-      @task_id = 0
-      @cache = {}
-      @calculating = Hash.new { |hash, key| hash[key] = Array.new }
-      @history = DRbQS::History.new
-      @logger = logger
-    end
-
-    def queue_task(task_id)
-      @queue.write(@cache[task_id].drb_args(task_id))
-    end
-    private :queue_task
-
-    # &hook take two arguments: a QueueServer object and a result of task.
-    # Return task ID (for debug).
-    def add(task)
-      @task_id += 1
-      @logger.info("New task: #{@task_id}") if @logger
-      @cache[@task_id] = task
-      queue_task(@task_id)
-      @history.set(@task_id, :add)
-      @task_id
-    end
-
-    def get_accept_signal
-      count = 0
-      begin
-        loop do
-          sym, task_id, node_id = @result.take([:accept, Fixnum, Fixnum], 0)
-          count += 1
-          @calculating[node_id] << task_id
-          @history.set(task_id, :calculate, node_id)
-          @logger.info("Accept: task #{task_id} by node #{node_id}.") if @logger
-        end
-      rescue Rinda::RequestExpiredError
-        @logger.debug("Accept: #{count} signals.") if @logger
-      end
-      count
-    end
-
-    def requeue_for_deleted_node_id(deleted)
-      deleted.each do |node_id|
-        if task_id_ary = @calculating[node_id]
-          task_id_ary.each do |task_id|
-            queue_task(task_id)
-            @history.set(task_id, :requeue)
-            @logger.info("Requeue: task #{task_id}.") if @logger
-          end
-          @calculating.delete(node_id)
-        end
-      end
-    end
-
-    def delete_task_id(node_id, task_id)
-      unless @calculating[node_id].delete(task_id)
-        @logger.error("Task #{task_id} does not exist in list of calculating tasks.") if @logger
-      end
-      if ary = @calculating.find { |k, v| v.include?(task_id) }
-        @logger.error("Node #{ary[0]} is calculating task #{task_id}, too.") if @logger
-      end
-    end
-    private :delete_task_id
-
-    def exec_task_hook(main_server, task_id, result)
-      if task = @cache.delete(task_id)
-        if hook = task.hook
-          @history.set(task_id, :hook)
-          hook.call(main_server, result)
-        end
-      else
-        @logger.error("Task #{task_id} is not cached.") if @logger
-      end
-    end
-
-    def get_result(main_server)
-      count = 0
-      begin
-        loop do
-          get_accept_signal
-          sym, task_id, node_id, result = @result.take([:result, Fixnum, Fixnum, nil], 0)
-          count += 1
-          @history.set(task_id, :result, node_id)
-          @logger.info("Get: result of #{task_id} from node #{node_id}.") if @logger
-          delete_task_id(node_id, task_id)
-          exec_task_hook(main_server, task_id, result)
-        end
-      rescue Rinda::RequestExpiredError
-        @logger.debug("Get: #{count} results.") if @logger
-      end
-      count
-    end
-
-    def calculating_task_number
-      @calculating.inject(0) { |s, key_val| s + key_val[1].size }
-    end
-
-    # If queue is empty, return true. Otherwise, false.
-    # Even if there are calculating tasks,
-    # the method can return true.
-    def empty?
-      @cache.size - calculating_task_number == 0
-    end
-
-    # If there are no tasks in queue and calculating,
-    # return true. Otherwise, false.
-    def finished?
-      @cache.size == 0
-    end
-
-    def all_logs
-      s = ''
-      @history.each do |task_id, events|
-        s << "Task #{task_id}\n"
-        events.each do |ev|
-          case ev[1]
-          when :add, :requeue, :hook
-            s << "  #{time_to_string(ev[0])}\t#{ev[1]}\n"
-          when :calculate, :result
-            s << "  #{time_to_string(ev[0])}\t#{ev[1]} (node #{ev[2]})\n"
-          end
-        end
-      end
-      s
-    end
-  end
-
-end

data/lib/drbqs/server_hook.rb

@@ -1,67 +0,0 @@
-module DRbQS
-  class ServerHook
-    def initialize
-      @hook = Hash.new { |h, k| h[k] = Array.new }
-      @argument_number = {}
-      @finish_exit = nil
-      set_argument_number(:empty_queue, 1)
-      set_argument_number(:finish, 1)
-    end
-
-    def set_argument_number(key, num)
-      @argument_number[key] = num
-    end
-    private :set_argument_number
-
-    def create_proc_name(key)
-      name = "#{key.to_s}#{rand(1000)}"
-      if @hook.has_key?(name)
-        create_proc_name(key)
-      else
-        name
-      end
-    end
-    private :create_proc_name
-
-    def add(key, name = nil, &block)
-      if (n = @argument_number[key]) && (block.arity != n)
-        raise ArgumentError, "Invalid argument number of hook of #{key.inspect}."
-      end
-      name ||= create_proc_name(key)
-      @hook[key] << [name, block]
-      name
-    end
-
-    def delete(key, name = nil)
-      if name
-        @hook[key].delete_if { |ary| ary[0] == name }
-      else
-        @hook[key].clear
-      end
-    end
-
-    def specific_proc(key)
-      case key
-      when :finish
-        @finish_exit.call if @finish_exit
-      end
-    end
-    private :specific_proc
-
-    def hook_names(key)
-      @hook[key].map { |a| a[0] }
-    end
-
-    def exec(key, *args)
-      @hook[key].each do |ary|
-        ary[1].call(*args)
-      end
-      specific_proc(key)
-    end
-
-    def set_finish_exit(&block)
-      @finish_exit = block
-    end
-  end
-
-end

data/lib/drbqs/ssh/host.rb

@@ -1,26 +0,0 @@
-require 'yaml'
-require 'drbqs/config'
-
-module DRbQS
-  class SSHHost
-    def initialize
-      @dir = DRbQS::Config.get_host_file_directory
-      @host_files = (Dir.glob("#{@dir}/*.yaml") + Dir.glob("#{@dir}/*.yml")).map { |s| File.basename(s) }
-    end
-
-    def get(name)
-      if (name.size > 0) && (host = @host_files.find { |s| /^#{name}/ =~ s })
-        return File.join(@dir, host)
-      end
-      return nil
-    end
-    private :get
-
-    def get_options(name)
-      if path = get(name)
-        return [path, YAML.load_file(path)]
-      end
-      return [nil, {}]
-    end
-  end
-end

data/lib/drbqs/ssh/shell.rb

@@ -1,139 +0,0 @@
-require 'net/ssh'
-require 'net/ssh/shell'
-
-module DRbQS
-
-  # Requirements:
-  #   bash
-  #   nohup
-  class SSHShell
-    class GetInvalidExitStatus < StandardError
-    end
-
-    attr_reader :user, :host, :directory, :port
-
-    DEFAULT_RVM_SCRIPT = '$HOME/.rvm/scripts/rvm'
-    DEFAULT_OUTPUT_FILE = 'drbqs_nohup.log'
-
-    # :shell     shell to use
-    # :dir       base directory of ssh server
-    # :rvm       version of ruby on rvm
-    # :rvm_init  path of script to initialize rvm
-    # :output    file to output of stdout and stderr
-    def initialize(dest, opts = {})
-      @user, @host, @port = split_destination(dest)
-      if !(@host && @user)
-        raise "Invalid destination of ssh server."
-      end
-      @shell = opts[:shell] || 'bash'
-      @rvm = opts[:rvm]
-      @rvm_init = opts[:rvm_init]
-      if (@rvm || @rvm_init) && !(String === @rvm_init)
-        @rvm_init = DEFAULT_RVM_SCRIPT
-      end
-      @nohup_output = opts[:output] || DEFAULT_OUTPUT_FILE
-      @directory = opts[:dir]
-      @out = $stdout
-      @nice = opts[:nice]
-      @nohup = opts[:nohup]
-    end
-
-    def split_destination(dest)
-      if (n = dest.index("@")) && n > 0
-        user = dest[0..(n - 1)]
-        host_dir = dest[(n + 1)..-1]
-      else
-        raise "Not include '@': #{dest}"
-      end
-      if n = host_dir.index(':')
-        host = host_dir[0..(n - 1)]
-        port = host_dir[(n + 1)..-1]
-      else
-        host = host_dir
-        port = nil
-      end
-      [user && user.size > 0 ? user : nil,
-       host && host.size > 0 ? host : nil,
-       port && port.size > 0 ? port.to_i : nil]
-    end
-    private :split_destination
-
-    def output_command(cmd, result)
-      @out.puts "#{@user}@#{@host}$ #{cmd}" if @out
-      @out.print result
-    end
-    private :output_command
-
-    def shell_exec_get_output(sh, cmd)
-      result = ''
-      pr_cmd = sh.execute!(cmd) do |sh_proc|
-        sh_proc.on_output do |pr, data|
-          result << data
-        end
-        sh_proc.on_error_output do |pr, data|
-          result << data
-        end
-      end
-      [pr_cmd, result]
-    end
-    private :shell_exec_get_output
-
-    def shell_exec(sh, cmd)
-      ary = shell_exec_get_output(sh, cmd)
-      output_command(cmd, ary[1])
-      ary
-    end
-    private :shell_exec
-
-    def shell_exec_check(sh, cmd)
-      ary = shell_exec(sh, cmd)
-      if ary[0].exit_status != 0
-        raise GetInvalidExitStatus, "Can not execute '#{cmd}' on #{@host} properly."
-      end
-      ary
-    end
-    private :shell_exec_check
-
-    def execute_command(&block)
-      Net::SSH.start(@host, @user, :port => @port) do |ssh|
-        ssh.shell(@shell) do |sh|
-          shell_exec_check(sh, "cd #{@directory}") if @directory
-          shell_exec_check(sh, "source #{@rvm_init}") if @rvm_init
-          shell_exec_check(sh, "rvm use #{@rvm}") if @rvm
-          yield(sh)
-          shell_exec(sh, "exit")
-        end
-      end
-    end
-    private :execute_command
-
-    def get_environment
-      execute_command do |sh|
-        ['echo "directory: " `pwd`',
-         'echo "files:"',
-         'ls',
-         'if which rvm > /dev/null; then rvm info; else ruby -v; fi'].each do |cmd|
-          shell_exec(sh, cmd)
-        end
-      end
-    end
-
-    def start(*args)
-      cmd = args.join(' ')
-      if @nice
-        if Integer === @nice
-          cmd = "nice -n #{@nice.to_s} " + cmd
-        else
-          cmd = "nice " + cmd
-        end
-      end
-      execute_command do |sh|
-        if @nohup
-          pr, path = shell_exec_check(sh, "filename-create new -p middle -D parent -t time #{@nohup_output}")
-          cmd = "nohup #{cmd} > #{path.strip} 2>&1 &"
-        end
-        shell_exec(sh, cmd)
-      end
-    end
-  end
-end

data/lib/drbqs/task_client.rb

@@ -1,86 +0,0 @@
-module DRbQS
-  class TaskClient
-    attr_reader :node_id, :calculating_task
-
-    def initialize(node_id, queue, result, logger = nil)
-      @node_id = node_id
-      @queue = queue
-      @result = result
-      @calculating_task = nil
-      @task_queue = Queue.new
-      @result_queue = Queue.new
-      @logger = logger
-    end
-
-    def calculating?
-      !!@calculating_task
-    end
-
-    def task_empty?
-      @task_queue.empty?
-    end
-
-    def result_empty?
-      @result_queue.empty?
-    end
-
-    def dequeue_result
-      @result_queue.deq
-    end
-    private :dequeue_result
-
-    def queue_task(task_id, ary)
-      @calculating_task = task_id
-      @task_queue.enq(ary)
-    end
-
-    def dequeue_task
-      @task_queue.deq
-    end
-
-    def get_task
-      begin
-        @queue.take([Fixnum, nil, Symbol, nil], 0)
-      rescue Rinda::RequestExpiredError
-        nil
-      end
-    end
-
-    def add_new_task
-      unless @calculating_task
-        if ary = get_task
-          task_id, obj, method_sym, args = ary
-          @logger.info("Send accept signal: node #{@node_id} caluclating #{task_id}") if @logger
-          @result.write([:accept, task_id, @node_id])
-          queue_task(task_id, [obj, method_sym, args])
-        end
-      end
-    end
-
-    def send_result
-      if !result_empty?
-        result = dequeue_result
-        @logger.info("Send result: #{@calculating_task}") { result.inspect } if @logger
-        @result.write([:result, @calculating_task, @node_id, result])
-        @calculating_task = nil
-      end
-    end
-
-    def queue_result(result)
-      @result_queue.enq(result)
-    end
-
-    def dump_result_queue
-      results = []
-      while !result_empty?
-        results << dequeue_result
-      end
-      if results.size > 0
-        Marshal.dump(results)
-      else
-        nil
-      end
-    end
-  end
-
-end

data/lib/drbqs/utils.rb

@@ -1,19 +0,0 @@
-module DRbQS
-  module Utils
-    def create_logger(log_file, log_level)
-      if log_file
-        if IO === log_file
-          log_output = log_file
-        else
-          log_output = FileName.create(log_file, :position => :middle, :directory => :parent, :type => :number)
-        end
-        logger = Logger.new(log_output)
-        logger.level = log_level || Logger::ERROR
-        return logger
-      end
-      return nil
-    end
-    module_function :create_logger
-  end
-
-end

data/spec/acl_file_spec.rb

@@ -1,9 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-require 'drbqs/acl_file'
-
-describe DRbQS::ACLFile do
-  it "should return an ACL object" do
-    acl = DRbQS::ACLFile.load(File.dirname(__FILE__) + '/data/acl.txt')
-    acl.should be_an_instance_of ACL
-  end
-end

data/spec/config_spec.rb

@@ -1,14 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-
-describe DRbQS::Config do
-  it "should return nil" do
-    DRbQS::Config.set_directory(File.dirname(__FILE__) + '/not_exist_path')
-    DRbQS::Config.get_acl_file.should be_nil
-  end
-
-  it "should return existing path" do
-    path = File.dirname(__FILE__) + '/data'
-    DRbQS::Config.set_directory(path)
-    DRbQS::Config.get_acl_file.should == File.join(path, 'acl.txt')
-  end
-end