drbqs 0.0.5 → 0.0.6

data/README.rdoc

@@ -30,7 +30,7 @@ We make server.rb as the following.
 
  require_relative 'sum.rb'
  
- DRbQS.define_server do |server|
+ DRbQS.define_server do |server, argv, opts|
    10.step(100, 10) do |i|
      task = DRbQS::Task.new(Sum.new(i - 10, i), :exec)
      server.queue.add(task)
@@ -47,13 +47,15 @@ In terminal, we load server.rb and execute server of drbqs.
 
 === Start node and connect server
 
-We type in terminal.
+Because nodes needs class Sum,
+the nodes load sum.rb when they starts.
+Then, we type in terminal.
 
- drbqs-node druby://localhost:13500/
+ drbqs-node druby://localhost:13500/ -l sum.rb
 
 To use two cpu cores we execute two processes by the following.
 
- drbqs-node 2 druby://localhost:13500/
+ drbqs-node 2 druby://localhost:13500/ -l sum.rb
 
 Then, if it succeeds, the calculation starts.
 If it finishes, the server and node ends.

data/VERSION

@@ -1 +1 @@
-0.0.5
+0.0.6

data/drbqs.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{drbqs}
-  s.version = "0.0.5"
+  s.version = "0.0.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Takayuki YAMAGUCHI"]
-  s.date = %q{2011-03-05}
+  s.date = %q{2011-03-08}
   s.description = %q{Task queuing system over network that is implemented by dRuby.}
   s.email = %q{d@ytak.info}
   s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server"]
@@ -41,29 +41,55 @@ Gem::Specification.new do |s|
     "lib/drbqs/connection.rb",
     "lib/drbqs/manage.rb",
     "lib/drbqs/message.rb",
+    "lib/drbqs/node_list.rb",
     "lib/drbqs/queue.rb",
     "lib/drbqs/server.rb",
     "lib/drbqs/server_define.rb",
+    "lib/drbqs/task.rb",
     "lib/drbqs/task_client.rb",
+    "lib/drbqs/task_generator.rb",
     "spec/acl_file_spec.rb",
+    "spec/connection_spec.rb",
     "spec/data/acl.txt",
-    "spec/drbqs_spec.rb",
+    "spec/manage_spec.rb",
+    "spec/message_spec.rb",
+    "spec/node_list_spec.rb",
+    "spec/queue_spec.rb",
+    "spec/server_define_spec.rb",
     "spec/server_spec.rb",
-    "spec/spec_helper.rb"
+    "spec/spec_helper.rb",
+    "spec/task_client_spec.rb",
+    "spec/task_generator_spec.rb",
+    "spec/task_spec.rb",
+    "spec/test/test1.rb",
+    "spec/test1_spec.rb",
+    "spec/test2_spec.rb"
   ]
   s.homepage = %q{http://github.com/ytaka/drbqs}
   s.licenses = ["GPL3"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.5.2}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{dRuby Queueing System}
   s.test_files = [
     "spec/acl_file_spec.rb",
-    "spec/drbqs_spec.rb",
+    "spec/connection_spec.rb",
+    "spec/manage_spec.rb",
+    "spec/message_spec.rb",
+    "spec/node_list_spec.rb",
+    "spec/queue_spec.rb",
+    "spec/server_define_spec.rb",
     "spec/server_spec.rb",
-    "spec/spec_helper.rb"
+    "spec/spec_helper.rb",
+    "spec/task_client_spec.rb",
+    "spec/task_generator_spec.rb",
+    "spec/task_spec.rb",
+    "spec/test/test1.rb",
+    "spec/test1_spec.rb",
+    "spec/test2_spec.rb"
   ]
 
   if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/lib/drbqs/client.rb

@@ -4,12 +4,18 @@ require 'drbqs/task_client'
 module DRbQS
 
   class Client
+
+    WAIT_NEW_TASK = 1
+    OUTPUT_NOT_SEND_RESULT = 'not_send_result'
+
+    # :continue
     def initialize(access_uri, opts = {})
       @access_uri = access_uri
       @logger = Logger.new(opts[:log_file] || 'drbqs_client.log')
       @logger.level = opts[:log_level] || Logger::ERROR
       @connection = nil
       @task_client = nil
+      @process_continue = opts[:continue]
     end
 
     def execute_task(marshal_obj, method_sym, args)
@@ -28,21 +34,55 @@ module DRbQS
       end
     end
 
-    WAIT_NEW_TASK = 1
+    def dump_not_send_result_to_file
+      if data = @task_client.dump_result_queue
+        path = OUTPUT_NOT_SEND_RESULT + Time.now.to_i.to_s + '.dat'
+        open(path, 'w') { |f| f.print data }
+      end
+    end
+    private :dump_not_send_result_to_file
+
+    def output_error(err)
+      if @logger
+        @logger.error("Raise error in calculating thread: #{err.to_s}") { "\n" + err.backtrace.join("\n") }
+      end
+    end
+    private :output_error
+
+    def process_exit
+      dump_not_send_result_to_file
+      unless @process_continue
+        Kernel.exit
+      end
+    end
+    private :process_exit
 
-    def calculate
+    def calculate(opts = {})
       cn = Thread.new do
-        loop do
-          @task_client.add_new_task
-          @connection.respond_alive_signal
-          @task_client.send_result
-          sleep(WAIT_NEW_TASK)
+        begin
+          loop do
+            @task_client.add_new_task
+            if @connection.respond_alive_signal == :exit
+              break
+            end
+            @task_client.send_result
+            sleep(WAIT_NEW_TASK)
+          end
+        rescue => err
+          output_error(err)
+        ensure
+          process_exit
         end
       end
       exec = Thread.new do
-        loop do
-          marshal_obj, method_sym, args = @task_client.get
-          @task_client.transmit(execute_task(marshal_obj, method_sym, args))
+        begin
+          loop do
+            marshal_obj, method_sym, args = @task_client.dequeue_task
+            @task_client.queue_result(execute_task(marshal_obj, method_sym, args))
+          end
+        rescue => err
+          output_error(err)
+          process_exit
         end
       end
       cn.join

data/lib/drbqs/connection.rb

@@ -6,21 +6,23 @@ module DRbQS
     def initialize(message, logger = nil)
       @message = message
       @logger = logger
-      @id = nil
+      @id_number = nil
+      @id_string = create_id_string
     end
 
     def create_id_string
       t = Time.now
-      sprintf("#{Socket.gethostname} %d%d%d", t.to_i, t.usec, rand(1000))
+      sprintf("%d%d%d:#{Socket.gethostname}", t.to_i, t.usec, rand(1000))
     end
     private :create_id_string
 
     def get_id
-      s = create_id_string
-      @message.write([:connect, s])
-      @id = @message.take([s, Fixnum])[1]
-      @logger.info("Get node id: #{@id}") if @logger
-      @id
+      unless @id_number
+        @message.write([:connect, @id_string])
+        @id_number = @message.take([@id_string, Fixnum])[1]
+        @logger.info("Get node id: #{@id_number}") if @logger
+      end
+      @id_number
     end
 
     def get_initialization
@@ -34,14 +36,14 @@ module DRbQS
 
     def respond_alive_signal
       begin
-        node_id, sym = @message.take([@id, Symbol], 0)
+        node_id, sym = @message.take([@id_number, Symbol], 0)
         case sym
         when :alive_p
-          @message.write([:alive, @id])
-          @logger.info("Send alive signal of node id #{@id}") if @logger
+          @message.write([:alive, @id_number])
+          @logger.info("Send alive signal of node id #{@id_number}") if @logger
         when :exit
           @logger.info("Get exit signal") if @logger
-          Kernel.exit
+          return :exit
         end
       rescue Rinda::RequestExpiredError
       end

data/lib/drbqs/message.rb

@@ -1,43 +1,6 @@
-module DRbQS
-  class NodeList
-    def initialize
-      @id = 0
-      @list = {}
-      @check = []
-    end
-
-    def get_new_id(id_str)
-      @id += 1
-      @list[@id] = id_str
-      @id
-    end
-
-    def each(&block)
-      @list.each(&block)
-    end
-
-    def set_check_connection
-      @check = @list.keys
-    end
-
-    def delete_not_alive
-      @check.each do |id|
-        @list.delete(id)
-      end
-      deleted = @check
-      @check = []
-      deleted
-    end
-
-    def set_alive(id)
-      @check.delete(id)
-    end
-
-    def empty?
-      @list.size == 0
-    end
-  end
+require 'drbqs/node_list'
 
+module DRbQS
   class MessageServer
     def initialize(message, logger = nil)
       @message = message
@@ -50,6 +13,7 @@ module DRbQS
         mes = @message.take([Symbol, nil], 0)
         manage_message(*mes)
       rescue Rinda::RequestExpiredError
+        nil
       end
     end
 
@@ -66,7 +30,8 @@ module DRbQS
         @logger.info("Get exit message from #{arg.to_s}") if @logger
         return :exit_server
       else
-        puts "Invalid message from #{arg.to_s}"
+        @logger.error("Invalid message from #{arg.to_s}") if @logger
+        return nil
       end
     end
     private :manage_message

data/lib/drbqs/node_list.rb

@@ -0,0 +1,41 @@
+module DRbQS
+  class NodeList
+    def initialize
+      @id = 0
+      @list = {}
+      @check = []
+    end
+
+    def get_new_id(id_str)
+      @id += 1
+      @list[@id] = id_str
+      @id
+    end
+
+    def each(&block)
+      @list.each(&block)
+    end
+
+    def set_check_connection
+      @check = @list.keys
+    end
+
+    def delete_not_alive
+      @check.each do |id|
+        @list.delete(id)
+      end
+      deleted = @check
+      @check = []
+      deleted
+    end
+
+    def set_alive(id)
+      @check.delete(id)
+    end
+
+    def empty?
+      @list.size == 0
+    end
+  end
+
+end

data/lib/drbqs/queue.rb

@@ -1,25 +1,4 @@
 module DRbQS
-  class Task
-    attr_reader :hook
-
-    def initialize(obj, method_sym, args = [], &hook)
-      begin
-        @marshal_obj = Marshal.dump(obj)
-      rescue
-        raise "Can not dump an instance of #{obj.class}."
-      end
-      unless Array === args
-        raise "Arguments of task must be an array."
-      end
-      @method_sym = method_sym.intern
-      @args = args
-      @hook = hook
-    end
-
-    def drb_args(task_id)
-      [task_id, @marshal_obj, @method_sym, @args]
-    end
-  end
 
   class QueueServer
 
@@ -38,11 +17,13 @@ module DRbQS
     private :queue_task
 
     # &hook take two arguments: a QueueServer object and a result of task.
+    # Return task ID (for debug).
     def add(task)
       @task_id += 1
       @logger.info("New task: #{@task_id}") if @logger
       @cache[@task_id] = task
       queue_task(@task_id)
+      @task_id
     end
 
     def get_accept_signal

data/lib/drbqs/server.rb

@@ -18,6 +18,8 @@ module DRbQS
     end
   end
 
+  # When we set both empty_queue_hook and task_generator,
+  # empty_queue_hook is prior to task_generator.
   class Server
     attr_reader :queue
 
@@ -43,6 +45,7 @@ module DRbQS
       @message = MessageServer.new(@ts[:message], @logger)
       @queue= QueueServer.new(@ts[:queue], @ts[:result], @logger)
       @check_alive = CheckAlive.new(opts[:check_alive])
+      @task_generator = []
       @empty_queue_hook = nil
     end
 
@@ -74,10 +77,31 @@ module DRbQS
     end
     private :check_connection
 
+    def set_task_generator(task_generator)
+      @task_generator << task_generator
+    end
+
+    def add_tasks_from_generator
+      if @task_generator.size > 0 && @queue.empty?
+        if tasks = @task_generator[0].new_tasks
+          tasks.each { |t| @queue.add(t) }
+          @logger.debug("Generator add #{tasks.size} tasks.") if @logger
+        else
+          @task_generator.delete_at(0)
+          @logger.info("Generator creates all tasks and then has been deleted.") if @logger
+          if @task_generator.size > 0
+            add_tasks_from_generator
+          end
+        end
+      end
+    end
+    private :add_tasks_from_generator
+
     def set_initialization_task(task)
       @message.set_initialization(task)
     end
 
+    # &block takes self as an argument.
     def set_empty_queue_hook(&block)
       if block_given?
         @empty_queue_hook = block
@@ -86,6 +110,7 @@ module DRbQS
       end
     end
 
+    # &block takes self as an argument.
     def set_finish_hook(&block)
       if block_given?
         @finish_hook = block
@@ -99,6 +124,7 @@ module DRbQS
         @logger.info("Execute empty queue hook.") if @logger
         @empty_queue_hook.call(self)
       end
+      add_tasks_from_generator
       if @finish_hook && @queue.finished?
         @logger.info("Execute finish hook.") if @logger
         @finish_hook.call(self)
@@ -132,6 +158,7 @@ module DRbQS
     private :check_message
 
     def wait
+      @task_generator.each { |tgen| tgen.init }
       loop do
         check_message
         check_connection

data/lib/drbqs/server_define.rb

@@ -44,7 +44,6 @@ HELP
       end
       server = DRbQS::Server.new(options)
       @server_create.call(server, @argv, @opts)
-      # @server_create.call(server)
       server.set_signal_trap
       server.start
       server.wait

data/lib/drbqs/task.rb

@@ -0,0 +1,38 @@
+module DRbQS
+
+  # The tasks defined by this class are sent to nodes and
+  # calculated by the nodes.
+  # After the node returns the result of calculation to server,
+  # the server execute the hook.
+  class Task
+    attr_reader :hook
+
+    # Nodes execute obj.method_sym(*args).
+    # Server executes &hook with a server instance and an array of result
+    # after the server accepts the results from nodes.
+    def initialize(obj, method_sym, args = [], &hook)
+      begin
+        @marshal_obj = Marshal.dump(obj)
+      rescue
+        raise "Can not dump an instance of #{obj.class}."
+      end
+      unless Array === args
+        raise "Arguments of task must be an array."
+      end
+      @method_sym = method_sym.intern
+      @args = args
+      @hook = hook
+    end
+
+    def drb_args(task_id)
+      [task_id, @marshal_obj, @method_sym, @args]
+    end
+
+    def same_target?(other)
+      @marshal_obj == other.instance_variable_get(:@marshal_obj) &&
+        @method_sym == other.instance_variable_get(:@method_sym) &&
+        @args == other.instance_variable_get(:@args)
+    end
+  end
+
+end

data/lib/drbqs/task_client.rb

@@ -1,5 +1,7 @@
 module DRbQS
   class TaskClient
+    attr_reader :node_id, :calculating_task
+
     def initialize(node_id, queue, result, logger = nil)
       @node_id = node_id
       @queue = queue
@@ -10,12 +12,33 @@ module DRbQS
       @logger = logger
     end
 
+    def task_empty?
+      @task_queue.empty?
+    end
+
+    def result_empty?
+      @result_queue.empty?
+    end
+
+    def dequeue_result
+      @result_queue.deq
+    end
+    private :dequeue_result
+
+    def queue_task(task_id, ary)
+      @task_queue.enq(ary)
+      @calculating_task = task_id
+    end
+
+    def dequeue_task
+      @task_queue.deq
+    end
+
     def add_new_task
       unless @calculating_task
         begin
           task_id, obj, method_sym, args = @queue.take([Fixnum, nil, Symbol, nil], 0)
-          @calculating_task = task_id
-          @task_queue.enq([obj, method_sym, args])
+          queue_task(task_id, [obj, method_sym, args])
           @logger.info("Send accept signal: node #{@node_id} caluclating #{@calculating_task}") if @logger
           @result.write([:accept, @calculating_task, @node_id])
         rescue Rinda::RequestExpiredError
@@ -24,22 +47,29 @@ module DRbQS
     end
 
     def send_result
-      if @result_queue.size > 0
-        result = @result_queue.deq
+      if !result_empty?
+        result = dequeue_result
         @logger.info("Send result: #{@calculating_task}") { result.inspect } if @logger
         @result.write([:result, @calculating_task, result])
         @calculating_task = nil
       end
     end
 
-    def get
-      @task_queue.deq
-    end
-
-    def transmit(result)
+    def queue_result(result)
       @result_queue.enq(result)
     end
 
+    def dump_result_queue
+      results = []
+      while !result_empty?
+        results << dequeue_result
+      end
+      if results.size > 0
+        Marshal.dump(results)
+      else
+        nil
+      end
+    end
   end
 
 end

data/lib/drbqs/task_generator.rb

@@ -0,0 +1,74 @@
+module DRbQS
+  class TaskGenerator
+    def initialize(data = {})
+      data.each do |key, val|
+        instance_variable_set("@#{key.to_s}", val)
+      end
+      @__fiber__ = nil
+      @__iterate__ = nil
+      @__fiber_init__ = nil
+    end
+
+    def have_next?
+      !!@__fiber__
+    end
+
+    def create_add_task(*args, &block)
+      Fiber.yield(DRbQS::Task.new(*args, &block))
+    end
+
+    def set(iterate = 1, &block)
+      @__iterate__ = iterate
+      @__fiber_init__ = lambda do
+        @__fiber__ = Fiber.new do
+          instance_eval(&block)
+          nil
+        end
+      end
+    end
+
+    def init
+      @__fiber_init__.call
+    end
+
+    # Return an array of new tasks.
+    def new_tasks
+      if @__fiber__
+        task_ary = []
+        @__iterate__.times do |i|
+          if task_new = @__fiber__.resume
+            case task_new
+            when DRbQS::Task
+              task_ary << task_new
+            when Array
+              task_ary.concat(task_new)
+            else
+              raise "Invalid type of new task."
+            end
+          else
+            @__fiber__ = nil
+            break
+          end
+        end
+        if task_ary.size > 0
+          return task_ary
+        end
+      end
+      nil
+    end
+
+    # Create all tasks for test and return true if all tasks created properly.
+    def debug_all_tasks(limit = nil)
+      i = 0
+      while ary = new_tasks
+        ary.each? do |t|
+          unless DRbQS::TaskGenerator === t
+            raise "Invalid task by #{i}th generation: #{t.inspect}"
+          end
+        end
+        i += 1
+      end
+      true
+    end
+  end
+end

data/lib/drbqs.rb

@@ -11,6 +11,8 @@ require 'drbqs/server_define'
 module DRbQS
   autoload :Server, 'drbqs/server'
   autoload :Client, 'drbqs/client'
+  autoload :Task, 'drbqs/task'
+  autoload :TaskGenerator, 'drbqs/task_generator'
   autoload :Manage, 'drbqs/manage'
 
   ROOT_DEFAULT_PORT = 13500