drbqs 0.0.19 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d97c7024904ace9423e4447cf220ee2c2b6a5b32
+  data.tar.gz: e2ea40c666d2033eebe43659f833ea032f05a720
+SHA512:
+  metadata.gz: 6cb2dfd423583c9426ce5138144dd46a37455c62208164c3ef26425578f991ce8e9e69998897170fb00e2cd13147846a89135d8a8048d6e7d48c413bd2944445
+  data.tar.gz: 069f933a4cdc05074e70e9a4081fe33e289a6f7959c9e9949f34f5e229bd4e968acb6ed9f594c4dc394923ee738eb79f033fe70231aa21a72abccff4f3492e9d

data/docs/TestProfiler.md

@@ -0,0 +1,30 @@
+# Test and profiler
+
+## Test
+
+When we are developing program,
+we want to execute tasks in single process and check errors.
+For that purpose we use the option --test.
+For example,
+
+    drbqs-server server.rb --test exec
+
+In order to execute only two tasks, we type
+
+    drbqs-server server.rb --test exec,2
+
+## Profiler
+
+At a time we can use profiler.
+
+    drbqs-server server.rb --test exec --profile
+
+Then drbqs creates drbqs_prof.txt that has profile data.
+
+To profile a program and show the result by kcachegrind,
+
+    drbqs-server server.rb --test exec --profile --profile-printer calltree
+
+and then
+
+    kcachegrind drbqs_prof.txt

data/drbqs.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # specify any dependencies here; for example:
-  s.add_development_dependency "rspec", ">= 2.6.0"
+  s.add_development_dependency "rspec", ">= 2.14.1"
   s.add_development_dependency "yard", ">= 0.7.2"
   s.add_runtime_dependency 'filename', '>= 0.1.0'
   s.add_runtime_dependency "user_config", ">= 0.0.2"
@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'net-ssh-shell', '>= 0.2.0'
   s.add_runtime_dependency "net-sftp", ">= 2.0.5"
   s.add_runtime_dependency "sys-proctable"
+  s.add_runtime_dependency "activesupport"
 end

data/{example → examples}/command/server_def.rb

@@ -7,7 +7,7 @@ DRbQS.define_server do |server, argv, opts|
   sleep_time = 2
   server.task_generator do |reg|
     3.times do |i|
-      reg.add(DRbQS::CommandTask.new(["sleep #{sleep_time}", 'echo hello world']))
+      reg.add(DRbQS::CommandTask.new("sleep #{sleep_time}", "echo 'hello world (#{i})'"))
     end
   end
 end

data/{example → examples}/error_task/server_def.rb

@@ -1,7 +1,7 @@
 require_relative 'error.rb'
 
 DRbQS.define_server(:check_alive => 300) do |server, argv, opts|
-  server.queue.add(DRbQS::Task.new(TaskError, :exec))
+  server.queue.add(DRbQS::Task.new(TaskError.new, :exec))
   server.add_hook(:finish) do |serv|
     serv.exit
   end

data/examples/execute/README.md

@@ -0,0 +1,7 @@
+## Usage
+
+    bin/drbqs-execute example/execute/execute.rb
+
+## Result
+
+Result is output in ../../example_execute.log

data/examples/execute/execute.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env drbqs-execute
+
+usage message: "Message of this file", server: File.join(File.dirname(__FILE__), 'server.rb')
+
+default server: :server, log: "/tmp/drbqs_log"
+
+server :server do |server|
+  server.load File.join(File.dirname(__FILE__), "server.rb")
+  server.log_level "debug"
+end

data/examples/execute/server.rb

@@ -0,0 +1,27 @@
+require_relative 'sum.rb'
+
+DRbQS.option_parser do |opt, hash|
+  opt.on('--step NUM', Integer) do |v|
+    hash[:step] = v
+  end
+end
+
+output_path = File.expand_path(File.join(File.dirname(__FILE__), "../../example_execute.log"))
+
+DRbQS.define_server(:check_alive => 5) do |server, argv, opts|
+  start_num = (argv[0] || 10).to_i
+  end_num = (argv[1] || 50).to_i
+  step_num = opts[:step] || 10
+  start_num.step(end_num, step_num) do |i|
+    task = DRbQS::Task.new(Sum.new(i - 10, i), :exec, :args => [], :note => "#{i-10} to #{i}") do |srv, ret|
+      open(output_path, "a+") do |f|
+        f.puts "Receive: #{ret.inspect}"
+      end
+    end
+    server.queue.add(task)
+  end
+
+  server.add_hook(:finish) do |serv|
+    serv.exit
+  end
+end

data/{example → examples}/group/execute.rb

@@ -1,8 +1,11 @@
-default :server => :server_local, :port => 13789, :log => '/tmp/drbqs_execute'
+default :server => :server_local, :port => 13789, :node => [:node_odd, :node_even], :log => '/tmp/drbqs_execute'
 
 current_dir = File.expand_path(File.dirname(__FILE__))
 
-usage :message => "Calculate sum of numbers", :server => File.join(current_dir, 'server.rb')
+usage :message => <<MES, :server => File.join(current_dir, 'server.rb')
+Calculate sum of numbers.
+Results are output to logs in /tmp/drbqs_execute
+MES
 
 server :server, "localhost" do |server|
   server.load File.expand_path(File.join(File.dirname(__FILE__), 'server.rb'))

data/{example → examples}/mandelbrot/execute.rb

@@ -2,7 +2,10 @@ DIR = File.dirname(__FILE__)
 
 current_dir = File.expand_path(File.dirname(__FILE__))
 
-usage :message => "Calculate Mandelbrot set", :server => File.join(current_dir, 'server.rb')
+usage :message => <<MES, :server => File.join(current_dir, 'server.rb')
+Calculate Mandelbrot set.
+Result is output to the directory "result_mandelbrot".
+MES
 
 server :local_server, "localhost" do |srv|
   srv.load File.join(DIR, 'server.rb')

data/examples/server/server.rb

@@ -0,0 +1,20 @@
+# 
+# Start server:
+# drbqs-server examples/server/server.rb
+# Start node:
+# drbqs-node bin/drbqs-node druby://:13501
+# Stop server:
+# drbqs-manage signal druby://:13501 server-exit
+# 
+
+server = DRbQS::Server.new(:port => 13501)
+server.task_generator do |reg|
+  5.times do |i|
+    reg.create_add(i, :to_s)
+  end
+end
+server.set_initialization_task(DRbQS::Task.new(Kernel, :puts, :args => ['hook: initialize']))
+server.set_finalization_task(DRbQS::Task.new(Kernel, :puts, :args => ['hook: finalize']))
+server.set_signal_trap
+server.start
+server.wait

data/{example → examples}/sum/server_def.rb

@@ -1,8 +1,12 @@
 # 
 # Usage:
+#  * Start only server
 #  drbqs-server server_def.rb -- 30 50
 #  drbqs-server server_def.rb -- 100 500 --step 100
 # 
+#  * Start server with 2 nodes
+#  drbqs-server examples/sum/server_def.rb --execute-node 2
+# 
 
 require_relative 'sum.rb'
 
@@ -17,7 +21,7 @@ DRbQS.define_server(:check_alive => 5) do |server, argv, opts|
   end_num = (argv[1] || 50).to_i
   step_num = opts[:step] || 10
   start_num.step(end_num, step_num) do |i|
-    task = DRbQS::Task.new(Sum.new(i - 10, i), :exec, [], "#{i-10} to #{i}") do |srv, ret|
+    task = DRbQS::Task.new(Sum.new(i - 10, i), :exec, :note => "#{i-10} to #{i}") do |srv, ret|
       puts "Receive: #{ret.inspect}"
     end
     server.queue.add(task)

data/examples/sum/sum.rb

@@ -0,0 +1,10 @@
+class Sum
+  def initialize(start_num, end_num)
+    @num = [start_num, end_num]
+  end
+
+  def exec
+    sleep(3) # Waiting time to interrupt
+    (@num[0]..@num[1]).inject(0) { |sum, i| sum += i }
+  end
+end

data/{example → examples}/sum2/server_def.rb

@@ -13,16 +13,15 @@ DRbQS.option_parser do |opt, hash|
 end
 
 DRbQS.define_server do |server, argv, opts|
-  start_num = (argv[0] || 10).to_i,
-  end_num = (argv[1] || 100).to_i,
+  start_num = (argv[0] || 10).to_i
+  end_num = (argv[1] || 100).to_i
   step_num = opts[:step] || 10
 
   server.task_generator do |reg|
-    start_num.step(end_num, step_num) do |i|
+    (start_num..end_num).step(step_num) do |i|
       reg.create_add(Sum.new(i - 10, i), :exec) do |srv, ret|
         puts "Receive: #{ret.inspect}"
       end
     end
   end
-  server.add_task_generator(tgen)
 end

data/lib/drbqs.rb

@@ -13,11 +13,16 @@ require 'drbqs/version'
 require 'drbqs/execute/server_define'
 require 'drbqs/utility/misc'
 
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/array/extract_options'
+
 module DRbQS
   autoload :Server, 'drbqs/server/server'
   autoload :Node, 'drbqs/node/node'
   autoload :Config, 'drbqs/config/config'
   autoload :Setting, 'drbqs/setting/setting'
+  autoload :Temporary, 'drbqs/utility/temporary'
+  autoload :CommandTask, 'drbqs/ext/task'
 
   ROOT_DEFAULT_PORT = 13500
 end

data/lib/drbqs/command_line/command_execute.rb

@@ -20,6 +20,7 @@ HELP
           set(:node, '-n STR', '--node STR', String, 'Set the comma sparated key of nodes.')
           set(:no_server, '--no-server', 'Not execute server.')
           set(:no_node, '--no-node', 'Not execute node.')
+          set(:wait_server_finish, "--wait-server-finish", "Wait finish of server process.")
           set(:information, '-i', '--information', 'Show information.')
           set(:help, '-h', '--help', 'Show this command help and usage of definition file.') do |opt|
             $stdout.print opt

data/lib/drbqs/command_line/command_server.rb

@@ -26,7 +26,7 @@ HELP
           set(:sftp_host, '--sftp-host HOST', String, 'Set the host of sftp destination.')
           set(:profile, '--profile', 'Use profile for test exec.')
           set(:profile_printer, '--profile-printer PRINTER', String,
-              'Set the printer type for profile. The value is :flat, :graph, :graphhtml, or :calltree.')
+              'Set the printer type for profile. The value is flat, graph, graphhtml, or calltree.')
           set(:test, '--test STR', String, 'Execute test.')
           set(:execute_node, '--execute-node NUM', Integer, 'Execute nodes.')
           set(:help, '-h', '--help', 'Show this command help and server specific help.') do |opt|

data/lib/drbqs/config/process_list.rb

@@ -1,3 +1,5 @@
+require "cgi"
+
 module DRbQS
   class ProcessList
     PROCESS_ROOT_DIRECTORY = 'process'
@@ -68,9 +70,9 @@ module DRbQS
       def uri_to_file(uri)
         case uri
         when /^druby.*:(\d+)$/
-          "druby_#{Regexp.last_match[1]}.yaml"
-        when /^drbunix:(.*)$/
-          "drbunix#{Regexp.last_match[1].gsub(/\//, '_')}.yaml"
+          "druby_#{CGI.escape(Regexp.last_match[1])}.yaml"
+        when /^drbunix:(.+)$/
+          "drbunix_#{CGI.escape(Regexp.last_match[1])}.yaml"
         else
           raise ArgumentError, "Invalid uri of drbqs server: #{uri}"
         end
@@ -80,10 +82,10 @@ module DRbQS
       def file_to_uri(file)
         s = file.sub(/\.yaml$/, '')
         case s
-        when /^druby/
-          s.sub(/_/, '://:')
-        when /^drbunix$/
-          s.gsub(/_/, '/')
+        when /^druby_(.+)$/
+          "druby://:#{CGI.unescape(Regexp.last_match[1])}"
+        when /^drbunix_(.+)$/
+          "drbunix:#{CGI.unescape(Regexp.last_match[1])}"
         else
           raise ArgumentError, "Invalid file name in process list: #{file}"
         end

data/lib/drbqs/execute/process_define.rb

@@ -5,6 +5,10 @@ module DRbQS
 
     attr_reader :register
 
+    # @param [Symbol] server Symbol of server name
+    # @param [Array] node An array of Symbol which means node name
+    # @param [String] port Port number
+    # @param [IO,nil] io IO object to output
     def initialize(server, node, port, io = nil)
       @server = server
       @node = node
@@ -97,14 +101,35 @@ module DRbQS
     private :server_port
 
     def server_uri(name)
+      uri = nil
       if ary = get_server_setting(name)
-        DRbQS::Misc.create_uri(:host => ary[1][:args][0], :port => server_port)
-      else
-        nil
+        data = ary[1]
+        if data[:unix_domain_socket]
+          uri = DRbQS::Misc.uri_drbunix(data[:setting].value.unix.first)
+        else
+          uri = DRbQS::Misc.create_uri(:host => data[:args][0], :port => server_port)
+        end
       end
+      uri
     end
     private :server_uri
 
+    PATH_CPUINFO = "/proc/cpuinfo"
+
+    def get_suitable_process_num
+      n = 0
+      if File.exist?(PATH_CPUINFO)
+        n = File.read(PATH_CPUINFO).lines.count { |l| /^processor/ =~ l }
+      end
+      if n <= 0
+        n = 1
+        puts_progress "Can not determine suitable process number, that is, can not count 'processor' lines in /proc/cpuinfo"
+      end
+      puts_progress "Execute #{n} processes to deal with tasks"
+      n
+    end
+    private :get_suitable_process_num
+
     def execute_server(server_args)
       if ary = get_server_setting(@server)
         name = ary[0].to_s
@@ -121,9 +146,18 @@ module DRbQS
           server_setting.value.daemon FileName.create(local_log_directory, "server_execute.log", :position => :middle)
         end
         server_setting.set_server_argument(*server_args)
-        server_setting.value.port server_port
-        unless server_setting.set?(:sftp_host)
-          server_setting.value.sftp_host hostname
+        if data[:unix_domain_socket]
+          unless server_setting.value.unix
+            server_setting.value.unix DRbQS::Temporary.socket_path
+          end
+          unless server_setting.value.execute_node
+            server_setting.value.execute_node get_suitable_process_num
+          end
+        else
+          server_setting.value.port server_port
+          unless server_setting.set?(:sftp_host)
+            server_setting.value.sftp_host hostname
+          end
         end
         setting.parse!
         unless data[:ssh]
@@ -158,12 +192,15 @@ module DRbQS
           setting.value.connect name.to_s
         end
       else
-        node_log_dir = FileName.create(local_log_directory, 'node_execute_log', :directory => :self)
+        node_log_dir = FileName.create(local_log_directory, "node_#{name}_log", :directory => :self)
         setting.clear :log_stdout
         setting.value.log_prefix File.join(node_log_dir, 'node')
         setting.value.daemon File.join(node_log_dir, 'execute.log')
       end
       setting.parse!
+      # TODO:
+      # If node is on localhost then program is terminated here,
+      # because the node is executed as daemon process.
       setting.exec
     rescue Exception => err
       puts_progress "Fail to execute node '#{name.to_s}'"
@@ -183,10 +220,16 @@ module DRbQS
     end
     private :each_node_to_execute
 
+    TIME_INTERVAL_EXECUTE_NODE = 1
+
     def execute_node
       uri = server_uri(@server)
-      each_node_to_execute do |name, data|
-        execute_one_node(name, data, uri)
+      if uri && /^drbunix/ !~ uri
+        each_node_to_execute do |name, data|
+          execute_one_node(name, data, uri)
+          # If there is no time interval then drb does not work properly.
+          sleep(TIME_INTERVAL_EXECUTE_NODE)
+        end
       end
     end
 
@@ -222,25 +265,39 @@ module DRbQS
       end
       string_name_size = ary.max
       info[:server].each do |name, data|
-        prop = (data[:ssh] ? 'ssh' : 'local')
-        prop << ',template' if data[:template]
-        str << (data[:template] ? " - " : (info[:default][:server] == name ? " * " : "   "))
+        if data[:unix_domain_socket]
+          prop = "local(unix socket domain)"
+        elsif data[:ssh]
+          prop = "ssh"
+        else
+          prop = "local(ssh)"
+        end
+        str << (info[:default][:server] == name ? " * " : (data[:template] ? " - " : "   "))
         str << sprintf("%- #{string_name_size}s  %s\n", name, prop)
       end
-      str << "Node:\n"
+      str << "\nNode:\n"
       info[:node].each do |name, data|
         if data[:type] == :group
           prop = 'group: ' << data[:args].map(&:to_s).join(',')
         else
           prop = (data[:ssh] ? 'ssh' : 'local')
-          if data[:template]
-            prop << ',template'
-          end
         end
-        str << (data[:template] ? " - " : (info[:default][:node].include?(name) ? " * " : "   "))
+        if info[:default][:node].include?(name)
+          str << " * "
+        elsif data[:type] == :group
+          str << " # "
+        elsif data[:template]
+          str << " - "
+        else
+          str << "   "
+        end
         str << sprintf("%- #{string_name_size}s  %s\n", name, prop)
       end
-      str << "Port: #{info[:default][:port]}"
+      str << "\nDefault port:\n   #{info[:default][:port]}"
+      str << "\n\nHelp:\n"
+      str << "   ssh:   Process over SSH\n"
+      str << "   local: Process on localhost\n"
+      str << "   *: default, -: template, #: node group"
     end
 
     def usage
@@ -257,5 +314,39 @@ module DRbQS
         ''
       end
     end
+
+    def test_consistency
+      # Test existence of default server
+      if @server && !get_server_setting(@server)
+        raise "Invalid default server: #{@server.inspect}"
+      end
+      # Test existences of default nodes
+      if node_names = default_value(:node)
+        all_node_find_p = true
+        node_names.each do |node|
+          unless get_node_data(node)
+            all_node_find_p = false
+            $stderr.puts "Node definition #{node.inspect} does not exist!"
+          end
+        end
+        unless all_node_find_p
+          raise "Invalid default node."
+        end
+      end
+    end
+
+    TIME_INTERVAL_WAIT_SERVER_FINISH = 3
+
+    def wait_server_finish
+      if uri = server_uri(@server)
+        puts_progress "Wait finish of server #{uri}"
+        manage = DRbQS::Manage.new(:uri => uri)
+        while manage.server_respond?
+          sleep(TIME_INTERVAL_EXECUTE_NODE)
+        end
+      else
+        puts_progress "We tried to wait finish, however, we can not determine server uri"
+      end
+    end
   end
 end