drbqs 0.0.13 → 0.0.14

data/drbqs.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{drbqs}
-  s.version = "0.0.13"
+  s.version = "0.0.14"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Takayuki YAMAGUCHI"]
-  s.date = %q{2011-04-23}
+  s.date = %q{2011-07-19}
   s.description = %q{Task queuing system over network that is implemented by dRuby.}
   s.email = %q{d@ytak.info}
-  s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server"]
+  s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server", "drbqs-ssh"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -28,12 +28,10 @@ Gem::Specification.new do |s|
     "bin/drbqs-manage",
     "bin/drbqs-node",
     "bin/drbqs-server",
+    "bin/drbqs-ssh",
     "drbqs.gemspec",
     "example/README.md",
     "example/command/server_def.rb",
-    "example/drbqs-manage-test.rb",
-    "example/drbqs-node-test.rb",
-    "example/drbqs-server-test.rb",
     "example/error/error.rb",
     "example/error/server_def.rb",
     "example/server/server.rb",
@@ -44,121 +42,144 @@ Gem::Specification.new do |s|
     "example/transfer/file.rb",
     "example/transfer/server_def.rb",
     "lib/drbqs.rb",
-    "lib/drbqs/acl_file.rb",
-    "lib/drbqs/client.rb",
-    "lib/drbqs/config.rb",
-    "lib/drbqs/connection.rb",
-    "lib/drbqs/execute_node.rb",
-    "lib/drbqs/history.rb",
-    "lib/drbqs/manage.rb",
-    "lib/drbqs/message.rb",
-    "lib/drbqs/node_list.rb",
-    "lib/drbqs/queue.rb",
-    "lib/drbqs/server.rb",
-    "lib/drbqs/server_define.rb",
-    "lib/drbqs/server_hook.rb",
-    "lib/drbqs/ssh/host.rb",
-    "lib/drbqs/ssh/shell.rb",
-    "lib/drbqs/ssh/transfer.rb",
-    "lib/drbqs/task.rb",
-    "lib/drbqs/task_client.rb",
-    "lib/drbqs/task_generator.rb",
-    "lib/drbqs/utils.rb",
-    "spec/acl_file_spec.rb",
-    "spec/config_spec.rb",
-    "spec/connection_spec.rb",
-    "spec/data/acl.txt",
-    "spec/history_spec.rb",
-    "spec/manage_spec.rb",
-    "spec/message_spec.rb",
-    "spec/node_list_spec.rb",
-    "spec/queue_spec.rb",
-    "spec/server_check_alive_spec.rb",
-    "spec/server_define_spec.rb",
-    "spec/server_hook_spec.rb",
-    "spec/server_spec.rb",
+    "lib/drbqs/config/config.rb",
+    "lib/drbqs/config/process_list.rb",
+    "lib/drbqs/config/ssh_host.rb",
+    "lib/drbqs/manage/execute_node.rb",
+    "lib/drbqs/manage/manage.rb",
+    "lib/drbqs/manage/send_signal.rb",
+    "lib/drbqs/manage/ssh_execute.rb",
+    "lib/drbqs/manage/ssh_shell.rb",
+    "lib/drbqs/node/connection.rb",
+    "lib/drbqs/node/node.rb",
+    "lib/drbqs/node/task_client.rb",
+    "lib/drbqs/server/acl_file.rb",
+    "lib/drbqs/server/check_alive.rb",
+    "lib/drbqs/server/history.rb",
+    "lib/drbqs/server/message.rb",
+    "lib/drbqs/server/node_list.rb",
+    "lib/drbqs/server/queue.rb",
+    "lib/drbqs/server/server.rb",
+    "lib/drbqs/server/server_hook.rb",
+    "lib/drbqs/server/transfer_setting.rb",
+    "lib/drbqs/task/command_task.rb",
+    "lib/drbqs/task/task.rb",
+    "lib/drbqs/task/task_generator.rb",
+    "lib/drbqs/utility/argument.rb",
+    "lib/drbqs/utility/command_line.rb",
+    "lib/drbqs/utility/command_line/command_base.rb",
+    "lib/drbqs/utility/command_line/command_manage.rb",
+    "lib/drbqs/utility/command_line/command_node.rb",
+    "lib/drbqs/utility/command_line/command_server.rb",
+    "lib/drbqs/utility/command_line/command_ssh.rb",
+    "lib/drbqs/utility/misc.rb",
+    "lib/drbqs/utility/server_define.rb",
+    "lib/drbqs/utility/temporary.rb",
+    "lib/drbqs/utility/transfer/file_transfer.rb",
+    "lib/drbqs/utility/transfer/transfer_client.rb",
+    "spec/config/config_spec.rb",
+    "spec/config/process_list_spec.rb",
+    "spec/config/ssh_host_spec.rb",
+    "spec/integration_test/01_basic_usage_spec.rb",
+    "spec/integration_test/02_use_generator_spec.rb",
+    "spec/integration_test/03_use_temporary_file_spec.rb",
+    "spec/integration_test/04_use_unix_domain_spec.rb",
+    "spec/integration_test/05_server_exit_signal_spec.rb",
+    "spec/integration_test/06_node_exit_after_task_spec.rb",
+    "spec/integration_test/07_command_server_with_node_spec.rb",
+    "spec/integration_test/definition/server01.rb",
+    "spec/integration_test/definition/server02.rb",
+    "spec/integration_test/definition/task_obj_definition.rb",
+    "spec/manage/manage_spec.rb",
+    "spec/manage/send_signal_spec.rb",
+    "spec/manage/ssh_shell_spec.rb",
+    "spec/node/connection_spec.rb",
+    "spec/node/task_client_spec.rb",
+    "spec/server/acl_file_spec.rb",
+    "spec/server/check_alive_spec.rb",
+    "spec/server/data/acl.txt",
+    "spec/server/history_spec.rb",
+    "spec/server/message_spec.rb",
+    "spec/server/node_list_spec.rb",
+    "spec/server/queue_spec.rb",
+    "spec/server/server_hook_spec.rb",
+    "spec/server/server_spec.rb",
+    "spec/server/transfer_setting_spec.rb",
     "spec/spec_helper.rb",
-    "spec/ssh_shell_spec.rb",
-    "spec/task_client_spec.rb",
-    "spec/task_generator_spec.rb",
-    "spec/task_spec.rb",
-    "spec/test/test1.rb",
-    "spec/test1_spec.rb",
-    "spec/test2_spec.rb",
-    "spec/transfer_spec.rb"
+    "spec/task/file_transfer_spec.rb",
+    "spec/task/task_generator_spec.rb",
+    "spec/task/task_spec.rb",
+    "spec/utility/argument_spec.rb",
+    "spec/utility/command_line/command_base_spec.rb",
+    "spec/utility/command_line/commands_spec.rb",
+    "spec/utility/misc_spec.rb",
+    "spec/utility/server_define_spec.rb",
+    "spec/utility/temporary_spec.rb"
   ]
   s.homepage = %q{http://github.com/ytaka/drbqs}
   s.licenses = ["GPL3"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.6.2}
   s.summary = %q{dRuby Queueing System}
-  s.test_files = [
-    "spec/acl_file_spec.rb",
-    "spec/config_spec.rb",
-    "spec/connection_spec.rb",
-    "spec/history_spec.rb",
-    "spec/manage_spec.rb",
-    "spec/message_spec.rb",
-    "spec/node_list_spec.rb",
-    "spec/queue_spec.rb",
-    "spec/server_check_alive_spec.rb",
-    "spec/server_define_spec.rb",
-    "spec/server_hook_spec.rb",
-    "spec/server_spec.rb",
-    "spec/spec_helper.rb",
-    "spec/ssh_shell_spec.rb",
-    "spec/task_client_spec.rb",
-    "spec/task_generator_spec.rb",
-    "spec/task_spec.rb",
-    "spec/test/test1.rb",
-    "spec/test1_spec.rb",
-    "spec/test2_spec.rb",
-    "spec/transfer_spec.rb"
-  ]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rspec>, [">= 2.5.0"])
-      s.add_development_dependency(%q<yard>, ["~> 0.6.0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rspec>, [">= 2.6.0"])
+      s.add_development_dependency(%q<yard>, [">= 0.7.2"])
+      s.add_development_dependency(%q<bundler>, [">= 1.0.15"])
+      s.add_development_dependency(%q<jeweler>, [">= 1.6.2"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
-      s.add_development_dependency(%q<filename>, [">= 0.0.5"])
-      s.add_development_dependency(%q<net-ssh>, [">= 2.1.3"])
-      s.add_development_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+      s.add_development_dependency(%q<filename>, [">= 0.1.0"])
+      s.add_development_dependency(%q<user_config>, [">= 0.0.1"])
+      s.add_development_dependency(%q<net-ssh>, [">= 2.1.4"])
+      s.add_development_dependency(%q<net-ssh-shell>, [">= 0.2.0"])
+      s.add_development_dependency(%q<net-sftp>, [">= 2.0.5"])
+      s.add_development_dependency(%q<sys-proctable>, [">= 0"])
+      s.add_runtime_dependency(%q<filename>, [">= 0.1.0"])
+      s.add_runtime_dependency(%q<user_config>, [">= 0.0.2"])
       s.add_runtime_dependency(%q<net-ssh>, [">= 2.1.3"])
       s.add_runtime_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
-      s.add_runtime_dependency(%q<filename>, [">= 0.0.5"])
-      s.add_development_dependency(%q<rspec>, [">= 2.5.0"])
+      s.add_runtime_dependency(%q<net-sftp>, [">= 2.0.5"])
+      s.add_runtime_dependency(%q<sys-proctable>, [">= 0"])
     else
-      s.add_dependency(%q<rspec>, [">= 2.5.0"])
-      s.add_dependency(%q<yard>, ["~> 0.6.0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rspec>, [">= 2.6.0"])
+      s.add_dependency(%q<yard>, [">= 0.7.2"])
+      s.add_dependency(%q<bundler>, [">= 1.0.15"])
+      s.add_dependency(%q<jeweler>, [">= 1.6.2"])
       s.add_dependency(%q<rcov>, [">= 0"])
-      s.add_dependency(%q<filename>, [">= 0.0.5"])
-      s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
-      s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+      s.add_dependency(%q<filename>, [">= 0.1.0"])
+      s.add_dependency(%q<user_config>, [">= 0.0.1"])
+      s.add_dependency(%q<net-ssh>, [">= 2.1.4"])
+      s.add_dependency(%q<net-ssh-shell>, [">= 0.2.0"])
+      s.add_dependency(%q<net-sftp>, [">= 2.0.5"])
+      s.add_dependency(%q<sys-proctable>, [">= 0"])
+      s.add_dependency(%q<filename>, [">= 0.1.0"])
+      s.add_dependency(%q<user_config>, [">= 0.0.2"])
       s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
       s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
-      s.add_dependency(%q<filename>, [">= 0.0.5"])
-      s.add_dependency(%q<rspec>, [">= 2.5.0"])
+      s.add_dependency(%q<net-sftp>, [">= 2.0.5"])
+      s.add_dependency(%q<sys-proctable>, [">= 0"])
     end
   else
-    s.add_dependency(%q<rspec>, [">= 2.5.0"])
-    s.add_dependency(%q<yard>, ["~> 0.6.0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rspec>, [">= 2.6.0"])
+    s.add_dependency(%q<yard>, [">= 0.7.2"])
+    s.add_dependency(%q<bundler>, [">= 1.0.15"])
+    s.add_dependency(%q<jeweler>, [">= 1.6.2"])
     s.add_dependency(%q<rcov>, [">= 0"])
-    s.add_dependency(%q<filename>, [">= 0.0.5"])
-    s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
-    s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
+    s.add_dependency(%q<filename>, [">= 0.1.0"])
+    s.add_dependency(%q<user_config>, [">= 0.0.1"])
+    s.add_dependency(%q<net-ssh>, [">= 2.1.4"])
+    s.add_dependency(%q<net-ssh-shell>, [">= 0.2.0"])
+    s.add_dependency(%q<net-sftp>, [">= 2.0.5"])
+    s.add_dependency(%q<sys-proctable>, [">= 0"])
+    s.add_dependency(%q<filename>, [">= 0.1.0"])
+    s.add_dependency(%q<user_config>, [">= 0.0.2"])
     s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
     s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
-    s.add_dependency(%q<filename>, [">= 0.0.5"])
-    s.add_dependency(%q<rspec>, [">= 2.5.0"])
+    s.add_dependency(%q<net-sftp>, [">= 2.0.5"])
+    s.add_dependency(%q<sys-proctable>, [">= 0"])
   end
 end
 

data/example/README.md

@@ -4,8 +4,8 @@
 
 We execute server by the following command:
 
-    ruby drbqs-server-test.rb sum/server_def.rb
+    ../bin/drbqs-server sum/server_def.rb
 
 For nodes we type the following command:
 
-    ruby drbqs-client-test.rb -l sum/sum.rb --log-stdout
+    ../bin/drbqs-node -l sum/sum.rb --log-stdout

data/lib/drbqs/config/config.rb

@@ -0,0 +1,88 @@
+require 'user_config'
+require 'drbqs/config/ssh_host'
+require 'drbqs/config/process_list'
+
+module DRbQS
+
+  ACL_DEFAULT_PATH = 'acl.txt'
+  ACL_SAMPLE_PATH = 'acl.txt.sample'
+  HOST_FILE_DIRECTORY = 'host'
+  HOST_FILE_SAMPLE_PATH = 'host.yaml.sample'
+  SHELL_FILE_DIRECTORY = 'shell'
+  SHELL_BASHRC = 'bashrc'
+
+  class Config
+    class Directory < UserConfig
+    end
+
+    DRBQS_CONFIG_DIRECTORY = '.drbqs'
+
+    @@home_directory = nil
+
+    def self.set_home_directory(path)
+      @@home_directory = File.expand_path(path)
+    end
+
+    def self.get_home_directory
+      @@home_directory || ENV['HOME']
+    end
+
+    ACL_SAMPLE =<<SAMPLE
+deny all
+allow localhost
+allow 127.0.0.1
+SAMPLE
+
+    HOST_YAML_SAMPLE =<<SAMPLE
+--- 
+:dest: user@example.com
+:dir:
+:shell: bash --noprofile --init-file ~/.drbqs/shell/bashrc
+:rvm:
+:rvm_init: ~/.rvm/scripts/rvm
+:output: 
+SAMPLE
+
+    BASHRC_SAMPLE = <<SAMPLE
+HISTFILE=$HOME/.drbqs/shell/bash_history
+HISTSIZE=10000
+HISTFILESIZE=20000
+SAMPLE
+
+    attr_reader :directory, :list, :ssh_host
+
+    def initialize
+      home = self.class.get_home_directory
+      @directory = DRbQS::Config::Directory.new(DRBQS_CONFIG_DIRECTORY, :home => home)
+      @list = DRbQS::ProcessList.new(File.join(home, DRBQS_CONFIG_DIRECTORY))
+      @ssh_host = DRbQS::Config::SSHHost.new(@directory.file_path(HOST_FILE_DIRECTORY))
+    end
+
+    def directory_path
+      @directory.directory
+    end
+
+    def save_sample
+      @directory.open(ACL_SAMPLE_PATH, 'w') do |f|
+        f.print ACL_SAMPLE
+      end
+      @directory.open(File.join(HOST_FILE_DIRECTORY, HOST_FILE_SAMPLE_PATH), 'w') do |f|
+        f.print HOST_YAML_SAMPLE
+      end
+      @directory.open(File.join(SHELL_FILE_DIRECTORY, SHELL_BASHRC), 'w') do |f|
+        f.print BASHRC_SAMPLE
+      end
+    end
+
+    # Return path of ACL file if '.drbqs/acl.txt' exists.
+    def get_acl_file
+      @directory.exist?(ACL_DEFAULT_PATH)
+    end
+
+    def get_shell_file_directory
+      @directory.file_path(SHELL_FILE_DIRECTORY)
+    end
+
+  end
+
+end

data/lib/drbqs/config/process_list.rb

@@ -0,0 +1,194 @@
+module DRbQS
+  class ProcessList
+    PROCESS_ROOT_DIRECTORY = 'process'
+    SERVER_DIRECTORY = 'server'
+    NODE_DIRECTORY = 'node'
+
+    class ListDirectory
+      def initialize(dir)
+        @directory = dir
+        FileUtils.mkdir_p(@directory) unless File.exist?(@directory)
+      end
+
+      def path_under_directory(file)
+        File.join(@directory, file)
+      end
+      private :path_under_directory
+
+      # If +file+ exists then this method returns false.
+      # Otherwise, return true.
+      def save_file(file, data)
+        path = path_under_directory(file)
+        if File.exist?(path)
+          false
+        else
+          open(path, 'w') do |f|
+            f.print data.to_yaml
+          end
+          true
+        end
+      end
+      private :save_file
+
+      # If +file+ does not exist then this method returns nil.
+      def load_file(file)
+        path = path_under_directory(file)
+        if File.exist?(path)
+          YAML.load_file(path)
+        else
+          nil
+        end
+      end
+      private :load_file
+
+      def delete_file(file)
+        path = path_under_directory(file)
+        FileUtils.remove(path) if File.exist?(path)
+      end
+      private :delete_file
+
+      def entries
+        Dir.entries(@directory).delete_if do |dir|
+          /^\.+$/ =~ dir
+        end
+      end
+      private :entries
+
+      def server_of_key_exist?(uri, key)
+        if data = get(uri)
+          return data[:key] == key
+        end
+        false
+      end
+    end
+
+    class Server < ListDirectory
+      def uri_to_file(uri)
+        case uri
+        when /^druby.*:(\d+)$/
+          "druby_#{Regexp.last_match[1]}.yaml"
+        when /^drbunix:(.*)$/
+          "drbunix#{Regexp.last_match[1].gsub(/\//, '_')}.yaml"
+        else
+          raise ArgumentError, "Invalid uri of drbqs server: #{uri}"
+        end
+      end
+      private :uri_to_file
+
+      def file_to_uri(file)
+        s = file.sub(/\.yaml$/, '')
+        case s
+        when /^druby/
+          s.sub(/_/, '://:')
+        when /^drbunix$/
+          s.gsub(/_/, '/')
+        else
+          raise ArgumentError, "Invalid file name in process list: #{file}"
+        end
+      end
+      private :file_to_uri
+
+      def list
+        h = {}
+        entries.each do |file|
+          h[file_to_uri(file)] = load_file(file)
+        end
+        h
+      end
+
+      # If file exists then this method overwrites the file.
+      def save(uri, data)
+        unless save_file(uri_to_file(uri), data)
+          delete(uri)
+          save(uri, data)
+        end
+      end
+
+      def get(uri)
+        load_file(uri_to_file(uri))
+      end
+
+      def delete(uri)
+        delete_file(uri_to_file(uri))
+      end
+
+      def clear_process_not_exist
+        list.each do |uri, data|
+          if !DRbQS::Misc.process_running_normally?(data[:pid]) || !DRbQS::Manage.new(:uri => uri).server_respond?
+            delete(uri)
+          end
+        end
+      end
+    end
+
+    class Node < ListDirectory
+      def pid_to_file(pid)
+        sprintf("%010d.yaml", pid)
+      end
+      private :pid_to_file
+
+      def file_to_pid(file)
+        file.sub(/\.yaml$/, '').to_i
+      end
+      private :file_to_pid
+
+      def list
+        h = {}
+        entries.each do |file|
+          h[file_to_pid(file)] = load_file(file)
+        end
+        h
+      end
+
+      # If file exists then this method overwrites the file.
+      def save(pid, data)
+        unless save_file(pid_to_file(pid), data)
+          delete(pid)
+          save(pid, data)
+        end
+      end
+
+      def get(pid)
+        load_file(pid_to_file(pid))
+      end
+
+      def delete(pid)
+        delete_file(pid_to_file(pid))
+      end
+
+      def clear_process_not_exist
+        server = Hash.new do |h, k|
+          h[k] = Array.new
+        end
+        list.each do |pid, data|
+          if DRbQS::Misc.process_running_normally?(pid)
+            server[data[:uri]] << pid
+          else
+            delete(pid)
+          end
+        end
+        server.each do |server_uri, pid_ary|
+          unless DRbQS::Manage.new(:uri => server_uri).server_respond?
+            pid_ary.each do |pid|
+              delete(pid)
+            end
+          end
+        end
+      end
+    end
+
+    attr_reader :root, :server, :node
+
+    def initialize(home)
+      hostname = Socket.gethostname
+      @root = File.expand_path(File.join(home, PROCESS_ROOT_DIRECTORY))
+      @server = DRbQS::ProcessList::Server.new(File.join(@root, SERVER_DIRECTORY, hostname))
+      @node = DRbQS::ProcessList::Node.new(File.join(@root, NODE_DIRECTORY, hostname))
+    end
+
+    def clear_process_not_exist
+      @server.clear_process_not_exist
+      @node.clear_process_not_exist
+    end
+  end
+end

data/lib/drbqs/config/ssh_host.rb

@@ -0,0 +1,41 @@
+module DRbQS
+  class Config
+    class SSHHost
+      def initialize(dir)
+        @dir = File.expand_path(dir)
+      end
+
+      def config_files
+        (Dir.glob("#{@dir}/*.yaml") + Dir.glob("#{@dir}/*.yml")).sort
+      end
+      private :config_files
+
+      def config_names
+        config_files.map do |s|
+          File.basename(s).sub(/\.ya?ml$/, '')
+        end
+      end
+
+      def find_file(name)
+        config_files.find do |s|
+          File.basename(s).sub(/\.ya?ml$/, '') == name
+        end
+      end
+      private :find_file
+
+      def get_path(name)
+        name = name.to_s
+        name.size > 0 && find_file(name)
+      end
+
+      # +name+ is file name without extension.
+      def get_options(name)
+        if path = get_path(name)
+          return [path, YAML.load_file(path)]
+        end
+        [nil, {}]
+      end
+
+    end
+  end
+end

data/lib/drbqs/{execute_node.rb → manage/execute_node.rb}

@@ -1,6 +1,8 @@
 module DRbQS
 
   class ExecuteNode
+    attr_reader :pid
+
     def initialize(uri, log_prefix, log_level)
       @uri = uri
       @log_level = log_level
@@ -24,9 +26,9 @@ module DRbQS
 
     def create_process
       @pid << fork do
-        client = DRbQS::Client.new(@uri, :log_level => @log_level, :log_file => get_log_file)
-        client.connect
-        client.calculate
+        node = DRbQS::Node.new(@uri, :log_level => @log_level, :log_file => get_log_file)
+        node.connect
+        node.calculate
       end
     end
     private :create_process
@@ -39,7 +41,7 @@ module DRbQS
     end
 
     def wait
-      Process.waitall 
+      Process.waitall
     end
   end