drbqs 0.0.15 → 0.0.16

data/lib/drbqs/utility/transfer/transfer_client_connect.rb

@@ -0,0 +1,83 @@
+require 'net/sftp'
+
+module DRbQS
+  class Transfer
+    class Client
+      class Base
+        def initialize(server_directory)
+          @directory = server_directory
+        end
+
+        def upload_name(path)
+          File.join(@directory, File.basename(path))
+        end
+        private :upload_name
+
+        def directory_for_download(path)
+          dir = DRbQS::Temporary.directory
+          [dir, File.join(dir, File.basename(path))]
+        end
+        private :directory_for_download
+      end
+
+      # Transfer files to directory on DRbQS server over sftp.
+      # Note that after we transfer files we delete the original files.
+      class SFTP < DRbQS::Transfer::Client::Base
+        attr_reader :user, :host, :directory
+
+        def initialize(user, host, directory)
+          super(directory)
+          @user = user
+          @host = host
+        end
+
+        def start_sftp(&block)
+          Net::SFTP.start(@host, @user, &block)
+        end
+        private :start_sftp
+
+        # Transfer and delete +files+.
+        def transfer(files)
+          start_sftp do |sftp|
+            files.each do |path|
+              sftp.upload(path, upload_name(path))
+              FileUtils.rm_r(path)
+            end
+          end
+        rescue => err
+          raise err.class, "user=#{@user}, host=#{@host}, directory=#{@directory}; #{err.to_s}", err.backtrace 
+        end
+
+        def download(files)
+          moved = []
+          start_sftp do |sftp|
+            files.each do |path|
+              dir, downloaded_path = directory_for_download(path)
+              sftp.download(path, dir, :recursive => true)
+              moved << downloaded_path
+            end
+          end
+          moved
+        rescue => err
+          raise err.class, "user=#{@user}, host=#{@host}, directory=#{@directory}; #{err.to_s}", err.backtrace 
+        end
+      end
+
+      class Local < DRbQS::Transfer::Client::Base
+        def transfer(files)
+          files.each do |path|
+            FileUtils.mv(path, upload_name(path))
+          end
+        end
+
+        def download(files)
+          files.map do |path|
+            dir, downloaded_path = directory_for_download(path)
+            FileUtils.cp_r(path, dir)
+            downloaded_path
+          end
+        end
+      end
+    end
+  end
+end

data/lib/drbqs/utility/transfer/transfer_file_list.rb

@@ -0,0 +1,40 @@
+module DRbQS
+  # Class for path of files to send from server to a node.
+  class Transfer
+    class FileList
+      # Initialization is executed on server.
+      # If :readonly option is true, nodes on same computer as server
+      # does not copy files.
+      # Therefore, if we edit the files then the change remains.
+      # If :readonly option is not true then the files is copied,
+      # so the original files are not changed.
+      def initialize(*files)
+        opts = (Hash === files[-1] ? files.pop : {})
+        @readonly = opts[:readonly]
+        @files = files.map do |path|
+          epath = File.expand_path(path)
+          unless File.exist?(epath)
+            raise ArgumentError, "#{epath} does not exist."
+          end
+          epath
+        end
+        @downloaded = nil
+        @path = nil
+      end
+
+      # This method is executed on a node.
+      def download
+        @downloaded = true
+        @path = DRbQS::Transfer::Client.get.download(@files, @readonly)
+      end
+      private :download
+
+      # Return an array of paths of downloaded files.
+      # Note that this method is executed on a node.
+      def path
+        download unless @downloaded
+        @path
+      end
+    end
+  end
+end

data/spec/execute/def/execute1.rb

@@ -5,20 +5,20 @@ usage message: "Usage of this definition.", server: server_file
 
 default log: '/tmp/drbqs_tmp_log'
 
-register_server 'server_local', 'localhost' do |server|
+server 'server_local', 'localhost' do |server|
   server.load server_file
 end
 
-register_server 'server_ssh', 'example.com' do |server, ssh|
+server 'server_ssh', 'example.com' do |server, ssh|
   ssh.connect "example.com"
   server.load 'server.rb'
 end
 
-register_node 'node_local' do |node|
+node 'node_local' do |node|
   node.log_level = Logger::DEBUG
 end
 
-register_node 'node_ssh' do |node, ssh|
+node 'node_ssh' do |node, ssh|
   ssh.connect "example.com"
   node.log_level = Logger::DEBUG
 end

data/spec/execute/def/execute2.rb

@@ -0,0 +1,24 @@
+# Use in process_define_spec.rb
+
+server_file = File.join(File.dirname(__FILE__), '../../integration_test/definition/server02.rb')
+usage message: "Usage of this definition.", server: server_file
+
+default log: '/tmp/drbqs_tmp_log', node: [:node_ssh]
+
+server 'server_local', 'localhost' do |server|
+  server.load server_file
+end
+
+server 'server_ssh', 'example.com' do |server, ssh|
+  ssh.connect "example.com"
+  server.load 'server.rb'
+end
+
+node 'node_local' do |node|
+  node.log_level = Logger::DEBUG
+end
+
+node 'node_ssh' do |node, ssh|
+  ssh.connect "example.com"
+  node.log_level = Logger::DEBUG
+end

data/spec/execute/process_define_spec.rb

@@ -18,13 +18,27 @@ describe DRbQS::ProcessDefinition do
     end
 
     it "should get server list." do
-      info = @info[:server].sort_by { |a| a [0] }
-      info.should == [[:server_local, :server], [:server_ssh, :ssh]].sort_by { |a| a [0] }
+      data = @info[:server].assoc(:server_local)[1]
+      data[:type].should == :server
+      data[:ssh].should_not be_true
+      data[:args].should == ["localhost"]
+
+      data = @info[:server].assoc(:server_ssh)[1]
+      data[:type].should == :server
+      data[:ssh].should be_true
+      data[:args].should == ["example.com"]
     end
 
     it "should get node list." do
-      info = @info[:node].sort_by { |a| a[0] }
-      info.should == [[:node_local, :node], [:node_ssh, :ssh]].sort_by { |a| a[0] }
+      data = @info[:node].assoc(:node_local)[1]
+      data[:type].should == :node
+      data[:ssh].should_not be_true
+      data[:args].should == []
+
+      data = @info[:node].assoc(:node_ssh)[1]
+      data[:type].should == :node
+      data[:ssh].should be_true
+      data[:args].should == []
     end
 
     it "should get default server." do
@@ -103,7 +117,7 @@ describe DRbQS::ProcessDefinition do
       @tmp = '/tmp/drbqs_tmp_log' # defined in execute1.rb
       @process_def = DRbQS::ProcessDefinition.new(nil, nil, @port)
       @process_def.load(definition_file('execute1.rb'))
-      @server_setting = subject.__send__(:get_server_setting)[:setting]
+      @server_setting = subject.__send__(:get_server_setting)[1][:setting]
       @node_local_setting = subject.__send__(:get_node_data, :node_local)[:setting]
       @node_ssh_setting = subject.__send__(:get_node_data, :node_ssh)[:setting]
     end
@@ -139,7 +153,7 @@ describe DRbQS::ProcessDefinition do
       @tmp = '/tmp/drbqs_tmp_log' # defined in execute1.rb
       @process_def = DRbQS::ProcessDefinition.new(nil, [:node_local], nil)
       @process_def.load(definition_file('execute1.rb'))
-      @server_setting = subject.__send__(:get_server_setting)[:setting]
+      @server_setting = subject.__send__(:get_server_setting)[1][:setting]
       @node_local_setting = subject.__send__(:get_node_data, :node_local)[:setting]
     end
 
@@ -164,4 +178,27 @@ describe DRbQS::ProcessDefinition do
       FileUtils.rm_r(@tmp)
     end
   end
+
+  context "when execute default nodes" do
+    before(:all) do
+      @port = DRbQS::ROOT_DEFAULT_PORT
+      @hostname = 'localhost'     # defined in execute1.rb
+      @tmp = '/tmp/drbqs_tmp_log' # defined in execute1.rb
+      @process_def = DRbQS::ProcessDefinition.new(nil, nil, nil)
+      @process_def.load(definition_file('execute2.rb'))
+      @server_setting = subject.__send__(:get_server_setting)[1][:setting]
+      @node_local_setting = subject.__send__(:get_node_data, :node_local)[:setting]
+      @node_ssh_setting = subject.__send__(:get_node_data, :node_ssh)[:setting]
+    end
+
+    subject do
+      @process_def
+    end
+
+    it "should check port number set in node." do
+      @node_local_setting.should_not_receive(:exec)
+      @node_ssh_setting.should_receive(:exec)
+      subject.execute_node
+    end
+  end
 end

data/spec/execute/register_spec.rb

@@ -10,53 +10,424 @@ describe DRbQS::ProcessDefinition::Register do
   context "when registering a server" do
     it "should define a server on localhost." do
       load_file = 'file.rb'
-      subject.register_server(:server1) do |server|
+      subject.server(:server1, "example.com") do |server|
         server.load load_file
       end
       name, data = subject.__server__.assoc(:server1)
       data[:type].should == :server
-      data[:setting].get(:load).should == [load_file]
+      data[:ssh].should be_false
+      data[:template].should be_false
+      setting = data[:setting]
+      setting.get(:load).should == [load_file]
+      setting.should be_an_instance_of DRbQS::Setting::Server
     end
 
     it "should define a server over ssh." do
       dest = "user@example.com"
       load_file = 'file.rb'
-      subject.register_server(:server1) do |server, ssh|
+      subject.server(:server2, "example.com") do |server, ssh|
         server.load load_file
         ssh.connect dest
       end
-      name, data = subject.__server__.assoc(:server1)
-      data[:type].should == :ssh
+      name, data = subject.__server__.assoc(:server2)
+      data[:type].should == :server
+      data[:ssh].should be_true
+      data[:template].should be_false
       setting = data[:setting]
+      setting.should be_an_instance_of DRbQS::Setting::SSH
       setting.get(:connect).should == [dest]
       setting.mode_setting.get(:load).should == [load_file]
     end
+
+    it "should raise error for arguments without hostname." do
+      load_file = 'file.rb'
+      lambda do
+        subject.server(:server3) do |server|
+          server.load load_file
+        end
+      end.should raise_error
+    end
+
+    it "should set template." do
+      load_file = 'file.rb'
+      subject.server(:server4, :template => true) do |server|
+        server.load load_file
+      end
+      name, data = subject.__server__.assoc(:server4)
+      data[:template].should be_true
+      data[:ssh].should be_false
+      data[:type].should == :server
+      setting = data[:setting]
+      setting.get(:load).should == [load_file]
+      setting.should be_an_instance_of DRbQS::Setting::Server
+    end
+
+    it "should define a server over ssh." do
+      load_file = 'file.rb'
+      bash = "bash"
+      subject.server(:server5, :template => true) do |server, ssh|
+        server.load load_file
+        ssh.shell bash
+      end
+      name, data = subject.__server__.assoc(:server5)
+      data[:type].should == :server
+      data[:ssh].should be_true
+      data[:template].should be_true
+      setting = data[:setting]
+      setting.should be_an_instance_of DRbQS::Setting::SSH
+      setting.get(:shell).should == [bash]
+      setting.mode_setting.get(:load).should == [load_file]
+    end
   end
 
   context "when registering a node" do
     it "should define a node on localhost." do
       uri = 'druby://:12345'
-      subject.register_node(:node1) do |node|
+      subject.node(:node1) do |node|
         node.connect uri
       end
       name, data = subject.__node__.assoc(:node1)
       data[:type].should == :node
-      data[:setting].get(:connect).should == [uri]
+      data[:ssh].should be_false
+      data[:template].should be_false
+      setting = data[:setting]
+      setting.should be_an_instance_of DRbQS::Setting::Node
+      setting.get(:connect).should == [uri]
     end
 
     it "should define a node over ssh." do
       dest = "user@example.com"
       uri = 'druby://:12345'
-      subject.register_node(:node2) do |node, ssh|
+      subject.node(:node2) do |node, ssh|
         node.connect uri
         ssh.connect dest
       end
       name, data = subject.__node__.assoc(:node2)
-      data[:type].should == :ssh
+      data[:type].should == :node
+      data[:ssh].should be_true
+      data[:template].should be_false
+      setting = data[:setting]
+      setting.should be_an_instance_of DRbQS::Setting::SSH
+      setting.get(:connect).should == [dest]
+      setting.mode_setting.get(:connect).should == [uri]
+    end
+
+    it "should define a node template." do
+      uri = 'druby://:12345'
+      subject.node(:node3, :template => true) do |node|
+        node.connect uri
+      end
+      name, data = subject.__node__.assoc(:node3)
+      data[:type].should == :node
+      data[:ssh].should be_false
+      data[:template].should be_true
+      setting = data[:setting]
+      setting.should be_an_instance_of DRbQS::Setting::Node
+      setting.get(:connect).should == [uri]
+    end
+
+    it "should define a node over ssh." do
+      dest = "user@example.com"
+      uri = 'druby://:12345'
+      subject.node(:node4, :template => true) do |node, ssh|
+        node.connect uri
+        ssh.connect dest
+      end
+      name, data = subject.__node__.assoc(:node4)
+      data[:type].should == :node
+      data[:ssh].should be_true
+      data[:template].should be_true
       setting = data[:setting]
+      setting.should be_an_instance_of DRbQS::Setting::SSH
       setting.get(:connect).should == [dest]
       setting.mode_setting.get(:connect).should == [uri]
     end
+
+    it "should set node group." do
+      nodes = [:node1, :node2]
+      subject.node(:node_group, :group => nodes)
+      name, data = subject.__node__.assoc(:node_group)
+      data[:type].should == :group
+      data[:template].should be_true
+      data[:ssh].should_not be_true
+      data[:args].should == nodes
+    end
+
+    it "should raise error for invalid node group." do
+      lambda do
+        subject.node(:node_group, :group => :node_error)
+      end.should raise_error
+    end
+  end
+
+  context "when loading other server difinition" do
+    it "should load server definition." do
+      subject.server :parent, template: true do |server|
+        server.log_level "debug"
+      end
+      subject.server :child, 'example.com', load: :parent do |server|
+        server.load "file.rb"
+      end
+
+      name, data = subject.__server__.assoc(:parent)
+      setting = data[:setting]
+      setting.value.load.should be_nil
+
+      name, data = subject.__server__.assoc(:child)
+      setting = data[:setting]
+      setting.value.log_level.should == ["debug"]
+      setting.value.load.should == ["file.rb"]
+    end
+
+    it "should load server definition that is over ssh." do
+      subject.server :parent, template: true do |server, ssh|
+        server.log_level "debug"
+        ssh.rvm "ruby-head"
+      end
+      subject.server :child, 'example.com', load: :parent do |server, ssh|
+        server.load "file.rb"
+        ssh.nice 20
+      end
+
+      name, data = subject.__server__.assoc(:parent)
+      setting = data[:setting]
+      setting.value.nice.should be_nil
+      setting.mode_setting.value.load.should be_nil
+
+      name, data = subject.__server__.assoc(:child)
+      setting = data[:setting]
+      setting.value.rvm.should == ["ruby-head"]
+      setting.value.nice.should == [20]
+      setting.mode_setting.value.log_level.should == ["debug"]
+      setting.mode_setting.value.load.should == ["file.rb"]
+    end
+
+    it "should load server definition on localhost for definition over ssh." do
+      subject.server :parent, template: true do |server|
+        server.log_level "debug"
+      end
+      subject.server :child, 'example.com', load: :parent do |server, ssh|
+        server.load "file.rb"
+        ssh.nice 20
+      end
+
+      name, data = subject.__server__.assoc(:parent)
+      setting = data[:setting]
+      setting.value.log.should be_nil
+
+      name, data = subject.__server__.assoc(:child)
+      setting = data[:setting]
+      setting.value.nice.should == [20]
+      setting.mode_setting.value.log_level.should == ["debug"]
+      setting.mode_setting.value.load.should == ["file.rb"]
+    end
+
+    it "should raise error because definition on localhost loads that over ssh." do
+      subject.server :parent, template: true do |server, ssh|
+        server.log_level "debug"
+        ssh.rvm "ruby-head"
+      end
+      lambda do
+        subject.server :child, 'example.com', load: :parent do |server|
+        end
+      end.should raise_error
+    end
+  end
+
+  context "when loading other node difinition" do
+    it "should load node definition." do
+      subject.node :parent, template: true do |node|
+        node.log_level "debug"
+      end
+      subject.node :child, load: :parent do |node|
+        node.load "file.rb"
+      end
+
+      name, data = subject.__node__.assoc(:parent)
+      setting = data[:setting]
+      setting.value.load.should be_nil
+
+      name, data = subject.__node__.assoc(:child)
+      setting = data[:setting]
+      setting.value.log_level.should == ["debug"]
+      setting.value.load.should == ["file.rb"]
+    end
+
+    it "should load node definition that is over ssh." do
+      subject.node :parent, template: true do |node, ssh|
+        node.log_level "debug"
+        ssh.rvm "ruby-head"
+      end
+      subject.node :child, load: :parent do |node, ssh|
+        node.load "file.rb"
+        ssh.nice 20
+      end
+
+      name, data = subject.__node__.assoc(:parent)
+      setting = data[:setting]
+      setting.value.nice.should be_nil
+      setting.mode_setting.value.load.should be_nil
+
+      name, data = subject.__node__.assoc(:child)
+      setting = data[:setting]
+      setting.value.rvm.should == ["ruby-head"]
+      setting.value.nice.should == [20]
+      setting.mode_setting.value.log_level.should == ["debug"]
+      setting.mode_setting.value.load.should == ["file.rb"]
+    end
+
+    it "should load node definition on localhost for definition over ssh." do
+      subject.node :parent, template: true do |node|
+        node.log_level "debug"
+      end
+      subject.node :child, load: :parent do |node, ssh|
+        node.load "file.rb"
+        ssh.nice 20
+      end
+
+      name, data = subject.__node__.assoc(:parent)
+      setting = data[:setting]
+      setting.value.log.should be_nil
+
+      name, data = subject.__node__.assoc(:child)
+      setting = data[:setting]
+      setting.value.nice.should == [20]
+      setting.mode_setting.value.log_level.should == ["debug"]
+      setting.mode_setting.value.load.should == ["file.rb"]
+    end
+
+    it "should raise error because definition on localhost loads that over ssh." do
+      subject.node :parent, template: true do |node, ssh|
+        node.log_level "debug"
+        ssh.rvm "ruby-head"
+      end
+      lambda do
+        subject.node :child, load: :parent do |node|
+        end
+      end.should raise_error
+    end
+
+    it "should raise error because group definition is loaded." do
+      subject.node :parent, group: [:node1, :node2, :node3]
+      lambda do
+        subject.node :child, load: :parent do |node|
+        end
+      end.should raise_error
+    end
+  end
+
+  context "when reconfiguring server" do
+    it "should reconfigure definition." do
+      load_file = 'file.rb'
+      subject.server(:server1, "example.com") do |server|
+        server.load load_file
+      end
+      subject.server(:server1, "example.com") do |server|
+        server.log_level 'info'
+      end
+
+      name, data = subject.__server__.assoc(:server1)
+      data[:type].should == :server
+      data[:ssh].should be_false
+      data[:template].should be_false
+      setting = data[:setting]
+      setting.get(:load).should == [load_file]
+      setting.get(:log_level).should == ['info']
+    end
+
+    it "should raise error for simultaneous reconfiguring and loading." do
+      subject.server(:server2, "example.com") do |server|
+      end
+      lambda do
+        subject.server(:server2, "example.com", load: :some_definition) do |server|
+        end
+      end.should raise_error
+    end
+  end
+
+  context "when reconfiguring node" do
+    it "should reconfigure definition." do
+      load_file = 'file.rb'
+      subject.node(:node1) do |node|
+        node.load load_file
+      end
+      subject.node(:node1) do |node|
+        node.log_level 'info'
+      end
+
+      name, data = subject.__node__.assoc(:node1)
+      data[:type].should == :node
+      data[:ssh].should be_false
+      data[:template].should be_false
+      setting = data[:setting]
+      setting.get(:load).should == [load_file]
+      setting.get(:log_level).should == ['info']
+    end
+
+    it "should raise error to change type of group." do
+      nodes = [:node1, :node2]
+      nodes2 = [:node3, :node4, :node5]
+      subject.node(:node_group, :group => nodes)
+      subject.node(:node_group, :group => nodes2)
+      name, data = subject.__node__.assoc(:node_group)
+      data[:type].should == :group
+      data[:template].should be_true
+      data[:ssh].should_not be_true
+      data[:args].should == nodes2
+    end
+
+    it "should raise error for simultaneous reconfiguring and loading." do
+      subject.node(:node2) do |node|
+      end
+      lambda do
+        subject.node(:node2, load: :some_definition) do |node|
+        end
+      end.should raise_error
+    end
+
+    it "should raise error to change type of group." do
+      subject.node(:node_group, group: [:node1, :node2])
+      lambda do
+        subject.node(:node_group) do |node|
+        end
+      end.should raise_error
+    end
+
+    it "should raise error to change type of process." do
+      subject.node(:node) do |node|
+      end
+      lambda do
+        subject.node(:node, group: [:node1, :node2])
+      end.should raise_error
+    end
+  end
+
+  context "when clearing" do
+    it "should clear servers" do
+      subject.server(:server1, "example.com") do |server|
+      end
+      subject.server(:server2, "example.com") do |server|
+      end
+      subject.server(:server3, "example.com") do |server|
+      end
+      subject.clear_server(:server1, :server3)
+      subject.__server__.assoc(:server1).should_not be_true
+      subject.__server__.assoc(:server2).should be_true
+      subject.__server__.assoc(:server3).should_not be_true
+    end
+
+    it "should clear nodes" do
+      subject.node(:node1) do |node|
+      end
+      subject.node(:node2) do |node|
+      end
+      subject.node(:node3) do |node|
+      end
+      subject.clear_node(:node1, :node3)
+      subject.__node__.assoc(:node1).should_not be_true
+      subject.__node__.assoc(:node2).should be_true
+      subject.__node__.assoc(:node3).should_not be_true
+    end
   end
 
   context "when setting default" do
@@ -73,5 +444,28 @@ describe DRbQS::ProcessDefinition::Register do
       h[:server].should == :server2
       h[:port].should == 12345
     end
+
+    it "should set default nodes." do
+      nodes = [:node1, :node2]
+      subject.default(:node => nodes)
+      h = subject.__default__
+      h[:node].should == nodes
+    end
+
+    it "should set default value for some keys." do
+      subject.default(:log => '/tmp/drbqs/log', :some_key => 'some_value')
+      h = subject.__default__
+      h[:log].should == '/tmp/drbqs/log'
+      h[:some_key].should == 'some_value'
+    end
+
+    it "should clear values." do
+      subject.default(key1: 'val1', key2: 'val2', key3: 'val3')
+      subject.default_clear(:key1, :key2)
+      h = subject.__default__
+      h[:key1].should be_nil
+      h[:key2].should be_nil
+      h[:key3].should == 'val3'
+    end
   end
 end