xli-dtr 0.0.5 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGES +7 -0
- data/README.rdoc +208 -0
- data/Rakefile +54 -63
- data/TODO +7 -16
- data/bin/dtr +26 -20
- data/dtr.gemspec +7 -10
- data/lib/dtr/agent/brain.rb +13 -22
- data/lib/dtr/agent/herald.rb +24 -14
- data/lib/dtr/agent/runner.rb +22 -35
- data/lib/dtr/agent/sync_codebase.rb +1 -1
- data/lib/dtr/agent/sync_logger.rb +38 -8
- data/lib/dtr/agent/test_case.rb +53 -0
- data/lib/dtr/agent/test_unit.rb +3 -5
- data/lib/dtr/agent/worker.rb +29 -32
- data/lib/dtr/agent/working_env_ext.rb +4 -2
- data/lib/dtr/agent.rb +2 -1
- data/lib/dtr/facade.rb +65 -0
- data/lib/dtr/master.rb +3 -3
- data/lib/dtr/monitor.rb +69 -11
- data/lib/dtr/raketasks.rb +91 -19
- data/lib/dtr/shared/adapter.rb +29 -26
- data/lib/dtr/shared/configuration.rb +39 -11
- data/lib/dtr/shared/message_decorator.rb +1 -1
- data/lib/dtr/shared/ruby_ext.rb +1 -25
- data/lib/dtr/shared/service/agent.rb +5 -1
- data/lib/dtr/shared/service/file.rb +1 -1
- data/lib/dtr/shared/service/rinda.rb +11 -3
- data/lib/dtr/shared/service/runner.rb +6 -5
- data/lib/dtr/shared/service/working_env.rb +1 -1
- data/lib/dtr/shared/service.rb +1 -1
- data/lib/dtr/shared/sync_codebase/{codebase.rb → copiable_package.rb} +13 -5
- data/lib/dtr/shared/sync_codebase/master_ext.rb +6 -18
- data/lib/dtr/shared/sync_codebase/package.rb +16 -2
- data/lib/dtr/shared/sync_codebase/sync_service.rb +7 -12
- data/lib/dtr/shared/sync_codebase.rb +2 -2
- data/lib/dtr/shared/sync_logger.rb +6 -14
- data/lib/dtr/shared/utils/cmd.rb +5 -5
- data/lib/dtr/shared/utils/env_store.rb +1 -1
- data/lib/dtr/shared/utils/logger.rb +33 -17
- data/lib/dtr/shared/utils.rb +1 -1
- data/lib/dtr/shared/working_env.rb +2 -2
- data/lib/dtr/shared.rb +1 -1
- data/lib/dtr/test_unit/drb_test_runner.rb +5 -14
- data/lib/dtr/test_unit/injection.rb +1 -2
- data/lib/dtr/test_unit/test_case_injection.rb +13 -13
- data/lib/dtr/test_unit/test_suite_injection.rb +24 -0
- data/lib/dtr/test_unit/testrunnermediator_injection.rb +11 -11
- data/lib/dtr/test_unit/thread_safe_test_result.rb +1 -3
- data/lib/dtr/test_unit/worker_club.rb +7 -7
- data/lib/dtr/test_unit.rb +2 -1
- data/lib/dtr/test_unit_injection.rb +1 -1
- data/lib/dtr.rb +5 -36
- data/test/acceptance/agent_working_env_test.rb +28 -34
- data/test/acceptance/dtr_package_task_test.rb +13 -3
- data/test/acceptance/general_test.rb +139 -83
- data/test/acceptance/raketasks_test.rb +23 -0
- data/test/acceptance/sync_codebase_test.rb +12 -13
- data/test/acceptance/sync_logger_test.rb +12 -21
- data/test/agent_helper.rb +8 -10
- data/test/logger_stub.rb +4 -0
- data/test/test_helper.rb +33 -5
- data/test/unit/adapter_test.rb +58 -16
- data/test/unit/configuration_test.rb +44 -0
- data/test/unit/facade_test.rb +41 -0
- data/test/unit/logger_test.rb +72 -0
- data/test/unit/test_unit_test.rb +0 -21
- data/testdata/Rakefile +1 -5
- data/testdata/hacked_run_method_test_case.rb +15 -0
- data/testdata/raketasks/Rakefile +7 -0
- data/testdata/raketasks/success_test_case.rb +6 -0
- data/testdata/sleep_3_secs_test_case.rb +9 -0
- data/testdata/{should_not_sync_codebase_and_setup_working_dir_when_agent_is_in_same_dir_with_master_process → verify_dir_pwd}/verify_dir_pwd_test_case.rb +1 -1
- metadata +33 -13
- data/README +0 -182
- data/install.rb +0 -88
- /data/testdata/{should_not_sync_codebase_and_setup_working_dir_when_agent_is_in_same_dir_with_master_process → verify_dir_pwd}/Rakefile +0 -0
data/CHANGES
CHANGED
@@ -1,5 +1,12 @@
|
|
1
1
|
= DTR Changelog
|
2
2
|
|
3
|
+
== release 1.0.0
|
4
|
+
* support synchronizing codebase
|
5
|
+
* lookup agents by broadcast
|
6
|
+
* group agents for different project or environment usage
|
7
|
+
* no need launch dtr rinda server anymore
|
8
|
+
* agents log would be output into master process log
|
9
|
+
|
3
10
|
== release 0.0.4
|
4
11
|
* added timeout for running test more stable
|
5
12
|
the default timeout is 60 sec, can be changed by environment variable 'RUN_TEST_TIMEOUT'
|
data/README.rdoc
ADDED
@@ -0,0 +1,208 @@
|
|
1
|
+
= DTR -- Distributed Test Runner
|
2
|
+
|
3
|
+
Supporting DTR version: 1.x.x
|
4
|
+
|
5
|
+
This package contains DTR, a distributed test runner program for decreasing
|
6
|
+
time of running ruby tests, only supporting Test::Unit ruby testing framework
|
7
|
+
currently.
|
8
|
+
|
9
|
+
DTR has the following features:
|
10
|
+
|
11
|
+
* Run tests in mutli processes or on distributed machines.
|
12
|
+
|
13
|
+
* Hot plug distributed agents.
|
14
|
+
|
15
|
+
* Synchronizing codebase between agent and master processes.
|
16
|
+
|
17
|
+
* Runtime injection, all tests run in same environment.
|
18
|
+
|
19
|
+
DTR works in two parts: Runner Agent and DTR Master.
|
20
|
+
|
21
|
+
* Runner Agent is a DRb service hosting on distributed machines to run tests. For using 'fork' to create runner process, Runner Agent can't run on Windows directly. Requisites for running agent: Unix/Linux based OS System for forking sub-process; 'unzip' command for extracting codebase package.
|
22
|
+
|
23
|
+
* DTR Master is the process finding runner service to run tests and collect test results. It works by loading 'dtr/test_unit_injection.rb' with all test files or defining a DTR::TestTask in your rake file. Requisites for running master: 'zip' command for creating codebase package.
|
24
|
+
|
25
|
+
DTR (version >= 1.0.0) supports synchronizing codebase by a DTR::PackageTask defined in your rake tasks. The DTR::PackageTask is a similar task with Rake::PackageTask, the following is a simple example:
|
26
|
+
|
27
|
+
require 'dtr/raketasks'
|
28
|
+
DTR::PackageTask.new do |p|
|
29
|
+
p.package_files.include("**/*")
|
30
|
+
p.package_files.exclude('tmp/**/*')
|
31
|
+
p.package_files.exclude('log/*')
|
32
|
+
end
|
33
|
+
|
34
|
+
The DTR::TestTask will create a DTR::PackageTask for you directly, and you can specify package_files too.
|
35
|
+
|
36
|
+
DTR::TestTask.new do |t|
|
37
|
+
t.test_files = FileList['test/*_test.rb']
|
38
|
+
t.processes = 2
|
39
|
+
t.package_files.include('**/*.rb')
|
40
|
+
t.package_files.exclude('tmp/**/*')
|
41
|
+
t.package_files.exclude('log/*')
|
42
|
+
end
|
43
|
+
|
44
|
+
Note: Exclude('log/*') only excludes all files inside 'log' directory except 'log' directory itself.
|
45
|
+
|
46
|
+
== Rails plugin
|
47
|
+
|
48
|
+
For Rails project, you just need copy dtr project directory into your project plugins directory. There are dtr tasks defined for you to run tests within dtr grid, which prefer you have 'config/database.yml.dtr' for setting up database. The test task is 'dtr:test'.
|
49
|
+
|
50
|
+
Your project directory name would be the default group name of dtr agents. You can specify an environment variable named 'DTR_GROUP' to change the group name. By default, dtr tasks would lookup the broadcast ip by 'ifconfig' command. If it doesn't work for you, you can specify an environment variable named 'BROADCAST_IP' to overwrite it.
|
51
|
+
|
52
|
+
== Download
|
53
|
+
|
54
|
+
The latest version of DTR can be found at
|
55
|
+
|
56
|
+
* http://github.com/xli/dtr/tree/master
|
57
|
+
|
58
|
+
== GEM Installation
|
59
|
+
|
60
|
+
=== Last stable version from rubyforge.org
|
61
|
+
|
62
|
+
Download and install DTR with the following.
|
63
|
+
|
64
|
+
gem install --remote dtr
|
65
|
+
|
66
|
+
=== Last version on github master branch
|
67
|
+
|
68
|
+
Run the following if you haven't already:
|
69
|
+
|
70
|
+
gem sources -a http://gems.github.com
|
71
|
+
|
72
|
+
Install the gem:
|
73
|
+
|
74
|
+
sudo gem install xli-dtr
|
75
|
+
|
76
|
+
== Running the DTR Test Suite
|
77
|
+
|
78
|
+
If you wish to run the unit and functional tests that come with DTR:
|
79
|
+
|
80
|
+
* CD into the top project directory of dtr.
|
81
|
+
* Type the following:
|
82
|
+
|
83
|
+
rake # You need a version of rake installed
|
84
|
+
|
85
|
+
== Simple Example
|
86
|
+
|
87
|
+
Start DTR agent with providing 1 runner in group 'mysql-firefox' as follows:
|
88
|
+
|
89
|
+
dtr -r runner1 -g mysql-firefox
|
90
|
+
|
91
|
+
Type "dtr --help" for an up-to-date option summary.
|
92
|
+
Invoking <tt>dtr</tt> without any options causes dtr to show help too.
|
93
|
+
|
94
|
+
Most of time your project test suite need setup environment before run tests, you
|
95
|
+
can set setup command by option '--setup', for example:
|
96
|
+
|
97
|
+
dtr -r runner1,runner2 --setup "rake db:test:prepare"
|
98
|
+
|
99
|
+
You also can specify DTR_AGENT_ENV_SETUP_CMD in your master process environment to let all agents
|
100
|
+
that are not started with '--setup' option specified to setup all agents environment.
|
101
|
+
|
102
|
+
At last, you need define a DTR::TestTask:
|
103
|
+
|
104
|
+
require 'dtr'
|
105
|
+
|
106
|
+
ENV['DTR_AGENT_ENV_SETUP_CMD'] = 'rake db:test:prepare'
|
107
|
+
DTR.broadcast_list = ['broadcast_ip']
|
108
|
+
DTR.group = 'mysql-firefox'
|
109
|
+
|
110
|
+
DTR::TestTask.new do |t|
|
111
|
+
t.test_files = FileList['test/**/*_test.rb']
|
112
|
+
t.processes = 0 # don't start agent in local machine, default is 1, so we reset to 0 here.
|
113
|
+
t.package_files.include("Rakefile")
|
114
|
+
t.package_files.include("app/**/*")
|
115
|
+
t.package_files.include("db/migrate/**/*")
|
116
|
+
t.package_files.include("config/**/*")
|
117
|
+
t.package_files.include("lib/**/*")
|
118
|
+
t.package_files.include("vendor/**/*")
|
119
|
+
t.package_files.include("test/**/*")
|
120
|
+
end
|
121
|
+
|
122
|
+
The default task name is 'dtr', and it also creates package tasks you need for packaging files need for running test.
|
123
|
+
More details about DTR::TestTask and DTR::PackageTask, see the API doc:
|
124
|
+
|
125
|
+
ri DTR::TestTask
|
126
|
+
ri DTR::PackageTask
|
127
|
+
|
128
|
+
Notes:
|
129
|
+
* DTR broadcast_list and group configuration would be cached in the directory. Name of configuration file is '.dtr_env_pstore'.
|
130
|
+
* For packaging codebase, DTR::PackageTask creates tasks: dtr_package, dtr_clobber_package and dtr_repackage; DTR Master just simply run a command 'rake dtr_repackage' to create the package and run 'rake dtr_clobber_package' to clean package. So make sure there are those tasks under root namespace in your rake tasks.
|
131
|
+
|
132
|
+
== Run tests in multi-processes on one machine
|
133
|
+
|
134
|
+
For running Runner in multi-processes.
|
135
|
+
The following is the test task example in the rake file:
|
136
|
+
|
137
|
+
require 'dtr/raketasks'
|
138
|
+
|
139
|
+
DTR::TestTask.new do |t|
|
140
|
+
t.test_files = FileList['test/*test.rb']
|
141
|
+
t.processes = 2 #default is 1
|
142
|
+
end
|
143
|
+
|
144
|
+
== Credits
|
145
|
+
|
146
|
+
[<b>Josh Price</b>] For fixing tests packer in release 0.0.1.
|
147
|
+
|
148
|
+
[<b>Wang Pengchao</b>] For sharing lots of ideas and code contributions.
|
149
|
+
|
150
|
+
[<b>Barrow H Kwan</b>] For patch of specifying DTR Rinda server port (version 0.0.x) and testing DTR new version.
|
151
|
+
|
152
|
+
[<b>Mingle team</b>(http://studios.thoughtworks.com/mingle-project-intelligence)] For making all these happen.
|
153
|
+
|
154
|
+
== License
|
155
|
+
|
156
|
+
DTR is available under an Apache License Version 2.
|
157
|
+
|
158
|
+
== Support
|
159
|
+
|
160
|
+
Feel free to submit commits or feature requests.
|
161
|
+
For other information, feel free to contact mailto:iam@li-xiao.com.
|
162
|
+
|
163
|
+
== Usage
|
164
|
+
|
165
|
+
DTR agent command is invoked from the command line using:
|
166
|
+
|
167
|
+
dtr [<em>options</em> ...]
|
168
|
+
|
169
|
+
=== Options are:
|
170
|
+
|
171
|
+
-p, --port PORT Port number of DTR agent listening. Default is 7788.
|
172
|
+
-g, --group GROUP_NAME If you have several DTR grids working for different project or environment, you should group your agents in different names for different usages. Default is none.
|
173
|
+
-r runner1_name,runner2_name Start DTR test runner agent with unique runner names.
|
174
|
+
-a, --broadcast_address ADDRESS Specify broadcast address for looking up dtr agent service, e.g. 192.168.255.255. Default is 'localhost'. DTR master and monitor would need this.
|
175
|
+
-i, --setup COMMAND Set command for initializing test runner test environment, e.g. 'rake db:test:prepare'. Default is do nothing. You also can specify DTR_AGENT_ENV_SETUP_CMD in your master process environment to let all agents setup same environment.
|
176
|
+
-d DIRECTORY Specify a directory as agent launching & working directory.
|
177
|
+
--working_directory
|
178
|
+
-m, --monitor Monitor the status of the dtr agents and master processes. Used for testing your dtr grid environment. CAUTION! monitoring agents causes all idle agents hang on by the monitor process.
|
179
|
+
-v, --version Show version
|
180
|
+
-h, --help Show this help doc
|
181
|
+
|
182
|
+
Notes:
|
183
|
+
* DTR would always add 'localhost' into broadcast list.
|
184
|
+
* Agent start by specifying runners by -r option, every runner would be started in different process for running test. e.g. dtr -r runner1,runner2
|
185
|
+
* DTR master environment options:
|
186
|
+
* DTR_MASTER_ENV: this variable would be copied into agent process for sharing info between master and agents. Normally used in agent setup environment command.
|
187
|
+
* DTR_AGENT_ENV_SETUP_CMD: this variable would be applied as agent setup environment command when agent have no setup environment command specified by --setup option.
|
188
|
+
* DTR_LOG_LEVEL: master process logger level, e.g. ENV['DTR_LOG_LEVEL'] = Logger::DEBUG. Agent process logs would be output in master process log file, so setting this option also changes agent logger level.
|
189
|
+
* DTR_RUNNER_NAME: this environment variable would be provided in runner process for test environment to get the runner name to setup, e.g. setting up database configuration.
|
190
|
+
|
191
|
+
Type "dtr --help" for an up-to-date option summary.
|
192
|
+
|
193
|
+
= Other stuff
|
194
|
+
|
195
|
+
Author: Li Xiao <iam@li-xiao.com>
|
196
|
+
|
197
|
+
Requires: Ruby 1.8.6 or later
|
198
|
+
|
199
|
+
License: Copyright 2007-2008 by Li Xiao.
|
200
|
+
Released under an Apache License 2. See the LICENSE file
|
201
|
+
included in the distribution.
|
202
|
+
|
203
|
+
== Warranty
|
204
|
+
|
205
|
+
This software is provided "as is" and without any express or
|
206
|
+
implied warranties, including, without limitation, the implied
|
207
|
+
warranties of merchantibility and fitness for a particular
|
208
|
+
purpose.
|
data/Rakefile
CHANGED
@@ -63,51 +63,28 @@ Rake::TestTask.new(:test_functionals) do |t|
|
|
63
63
|
t.verbose = false
|
64
64
|
end
|
65
65
|
|
66
|
-
begin
|
67
|
-
require 'rcov/rcovtask'
|
68
|
-
|
69
|
-
Rcov::RcovTask.new do |t|
|
70
|
-
t.libs << "test"
|
71
|
-
t.rcov_opts = [
|
72
|
-
'-xRakefile', '-xrakefile', '-xpublish.rf', '--text-report',
|
73
|
-
]
|
74
|
-
t.test_files = FileList[
|
75
|
-
'test/*test.rb'
|
76
|
-
]
|
77
|
-
t.output_dir = 'coverage'
|
78
|
-
t.verbose = true
|
79
|
-
end
|
80
|
-
rescue LoadError
|
81
|
-
# No rcov available
|
82
|
-
end
|
83
|
-
|
84
66
|
directory 'testdata'
|
85
67
|
[:test_units].each do |t|
|
86
68
|
task t => ['testdata']
|
87
69
|
end
|
88
70
|
|
89
|
-
|
90
|
-
|
91
|
-
# Install DTR using the standard install.rb script.
|
92
|
-
|
93
|
-
desc "Install the application"
|
94
|
-
task :install do
|
95
|
-
ruby "install.rb"
|
71
|
+
task :monitor do
|
72
|
+
DTR.monitor
|
96
73
|
end
|
97
74
|
|
75
|
+
# CVS Tasks ----------------------------------------------------------
|
76
|
+
|
98
77
|
# Create a task to build the RDOC documentation tree.
|
99
78
|
|
100
79
|
rd = Rake::RDocTask.new("rdoc") { |rdoc|
|
101
80
|
rdoc.rdoc_dir = 'html'
|
102
|
-
|
103
|
-
# rdoc.template = 'css2'
|
104
|
-
rdoc.template = 'doc/jamis.rb'
|
81
|
+
rdoc.template = 'html'
|
105
82
|
rdoc.title = "DTR -- Distributed Test Runner"
|
106
83
|
rdoc.options << '--line-numbers' << '--inline-source' <<
|
107
|
-
'--main' << 'README' <<
|
84
|
+
'--main' << 'README.rdoc' <<
|
108
85
|
'--title' << '"DTR -- Distributed Test Runner'
|
109
|
-
rdoc.rdoc_files.include('README', 'LICENSE.txt', 'TODO', 'CHANGES')
|
110
|
-
rdoc.rdoc_files.include('lib/**/*.rb'
|
86
|
+
rdoc.rdoc_files.include('README.rdoc', 'LICENSE.txt', 'TODO', 'CHANGES')
|
87
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
111
88
|
}
|
112
89
|
|
113
90
|
# ====================================================================
|
@@ -117,6 +94,48 @@ rd = Rake::RDocTask.new("rdoc") { |rdoc|
|
|
117
94
|
if ! defined?(Gem)
|
118
95
|
puts "Package Target requires RubyGEMs"
|
119
96
|
else
|
97
|
+
Dir.glob(File.dirname(__FILE__) + "/testdata/**/log").each do |log_dir|
|
98
|
+
FileUtils.rm_rf(log_dir)
|
99
|
+
end
|
100
|
+
|
101
|
+
gem_content = <<-GEM
|
102
|
+
Gem::Specification.new do |spec|
|
103
|
+
spec.name = 'dtr'
|
104
|
+
spec.version = "1.0.0"
|
105
|
+
spec.summary = "DTR is a distributed test runner to run tests on distributed computers for decreasing build time."
|
106
|
+
|
107
|
+
#### Dependencies and requirements.
|
108
|
+
spec.files = #{(Dir.glob("lib/**/*.rb") + ["bin/dtr", "CHANGES", "dtr.gemspec", "lib", "LICENSE.TXT", "Rakefile", "README.rdoc", "TODO"]).inspect}
|
109
|
+
|
110
|
+
spec.test_files = #{(Dir.glob("test/**/*.rb") + Dir.glob("testdata/**/*")).inspect}
|
111
|
+
|
112
|
+
#### Load-time details: library and application (you will need one or both).
|
113
|
+
|
114
|
+
spec.require_path = 'lib' # Use these for libraries.
|
115
|
+
|
116
|
+
spec.bindir = "bin" # Use these for applications.
|
117
|
+
spec.executables = ["dtr"]
|
118
|
+
spec.default_executable = "dtr"
|
119
|
+
|
120
|
+
#### Documentation and testing.
|
121
|
+
|
122
|
+
spec.has_rdoc = true
|
123
|
+
spec.extra_rdoc_files = #{rd.rdoc_files.reject { |fn| fn =~ /\.rb$/ }.to_a.inspect}
|
124
|
+
spec.rdoc_options = #{rd.options.inspect}
|
125
|
+
|
126
|
+
#### Author and project details.
|
127
|
+
|
128
|
+
spec.author = "Li Xiao"
|
129
|
+
spec.email = "iam@li-xiao.com"
|
130
|
+
spec.homepage = "http://github.com/xli/dtr/tree/master"
|
131
|
+
spec.rubyforge_project = "dtr"
|
132
|
+
end
|
133
|
+
GEM
|
134
|
+
File.open(File.dirname(__FILE__) + '/dtr.gemspec', 'w') do |f|
|
135
|
+
f.write(gem_content)
|
136
|
+
end
|
137
|
+
|
138
|
+
#build gem package same steps with github
|
120
139
|
File.open(File.dirname(__FILE__) + '/dtr.gemspec') do |f|
|
121
140
|
data = f.read
|
122
141
|
spec = nil
|
@@ -193,33 +212,11 @@ file "TAGS" => RUBY_FILES do
|
|
193
212
|
sh "#{TAGS} #{RUBY_FILES}", :verbose => false
|
194
213
|
end
|
195
214
|
|
196
|
-
# --------------------------------------------------------------------
|
197
|
-
# Creating a release
|
198
|
-
|
199
215
|
task :update_site do
|
200
216
|
puts %x[scp -r html/* lixiao@rubyforge.org:/var/www/gforge-projects/dtr/]
|
201
217
|
end
|
202
218
|
|
203
|
-
|
204
|
-
DTR.launch_agent(['c1'], nil)
|
205
|
-
end
|
206
|
-
|
207
|
-
task :c3 do
|
208
|
-
DTR.launch_agent(['c1', 'c2', 'c3'], nil)
|
209
|
-
end
|
210
|
-
task :c10 do
|
211
|
-
DTR.launch_agent(['c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9', 'c10'], nil)
|
212
|
-
end
|
213
|
-
|
214
|
-
task :c2 do
|
215
|
-
DTR_AGENT_OPTIONS[:runners] = ['c1', 'c2']
|
216
|
-
DTR_AGENT_OPTIONS[:agent_env_setup_cmd] = nil
|
217
|
-
Dir.chdir('testdata') do
|
218
|
-
DTR.start_agent
|
219
|
-
end
|
220
|
-
end
|
221
|
-
|
222
|
-
Rake::TestTask.new(:dtr) do |t|
|
219
|
+
Rake::TestTask.new(:dtr_injected) do |t|
|
223
220
|
t.libs.unshift DTR.lib_path
|
224
221
|
t.test_files = FileList['dtr/test_unit_injection.rb', 'testdata/*.rb']
|
225
222
|
t.warning = true
|
@@ -228,14 +225,8 @@ end
|
|
228
225
|
|
229
226
|
require 'dtr/raketasks'
|
230
227
|
|
231
|
-
DTR::TestTask.new
|
228
|
+
DTR::TestTask.new do |t|
|
232
229
|
t.test_files = FileList['testdata/*.rb']
|
233
|
-
t.processes =
|
234
|
-
|
235
|
-
|
236
|
-
DTR::PackageTask.new do |p|
|
237
|
-
p.package_files.include("**/*")
|
238
|
-
p.package_files.exclude("tmp")
|
239
|
-
p.package_files.exclude("log")
|
230
|
+
t.processes = 0
|
231
|
+
t.package_files.include('**/*')
|
240
232
|
end
|
241
|
-
|
data/TODO
CHANGED
@@ -3,21 +3,12 @@
|
|
3
3
|
Send suggestions for this list to iam@li-xiao.com
|
4
4
|
|
5
5
|
=== To Do
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
should trigger started on runner in agent runner, for some tools is using this hook(e.g. inbrowser_test)
|
14
|
-
|
15
|
-
???? agent brain should wakeup when worker is dead
|
16
|
-
problem: when agent can't find test files, drb can't undump test object, the error would be: undefined method `run' for #<DRb::DRbUnknown:0x318800>
|
17
|
-
* tell Master this error that agent can't know what's item sent from Master
|
18
|
-
|
19
|
-
monitor dtr rinda server status
|
20
|
-
monitor acceptance test
|
21
|
-
|
6
|
+
|
7
|
+
=== TBD for next release
|
8
|
+
should trigger started notification on test runner in agent Herald, for some tools are using this hook(e.g. inbrowser_test) to setup test environment.
|
9
|
+
agent should wakeup when worker is dead, so that it could be picked up by another master process
|
10
|
+
auto prepare database for each runner, so that run tests in multi-processes could be easy setup.
|
11
|
+
provide a way to monitor agent runner efficiency
|
12
|
+
use system instead of fork to launch sub process in agent, which makes dtr support windows and jruby
|
22
13
|
|
23
14
|
(moved DONE list to CHANGES file)
|
data/bin/dtr
CHANGED
@@ -11,12 +11,13 @@ end
|
|
11
11
|
NOTES = <<-NOTES
|
12
12
|
----------------
|
13
13
|
Notes:
|
14
|
-
*
|
15
|
-
*
|
14
|
+
* DTR would always add 'localhost' into broadcast list.
|
15
|
+
* Agent start by specifying runners by -r option, every runner would be started in different process for running test. e.g. dtr -r runner1,runner2
|
16
16
|
* DTR master environment options:
|
17
|
-
* DTR_MASTER_ENV: this variable would be copied into
|
18
|
-
* DTR_AGENT_ENV_SETUP_CMD: this variable would be applied as agent setup environment command when agent have no setup environment command specified.
|
19
|
-
* DTR_LOG_LEVEL: master process logger level, e.g. ENV['DTR_LOG_LEVEL'] = Logger::DEBUG
|
17
|
+
* DTR_MASTER_ENV: this variable would be copied into agent process for sharing info between master and agents. Normally used in agent setup environment command.
|
18
|
+
* DTR_AGENT_ENV_SETUP_CMD: this variable would be applied as agent setup environment command when agent have no setup environment command specified by --setup option.
|
19
|
+
* DTR_LOG_LEVEL: master process logger level, e.g. ENV['DTR_LOG_LEVEL'] = Logger::DEBUG. Agent process logs would be output in master process log file, so setting this option also changes agent logger level.
|
20
|
+
* DTR_RUNNER_NAME: this environment variable would be provided in runner process for test environment to get the runner name to setup, e.g. setting up database configuration.
|
20
21
|
|
21
22
|
DTR is a distributed test runner program for decreasing time of running ruby tests based on ruby 'test/unit' package.
|
22
23
|
For additional information, see http://dtr.rubyforge.org/
|
@@ -26,41 +27,41 @@ opts = OptionParser.new do |opts|
|
|
26
27
|
opts.banner = "DTR usage: #{$0} [options]"
|
27
28
|
opts.separator ""
|
28
29
|
opts.separator "Synopsis:"
|
29
|
-
opts.separator "dtr -r runner1_name,runner2_name"
|
30
|
+
opts.separator "dtr -g group_name -r runner1_name,runner2_name"
|
30
31
|
opts.separator "dtr -a broadcast_ip -m"
|
31
32
|
opts.separator ""
|
32
33
|
opts.separator "Options:"
|
33
34
|
|
34
|
-
opts.on_tail("-m", "--monitor", "Monitor the status of the dtr agents.") do
|
35
|
-
|
35
|
+
opts.on_tail("-m", "--monitor", "Monitor the status of the dtr agents and master processes. Used for testing your dtr grid environment. CAUTION! monitoring agents causes all idle agents hang on by the monitor process.") do
|
36
|
+
$monitor = true
|
36
37
|
end
|
37
38
|
|
38
|
-
opts.on("-p", "--port PORT", "Port number of DTR
|
39
|
-
DTR.
|
39
|
+
opts.on("-p", "--port PORT", "Port number of DTR agent listening. Default is 7788.") do |port|
|
40
|
+
DTR.agent_listen_port = port
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on("-g", "--group GROUP_NAME", "If you have several DTR grids working for different project or environment, you should group your agents in different names for different usages. Default is none.") do |group_name|
|
44
|
+
DTR.group = group_name
|
40
45
|
end
|
41
46
|
|
42
47
|
opts.on("-r runner1_name,runner2_name", Array, "Start DTR test runner agent with unique runner names.") do |names|
|
43
48
|
DTR_AGENT_OPTIONS[:runners] = names.collect{|name| name.untaint}
|
44
49
|
end
|
45
50
|
|
46
|
-
opts.on("-a", "--broadcast_address ADDRESS", "Specify broadcast address for looking up dtr
|
51
|
+
opts.on("-a", "--broadcast_address ADDRESS", "Specify broadcast address for looking up dtr agent service, e.g. 192.168.255.255. Default is 'localhost'. DTR master and monitor would need this.") do |address|
|
47
52
|
if (!address.nil?) && (!address.empty?)
|
48
53
|
DTR.broadcast_list = [address]
|
49
54
|
end
|
50
55
|
end
|
51
56
|
|
52
|
-
opts.on("-i", "--setup COMMAND", "Set command for initializing test runner test environment, e.g. 'rake db:test:prepare'. Default is do nothing.") do |command|
|
57
|
+
opts.on("-i", "--setup COMMAND", "Set command for initializing test runner test environment, e.g. 'rake db:test:prepare'. Default is do nothing. You also can specify DTR_AGENT_ENV_SETUP_CMD in your master process environment to let all agents setup same environment.") do |command|
|
53
58
|
DTR_AGENT_OPTIONS[:agent_env_setup_cmd] = command.untaint
|
54
59
|
end
|
55
60
|
|
56
|
-
opts.
|
57
|
-
|
58
|
-
end
|
59
|
-
|
60
|
-
opts.on_tail("-b", "--be_silent", "Only show error messages") do
|
61
|
-
DTR.logger.log_level = Logger::ERROR
|
61
|
+
opts.on("-d", "--working_directory DIRECTORY", "Specify a directory as agent launching & working directory.") do |dir|
|
62
|
+
DTR_AGENT_OPTIONS[:launching_dir] = dir
|
62
63
|
end
|
63
|
-
|
64
|
+
|
64
65
|
opts.on_tail("-v", "--version", "Show version") do
|
65
66
|
puts "dtr, version " + DTRVERSION
|
66
67
|
end
|
@@ -80,6 +81,11 @@ if no_argv
|
|
80
81
|
puts NOTES
|
81
82
|
end
|
82
83
|
|
84
|
+
if $monitor
|
85
|
+
DTR.monitor
|
86
|
+
end
|
83
87
|
if DTR_AGENT_OPTIONS[:runners] && !DTR_AGENT_OPTIONS[:runners].empty?
|
84
|
-
|
88
|
+
Dir.chdir(DTR_AGENT_OPTIONS[:launching_dir] || '.') do
|
89
|
+
DTR.start_agent
|
90
|
+
end
|
85
91
|
end
|
data/dtr.gemspec
CHANGED
@@ -1,18 +1,13 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = 'dtr'
|
3
|
-
spec.version = "0.0
|
3
|
+
spec.version = "1.0.0"
|
4
4
|
spec.summary = "DTR is a distributed test runner to run tests on distributed computers for decreasing build time."
|
5
5
|
|
6
6
|
#### Dependencies and requirements.
|
7
|
+
spec.files = ["lib/dtr/agent/brain.rb", "lib/dtr/agent/herald.rb", "lib/dtr/agent/runner.rb", "lib/dtr/agent/sync_codebase.rb", "lib/dtr/agent/sync_logger.rb", "lib/dtr/agent/test_case.rb", "lib/dtr/agent/test_unit.rb", "lib/dtr/agent/worker.rb", "lib/dtr/agent/working_env_ext.rb", "lib/dtr/agent.rb", "lib/dtr/facade.rb", "lib/dtr/master.rb", "lib/dtr/monitor.rb", "lib/dtr/raketasks.rb", "lib/dtr/shared/adapter.rb", "lib/dtr/shared/configuration.rb", "lib/dtr/shared/message_decorator.rb", "lib/dtr/shared/ruby_ext.rb", "lib/dtr/shared/service/agent.rb", "lib/dtr/shared/service/file.rb", "lib/dtr/shared/service/rinda.rb", "lib/dtr/shared/service/runner.rb", "lib/dtr/shared/service/working_env.rb", "lib/dtr/shared/service.rb", "lib/dtr/shared/sync_codebase/copiable_package.rb", "lib/dtr/shared/sync_codebase/master_ext.rb", "lib/dtr/shared/sync_codebase/package.rb", "lib/dtr/shared/sync_codebase/sync_service.rb", "lib/dtr/shared/sync_codebase.rb", "lib/dtr/shared/sync_logger.rb", "lib/dtr/shared/utils/cmd.rb", "lib/dtr/shared/utils/env_store.rb", "lib/dtr/shared/utils/logger.rb", "lib/dtr/shared/utils.rb", "lib/dtr/shared/working_env.rb", "lib/dtr/shared.rb", "lib/dtr/test_unit/drb_test_runner.rb", "lib/dtr/test_unit/injection.rb", "lib/dtr/test_unit/test_case_injection.rb", "lib/dtr/test_unit/test_suite_injection.rb", "lib/dtr/test_unit/testrunnermediator_injection.rb", "lib/dtr/test_unit/thread_safe_test_result.rb", "lib/dtr/test_unit/worker_club.rb", "lib/dtr/test_unit.rb", "lib/dtr/test_unit_injection.rb", "lib/dtr.rb", "bin/dtr", "CHANGES", "dtr.gemspec", "lib", "LICENSE.TXT", "Rakefile", "README.rdoc", "TODO"]
|
7
8
|
|
8
|
-
|
9
|
-
# p Dir.glob("lib/**/*.rb") + ["bin/dtr", "CHANGES", "dtr.gemspec", "install.rb", "lib", "LICENSE.TXT", "Rakefile", "README", "TODO"]
|
10
|
-
# puts '------------'
|
11
|
-
# p Dir.glob("test/**/*.rb") + Dir.glob("testdata/**/*")
|
12
|
-
|
13
|
-
spec.files = ["lib/dtr/agent/brain.rb", "lib/dtr/agent/herald.rb", "lib/dtr/agent/runner.rb", "lib/dtr/agent/sync_codebase.rb", "lib/dtr/agent/sync_logger.rb", "lib/dtr/agent/test_unit.rb", "lib/dtr/agent/worker.rb", "lib/dtr/agent/working_env_ext.rb", "lib/dtr/agent.rb", "lib/dtr/master.rb", "lib/dtr/monitor.rb", "lib/dtr/raketasks.rb", "lib/dtr/shared/adapter.rb", "lib/dtr/shared/configuration.rb", "lib/dtr/shared/message_decorator.rb", "lib/dtr/shared/ruby_ext.rb", "lib/dtr/shared/service/agent.rb", "lib/dtr/shared/service/file.rb", "lib/dtr/shared/service/rinda.rb", "lib/dtr/shared/service/runner.rb", "lib/dtr/shared/service/working_env.rb", "lib/dtr/shared/service.rb", "lib/dtr/shared/sync_codebase/codebase.rb", "lib/dtr/shared/sync_codebase/master_ext.rb", "lib/dtr/shared/sync_codebase/package.rb", "lib/dtr/shared/sync_codebase/sync_service.rb", "lib/dtr/shared/sync_codebase.rb", "lib/dtr/shared/sync_logger.rb", "lib/dtr/shared/utils/cmd.rb", "lib/dtr/shared/utils/env_store.rb", "lib/dtr/shared/utils/logger.rb", "lib/dtr/shared/utils.rb", "lib/dtr/shared/working_env.rb", "lib/dtr/shared.rb", "lib/dtr/test_unit/drb_test_runner.rb", "lib/dtr/test_unit/injection.rb", "lib/dtr/test_unit/test_case_injection.rb", "lib/dtr/test_unit/testrunnermediator_injection.rb", "lib/dtr/test_unit/thread_safe_test_result.rb", "lib/dtr/test_unit/worker_club.rb", "lib/dtr/test_unit.rb", "lib/dtr/test_unit_injection.rb", "lib/dtr.rb", "bin/dtr", "CHANGES", "dtr.gemspec", "install.rb", "lib", "LICENSE.TXT", "Rakefile", "README", "TODO"]
|
9
|
+
spec.test_files = ["test/acceptance/agent_working_env_test.rb", "test/acceptance/dtr_package_task_test.rb", "test/acceptance/general_test.rb", "test/acceptance/raketasks_test.rb", "test/acceptance/sync_codebase_test.rb", "test/acceptance/sync_logger_test.rb", "test/agent_helper.rb", "test/logger_stub.rb", "test/test_helper.rb", "test/unit/adapter_test.rb", "test/unit/configuration_test.rb", "test/unit/facade_test.rb", "test/unit/logger_test.rb", "test/unit/test_unit_test.rb", "test/unit/working_env_test.rb", "testdata/a_failed_test_case.rb", "testdata/a_file_system_test_case.rb", "testdata/a_test_case.rb", "testdata/a_test_case2.rb", "testdata/an_error_test_case.rb", "testdata/another_project", "testdata/another_project/passed_test_case.rb", "testdata/another_project/Rakefile", "testdata/hacked_run_method_test_case.rb", "testdata/is_required_by_a_test.rb", "testdata/lib", "testdata/lib/lib_test_case.rb", "testdata/package_task_test_rakefile", "testdata/Rakefile", "testdata/raketasks", "testdata/raketasks/Rakefile", "testdata/raketasks/success_test_case.rb", "testdata/scenario_test_case.rb", "testdata/setup_agent_env_test_case.rb", "testdata/sleep_3_secs_test_case.rb", "testdata/verify_dir_pwd", "testdata/verify_dir_pwd/Rakefile", "testdata/verify_dir_pwd/verify_dir_pwd_test_case.rb"]
|
14
10
|
|
15
|
-
spec.test_files = ["test/acceptance/agent_working_env_test.rb", "test/acceptance/dtr_package_task_test.rb", "test/acceptance/general_test.rb", "test/acceptance/sync_codebase_test.rb", "test/acceptance/sync_logger_test.rb", "test/agent_helper.rb", "test/logger_stub.rb", "test/test_helper.rb", "test/unit/adapter_test.rb", "test/unit/test_unit_test.rb", "test/unit/working_env_test.rb", "testdata/a_failed_test_case.rb", "testdata/a_file_system_test_case.rb", "testdata/a_test_case.rb", "testdata/a_test_case2.rb", "testdata/an_error_test_case.rb", "testdata/another_project", "testdata/another_project/passed_test_case.rb", "testdata/another_project/Rakefile", "testdata/is_required_by_a_test.rb", "testdata/lib", "testdata/lib/lib_test_case.rb", "testdata/package_task_test_rakefile", "testdata/Rakefile", "testdata/scenario_test_case.rb", "testdata/setup_agent_env_test_case.rb", "testdata/should_not_sync_codebase_and_setup_working_dir_when_agent_is_in_same_dir_with_master_process", "testdata/should_not_sync_codebase_and_setup_working_dir_when_agent_is_in_same_dir_with_master_process/Rakefile", "testdata/should_not_sync_codebase_and_setup_working_dir_when_agent_is_in_same_dir_with_master_process/verify_dir_pwd_test_case.rb"]
|
16
11
|
#### Load-time details: library and application (you will need one or both).
|
17
12
|
|
18
13
|
spec.require_path = 'lib' # Use these for libraries.
|
@@ -23,7 +18,9 @@ Gem::Specification.new do |spec|
|
|
23
18
|
|
24
19
|
#### Documentation and testing.
|
25
20
|
|
26
|
-
spec.has_rdoc =
|
21
|
+
spec.has_rdoc = true
|
22
|
+
spec.extra_rdoc_files = ["README.rdoc", "LICENSE.txt", "TODO", "CHANGES"]
|
23
|
+
spec.rdoc_options = ["--line-numbers", "--inline-source", "--main", "README.rdoc", "--title", "\"DTR -- Distributed Test Runner"]
|
27
24
|
|
28
25
|
#### Author and project details.
|
29
26
|
|
@@ -31,4 +28,4 @@ Gem::Specification.new do |spec|
|
|
31
28
|
spec.email = "iam@li-xiao.com"
|
32
29
|
spec.homepage = "http://github.com/xli/dtr/tree/master"
|
33
30
|
spec.rubyforge_project = "dtr"
|
34
|
-
end
|
31
|
+
end
|
data/lib/dtr/agent/brain.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (c) 2007-2008 Li Xiao
|
1
|
+
# Copyright (c) 2007-2008 Li Xiao <iam@li-xiao.com>
|
2
2
|
#
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
4
|
# you may not use this file except in compliance with the License.
|
@@ -22,23 +22,26 @@ module DTR
|
|
22
22
|
raise 'No runner? What can I do for you?' if runner_names.blank?
|
23
23
|
@runner_names = runner_names
|
24
24
|
@agent_env_setup_cmd = agent_env_setup_cmd
|
25
|
-
DTR.info ""
|
26
|
-
DTR.info "--------------------beautiful line--------------------------"
|
27
|
-
DTR.info "=> Agent environment setup
|
28
|
-
DTR.info "=> Runner names: #{@runner_names.join(', ')}"
|
25
|
+
DTR.info {""}
|
26
|
+
DTR.info {"--------------------beautiful line--------------------------"}
|
27
|
+
DTR.info {"=> Agent environment setup command: #{@agent_env_setup_cmd}"}
|
28
|
+
DTR.info {"=> Runner names: #{@runner_names.join(', ')}"}
|
29
|
+
DTR.info {"=> Broadcast list: #{DTR.configuration.broadcast_list.inspect}"}
|
30
|
+
DTR.info {"=> Listening port: #{DTR.configuration.agent_listen_port}"}
|
31
|
+
DTR.info {"=> Group: #{DTR.configuration.group}"}
|
29
32
|
end
|
30
33
|
|
31
34
|
def hypnotize
|
32
35
|
loop do
|
33
36
|
if wakeup?
|
34
37
|
DTR.info {"Agent brain wakes up"}
|
35
|
-
work(
|
38
|
+
work(DTR.fork_process { Worker.new(@runner_names, @agent_env_setup_cmd).launch })
|
36
39
|
DTR.info {"Agent brain is going to sleep"}
|
37
40
|
end
|
38
41
|
end
|
39
|
-
rescue
|
40
|
-
|
41
|
-
|
42
|
+
rescue Interrupt, SystemExit, SignalException
|
43
|
+
ensure
|
44
|
+
relax
|
42
45
|
end
|
43
46
|
|
44
47
|
def work(worker)
|
@@ -47,19 +50,7 @@ module DTR
|
|
47
50
|
end
|
48
51
|
ensure
|
49
52
|
DTR.info {"Killing worker"}
|
50
|
-
|
51
|
-
end
|
52
|
-
|
53
|
-
def wakeup_worker
|
54
|
-
Process.fork do
|
55
|
-
begin
|
56
|
-
Worker.new(@runner_names, @agent_env_setup_cmd).launch
|
57
|
-
rescue Interrupt, SystemExit, SignalException
|
58
|
-
rescue Exception => e
|
59
|
-
DTR.error {"Worker is stopped by Exception => #{e.class.name}, message => #{e.message}"}
|
60
|
-
DTR.debug {e.backtrace.join("\n")}
|
61
|
-
end
|
62
|
-
end
|
53
|
+
DTR.kill_process worker
|
63
54
|
end
|
64
55
|
end
|
65
56
|
end
|