rails_parallel 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/README.markdown +33 -0
- data/Rakefile +2 -0
- data/bin/rails_parallel_worker +23 -0
- data/lib/rails_parallel/collector.rb +32 -0
- data/lib/rails_parallel/forks.rb +39 -0
- data/lib/rails_parallel/object_socket.rb +76 -0
- data/lib/rails_parallel/rake.rb +182 -0
- data/lib/rails_parallel/runner/child.rb +84 -0
- data/lib/rails_parallel/runner/parent.rb +179 -0
- data/lib/rails_parallel/runner/schema.rb +93 -0
- data/lib/rails_parallel/runner/test_runner.rb +17 -0
- data/lib/rails_parallel/runner.rb +38 -0
- data/lib/rails_parallel/timings.rb +32 -0
- data/lib/rails_parallel/version.rb +3 -0
- data/lib/rails_parallel.rb +3 -0
- data/rails_parallel.gemspec +25 -0
- metadata +131 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.markdown
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
rails_parallel
|
2
|
+
==============
|
3
|
+
|
4
|
+
rails_parallel makes your Rails tests scale with the number of CPU cores available.
|
5
|
+
|
6
|
+
It also speeds up the testing process in general, by making heavy use of forking to only have to load the Rails environment once.
|
7
|
+
|
8
|
+
Installation
|
9
|
+
------------
|
10
|
+
|
11
|
+
To load rails_parallel, require "rails_parallel/rake" early in your Rakefile. One possibility is to load it conditionally based on an environment variable:
|
12
|
+
|
13
|
+
require 'rails_parallel/rake' if ENV['PARALLEL']
|
14
|
+
|
15
|
+
You'll want to add a lib/tasks/rails_parallel.rake with at least the following:
|
16
|
+
|
17
|
+
# RailsParallel handles the DB schema.
|
18
|
+
Rake::Task['test:prepare'].clear_prerequisites if Object.const_get(:RailsParallel)
|
19
|
+
|
20
|
+
namespace :parallel do
|
21
|
+
# Run this task if you have non-test tasks to run first and you want the
|
22
|
+
# RailsParallel worker to start loading your environment earlier.
|
23
|
+
task :launch do
|
24
|
+
RailsParallel::Rake.launch
|
25
|
+
end
|
26
|
+
|
27
|
+
# RailsParallel runs this if it needs to reload the DB.
|
28
|
+
namespace :db do
|
29
|
+
task :setup => ['db:drop', 'db:create', 'db:schema:load']
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
This gem was designed as an internal project and currently makes certain assumptions about your project setup, such as the use of MySQL and a separate versioned schema (rather than db/schema.rb). These will become more generic in future versions.
|
data/Rakefile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
ENV['RAILS_ENV'] = 'test'
|
4
|
+
|
5
|
+
begin
|
6
|
+
puts 'RP: Loading RailsParallel.'
|
7
|
+
$LOAD_PATH << 'lib'
|
8
|
+
require 'rails_parallel/runner'
|
9
|
+
require 'rails_parallel/object_socket'
|
10
|
+
|
11
|
+
socket = ObjectSocket.new(IO.for_fd(ARGV.first.to_i))
|
12
|
+
socket << :started
|
13
|
+
|
14
|
+
puts 'RP: Loading Rails.'
|
15
|
+
require "#{ENV['RAILS_PARALLEL_ROOT']}/config/environment"
|
16
|
+
|
17
|
+
puts 'RP: Ready for testing.'
|
18
|
+
RailsParallel::Runner.launch(socket)
|
19
|
+
puts 'RP: Shutting down.'
|
20
|
+
Kernel.exit!(0)
|
21
|
+
rescue Interrupt, SignalException
|
22
|
+
Kernel.exit!(1)
|
23
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'test/unit/collector'
|
2
|
+
|
3
|
+
module RailsParallel
|
4
|
+
class Collector
|
5
|
+
include Test::Unit::Collector
|
6
|
+
|
7
|
+
NAME = 'collected from the ObjectSpace'
|
8
|
+
|
9
|
+
def prepare(timings, test_name)
|
10
|
+
@suites = {}
|
11
|
+
::ObjectSpace.each_object(Class) do |klass|
|
12
|
+
@suites[klass.name] = klass.suite if Test::Unit::TestCase > klass
|
13
|
+
end
|
14
|
+
|
15
|
+
@pending = @suites.keys.sort_by do |name|
|
16
|
+
[
|
17
|
+
0 - timings.fetch(test_name, name), # runtime, descending
|
18
|
+
0 - @suites[name].size, # no. of tests, descending
|
19
|
+
name
|
20
|
+
]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def next_suite
|
25
|
+
@pending.shift
|
26
|
+
end
|
27
|
+
|
28
|
+
def suite_for(name)
|
29
|
+
@suites[name]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module RailsParallel
|
2
|
+
module Forks
|
3
|
+
def fork_and_run
|
4
|
+
ActiveRecord::Base.connection.disconnect! if ActiveRecord::Base.connected?
|
5
|
+
|
6
|
+
fork do
|
7
|
+
begin
|
8
|
+
yield
|
9
|
+
Kernel.exit!(0)
|
10
|
+
rescue Interrupt, SignalException
|
11
|
+
Kernel.exit!(1)
|
12
|
+
rescue Exception => e
|
13
|
+
puts "Error: #{e}"
|
14
|
+
puts(*e.backtrace.map {|t| "\t#{t}"})
|
15
|
+
before_exit
|
16
|
+
Kernel.exit!(1)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def wait_for(pid, nonblock = false)
|
22
|
+
pid = Process.waitpid(pid, nonblock ? Process::WNOHANG : 0)
|
23
|
+
check_status($?) if pid
|
24
|
+
pid
|
25
|
+
end
|
26
|
+
|
27
|
+
def wait_any(nonblock = false)
|
28
|
+
wait_for(-1, nonblock)
|
29
|
+
end
|
30
|
+
|
31
|
+
def check_status(stat)
|
32
|
+
raise "error: #{stat.inspect}" unless stat.success?
|
33
|
+
end
|
34
|
+
|
35
|
+
def before_exit
|
36
|
+
# cleanup here (in children)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'socket'
|
3
|
+
|
4
|
+
class ObjectSocket
|
5
|
+
BLOCK_SIZE = 4096
|
6
|
+
|
7
|
+
attr_reader :socket
|
8
|
+
|
9
|
+
def self.pair
|
10
|
+
Socket.pair(Socket::AF_UNIX, Socket::SOCK_STREAM, 0).map { |s| new(s) }
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(socket)
|
14
|
+
@socket = socket
|
15
|
+
@buffer = ''
|
16
|
+
end
|
17
|
+
|
18
|
+
def nonblock=(val)
|
19
|
+
@nonblock = val
|
20
|
+
end
|
21
|
+
|
22
|
+
def close
|
23
|
+
@socket.close
|
24
|
+
end
|
25
|
+
|
26
|
+
def nonblocking(&block)
|
27
|
+
with_nonblock(true, &block)
|
28
|
+
end
|
29
|
+
def blocking(&block)
|
30
|
+
with_nonblock(false, &block)
|
31
|
+
end
|
32
|
+
|
33
|
+
def each_object(&block)
|
34
|
+
first = true
|
35
|
+
loop do
|
36
|
+
process_buffer(&block) if first
|
37
|
+
first = false
|
38
|
+
|
39
|
+
@buffer += @nonblock ? @socket.read_nonblock(BLOCK_SIZE) : @socket.readpartial(BLOCK_SIZE)
|
40
|
+
process_buffer(&block)
|
41
|
+
end
|
42
|
+
rescue Errno::EAGAIN
|
43
|
+
# end of nonblocking data
|
44
|
+
end
|
45
|
+
|
46
|
+
def next_object
|
47
|
+
each_object { |o| return o }
|
48
|
+
nil # no pending data in nonblock mode
|
49
|
+
end
|
50
|
+
|
51
|
+
def <<(obj)
|
52
|
+
data = Marshal.dump(obj)
|
53
|
+
@socket.syswrite [data.size, data].pack('Na*')
|
54
|
+
self # chainable
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def process_buffer
|
60
|
+
while @buffer.size >= 4
|
61
|
+
size = 4 + @buffer.unpack('N').first
|
62
|
+
break unless @buffer.size >= size
|
63
|
+
|
64
|
+
packet = @buffer.slice!(0, size)
|
65
|
+
yield Marshal.load(packet[4..-1])
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def with_nonblock(value)
|
70
|
+
old_value = @nonblock
|
71
|
+
@nonblock = value
|
72
|
+
return yield
|
73
|
+
ensure
|
74
|
+
@nonblock = old_value
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,182 @@
|
|
1
|
+
require 'rake/testtask'
|
2
|
+
require 'fcntl'
|
3
|
+
|
4
|
+
require 'rails_parallel/object_socket'
|
5
|
+
|
6
|
+
module RailsParallel
|
7
|
+
class Rake
|
8
|
+
include Singleton
|
9
|
+
|
10
|
+
SCHEMA_DIR = 'tmp/rails_parallel/schema'
|
11
|
+
|
12
|
+
def self.launch
|
13
|
+
instance.launch
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.run(name, ruby_opts, files)
|
17
|
+
instance.launch
|
18
|
+
instance.run(name, ruby_opts, files)
|
19
|
+
end
|
20
|
+
|
21
|
+
def launch
|
22
|
+
return if @pid
|
23
|
+
at_exit { shutdown }
|
24
|
+
|
25
|
+
create_test_db
|
26
|
+
|
27
|
+
my_socket, c_socket = ObjectSocket.pair
|
28
|
+
sock = c_socket.socket
|
29
|
+
sock.fcntl(Fcntl::F_SETFD, sock.fcntl(Fcntl::F_GETFD, 0) & ~Fcntl::FD_CLOEXEC)
|
30
|
+
|
31
|
+
@pid = fork do
|
32
|
+
my_socket.close
|
33
|
+
ENV['RAILS_PARALLEL_ROOT'] = Rails.root
|
34
|
+
exec('rails_parallel_worker', sock.fileno.to_s)
|
35
|
+
raise 'exec failed'
|
36
|
+
end
|
37
|
+
|
38
|
+
c_socket.close
|
39
|
+
@socket = my_socket
|
40
|
+
|
41
|
+
expect(:started)
|
42
|
+
end
|
43
|
+
|
44
|
+
def run(name, ruby_opts, files)
|
45
|
+
options = parse_options(ruby_opts)
|
46
|
+
schema = schema_file
|
47
|
+
|
48
|
+
expect(:ready)
|
49
|
+
@socket << {
|
50
|
+
:name => name,
|
51
|
+
:schema => schema,
|
52
|
+
:options => options,
|
53
|
+
:files => files.to_a
|
54
|
+
}
|
55
|
+
expect(:done)
|
56
|
+
end
|
57
|
+
|
58
|
+
def shutdown
|
59
|
+
if @pid
|
60
|
+
@socket << :shutdown
|
61
|
+
Process.waitpid(@pid)
|
62
|
+
@pid = nil
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def expect(want)
|
69
|
+
got = @socket.next_object
|
70
|
+
raise "Expected #{want}, got #{got}" unless want == got
|
71
|
+
end
|
72
|
+
|
73
|
+
def parse_options(ruby_opts)
|
74
|
+
ruby_opts.collect do |opt|
|
75
|
+
case opt
|
76
|
+
when /^-r/
|
77
|
+
[:require, $']
|
78
|
+
else
|
79
|
+
raise "Unhandled Ruby option: #{opt.inspect}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def create_test_db
|
85
|
+
dbconfig = YAML.load_file('config/database.yml')['test']
|
86
|
+
ActiveRecord::Base.establish_connection(dbconfig.merge('database' => nil))
|
87
|
+
ActiveRecord::Base.connection.execute("CREATE DATABASE IF NOT EXISTS #{dbconfig['database']}")
|
88
|
+
end
|
89
|
+
|
90
|
+
def schema_digest
|
91
|
+
files = FileList['db/schema.versioned.rb', 'db/migrate/*.rb'].sort
|
92
|
+
digest = Digest::MD5.new
|
93
|
+
files.each { |f| digest.update("#{f}|#{File.read(f)}|") }
|
94
|
+
digest.hexdigest
|
95
|
+
end
|
96
|
+
|
97
|
+
def schema_file
|
98
|
+
digest = schema_digest
|
99
|
+
basename = "#{digest}.sql"
|
100
|
+
schema = "#{SCHEMA_DIR}/#{basename}"
|
101
|
+
|
102
|
+
if File.exist? schema
|
103
|
+
puts "RP: Using cached schema: #{basename}"
|
104
|
+
else
|
105
|
+
puts 'RP: Building new schema ... '
|
106
|
+
|
107
|
+
silently { generate_schema(digest, schema) }
|
108
|
+
|
109
|
+
puts "RP: Generated new schema: #{basename}"
|
110
|
+
end
|
111
|
+
|
112
|
+
schema
|
113
|
+
end
|
114
|
+
|
115
|
+
def silently
|
116
|
+
File.open('/dev/null', 'w') do |fh|
|
117
|
+
$stdout = $stderr = fh
|
118
|
+
yield
|
119
|
+
end
|
120
|
+
ensure
|
121
|
+
$stdout = STDOUT
|
122
|
+
$stderr = STDERR
|
123
|
+
end
|
124
|
+
|
125
|
+
def generate_schema(digest, schema)
|
126
|
+
FileUtils.mkdir_p(SCHEMA_DIR)
|
127
|
+
Tempfile.open(["#{digest}.", ".sql"], SCHEMA_DIR) do |file|
|
128
|
+
::Rake::Task['parallel:db:setup'].invoke
|
129
|
+
sh "mysqldump --no-data -u root shopify_dev > #{file.path}"
|
130
|
+
|
131
|
+
raise 'No schema dumped' unless file.size > 0
|
132
|
+
File.rename(file.path, schema)
|
133
|
+
$schema_dump_file = nil
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
module Rake
|
140
|
+
class TestTask
|
141
|
+
@@patched = false
|
142
|
+
|
143
|
+
def initialize(name=:test)
|
144
|
+
if name.kind_of? Hash
|
145
|
+
@name = name.keys.first
|
146
|
+
@depends = name.values.first
|
147
|
+
else
|
148
|
+
@name = name
|
149
|
+
@depends = []
|
150
|
+
end
|
151
|
+
@full_name = [Rake.application.current_scope, @name].join(':')
|
152
|
+
|
153
|
+
@libs = ["lib"]
|
154
|
+
@pattern = nil
|
155
|
+
@options = nil
|
156
|
+
@test_files = nil
|
157
|
+
@verbose = false
|
158
|
+
@warning = false
|
159
|
+
@loader = :rake
|
160
|
+
@ruby_opts = []
|
161
|
+
yield self if block_given?
|
162
|
+
@pattern = 'test/test*.rb' if @pattern.nil? && @test_files.nil?
|
163
|
+
|
164
|
+
if !@@patched && self.class.name == 'TestTaskWithoutDescription'
|
165
|
+
TestTaskWithoutDescription.class_eval { def define; super(false); end }
|
166
|
+
@@patched = true
|
167
|
+
end
|
168
|
+
|
169
|
+
define
|
170
|
+
end
|
171
|
+
|
172
|
+
def define(describe = true)
|
173
|
+
lib_path = @libs.join(File::PATH_SEPARATOR)
|
174
|
+
desc "Run tests" + (@full_name == :test ? "" : " for #{@name}") if describe
|
175
|
+
task @name => @depends do
|
176
|
+
files = file_list.map {|f| f =~ /[\*\?\[\]]/ ? FileList[f] : f }.flatten(1)
|
177
|
+
RailsParallel::Rake.run(@full_name, ruby_opts, files)
|
178
|
+
end
|
179
|
+
self
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'rails_parallel/object_socket'
|
2
|
+
require 'rails_parallel/runner/test_runner'
|
3
|
+
|
4
|
+
module RailsParallel
|
5
|
+
class Runner
|
6
|
+
class Child
|
7
|
+
include Forks
|
8
|
+
|
9
|
+
attr_reader :socket, :pid, :number, :last_suite, :last_time
|
10
|
+
|
11
|
+
def initialize(number, schema, collector)
|
12
|
+
@number = number
|
13
|
+
@schema = schema
|
14
|
+
@collector = collector
|
15
|
+
@buffer = ''
|
16
|
+
@state = :waiting
|
17
|
+
end
|
18
|
+
|
19
|
+
def launch
|
20
|
+
parent_socket, child_socket = ObjectSocket.pair
|
21
|
+
|
22
|
+
@pid = fork_and_run do
|
23
|
+
parent_socket.close
|
24
|
+
@socket = child_socket
|
25
|
+
main_loop
|
26
|
+
end
|
27
|
+
|
28
|
+
child_socket.close
|
29
|
+
@socket = parent_socket
|
30
|
+
@socket.nonblock = true
|
31
|
+
end
|
32
|
+
|
33
|
+
def run_suite(name)
|
34
|
+
@last_suite = name
|
35
|
+
@last_time = Time.now
|
36
|
+
@socket << name
|
37
|
+
end
|
38
|
+
|
39
|
+
def finish
|
40
|
+
@socket << :finish
|
41
|
+
end
|
42
|
+
|
43
|
+
def close
|
44
|
+
@socket.close
|
45
|
+
end
|
46
|
+
|
47
|
+
def kill
|
48
|
+
Process.kill('KILL', @pid)
|
49
|
+
close rescue nil
|
50
|
+
end
|
51
|
+
|
52
|
+
def socket
|
53
|
+
@socket.socket
|
54
|
+
end
|
55
|
+
|
56
|
+
def poll
|
57
|
+
output = []
|
58
|
+
@socket.each_object { |obj| output << obj }
|
59
|
+
output
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def main_loop
|
65
|
+
@schema.load_db(@number)
|
66
|
+
|
67
|
+
@socket << :started << :ready
|
68
|
+
|
69
|
+
@socket.each_object do |obj|
|
70
|
+
break if obj == :finish
|
71
|
+
|
72
|
+
($rp_suites ||= []) << obj
|
73
|
+
suite = @collector.suite_for(obj)
|
74
|
+
runner = TestRunner.new(suite)
|
75
|
+
runner.start
|
76
|
+
|
77
|
+
@socket << [obj, runner.result, runner.faults] << :ready
|
78
|
+
end
|
79
|
+
|
80
|
+
@socket << :finished
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require 'rails_parallel/forks'
|
2
|
+
require 'rails_parallel/collector'
|
3
|
+
require 'rails_parallel/timings'
|
4
|
+
require 'rails_parallel/runner/child'
|
5
|
+
require 'rails_parallel/runner/schema'
|
6
|
+
require 'rails_parallel/runner/test_runner'
|
7
|
+
|
8
|
+
class Test::Unit::TestResult
|
9
|
+
attr_reader :failures, :errors
|
10
|
+
|
11
|
+
def append(other)
|
12
|
+
@run_count += other.run_count
|
13
|
+
@assertion_count += other.assertion_count
|
14
|
+
@failures += other.failures
|
15
|
+
@errors += other.errors
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
module RailsParallel
|
20
|
+
class Runner
|
21
|
+
class Parent
|
22
|
+
include Forks
|
23
|
+
|
24
|
+
def initialize(params)
|
25
|
+
@name = params[:name]
|
26
|
+
@schema = Schema.new(params[:schema])
|
27
|
+
@options = params[:options]
|
28
|
+
@files = params[:files]
|
29
|
+
@max_children = number_of_workers
|
30
|
+
|
31
|
+
@timings = Timings.new
|
32
|
+
|
33
|
+
@children = []
|
34
|
+
@launched = 0
|
35
|
+
@by_pid = {}
|
36
|
+
@by_socket = {}
|
37
|
+
|
38
|
+
@result = Test::Unit::TestResult.new
|
39
|
+
@faults = {}
|
40
|
+
end
|
41
|
+
|
42
|
+
def run
|
43
|
+
@schema.load_main_db
|
44
|
+
|
45
|
+
pid = fork_and_run do
|
46
|
+
status "RP: Preparing #{@name} ... "
|
47
|
+
handle_options
|
48
|
+
prepare
|
49
|
+
puts "ready."
|
50
|
+
|
51
|
+
puts "RP: Running #{@name}."
|
52
|
+
start = Time.now
|
53
|
+
begin
|
54
|
+
launch_next_child
|
55
|
+
monitor
|
56
|
+
ensure
|
57
|
+
@children.each(&:kill)
|
58
|
+
output_result(Time.now - start)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
wait_for(pid)
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def status(msg)
|
67
|
+
$stdout.print(msg)
|
68
|
+
$stdout.flush
|
69
|
+
end
|
70
|
+
|
71
|
+
def handle_options
|
72
|
+
@options.each do |opt, value|
|
73
|
+
case opt
|
74
|
+
when :require
|
75
|
+
value = 'rubygems' if value == 'ubygems'
|
76
|
+
status "#{value}, "
|
77
|
+
require value
|
78
|
+
else
|
79
|
+
raise "Unknown option type: #{opt}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def prepare
|
85
|
+
status "#{@files.count} test files ... "
|
86
|
+
@files.each { |f| load f }
|
87
|
+
@collector = Collector.new
|
88
|
+
@collector.prepare(@timings, @name)
|
89
|
+
end
|
90
|
+
|
91
|
+
def launch_next_child
|
92
|
+
return if @launched >= @max_children
|
93
|
+
return if @complete
|
94
|
+
|
95
|
+
child = Child.new(@launched += 1, @schema, @collector)
|
96
|
+
child.launch
|
97
|
+
|
98
|
+
@children << child
|
99
|
+
@by_pid[child.pid] = child
|
100
|
+
@by_socket[child.socket] = child
|
101
|
+
end
|
102
|
+
|
103
|
+
def monitor
|
104
|
+
until @children.empty?
|
105
|
+
watching = @children.map(&:socket)
|
106
|
+
IO.select(watching).first.each do |socket|
|
107
|
+
child = @by_socket[socket]
|
108
|
+
|
109
|
+
begin
|
110
|
+
child.poll.each do |packet|
|
111
|
+
case packet
|
112
|
+
when :started
|
113
|
+
launch_next_child
|
114
|
+
when :ready
|
115
|
+
@timings.record(@name, child.last_suite, Time.now - child.last_time) if child.last_suite
|
116
|
+
|
117
|
+
suite = @collector.next_suite
|
118
|
+
if suite
|
119
|
+
child.run_suite(suite)
|
120
|
+
else
|
121
|
+
@complete = true
|
122
|
+
child.finish
|
123
|
+
end
|
124
|
+
when :finished
|
125
|
+
close_child(child)
|
126
|
+
else
|
127
|
+
suite, result, faults = packet
|
128
|
+
@result.append(result)
|
129
|
+
@faults[suite] = faults
|
130
|
+
end
|
131
|
+
end
|
132
|
+
rescue EOFError
|
133
|
+
close_child(child)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
while pid = wait_any(true)
|
138
|
+
child = @by_pid[pid]
|
139
|
+
close_child(child) if child
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def close_child(child)
|
145
|
+
child.close rescue nil
|
146
|
+
@children.delete(child)
|
147
|
+
@by_socket.delete(child.socket)
|
148
|
+
@by_pid.delete(child.pid)
|
149
|
+
end
|
150
|
+
|
151
|
+
def output_result(elapsed)
|
152
|
+
runner = TestRunner.new(nil, Test::Unit::UI::NORMAL)
|
153
|
+
runner.result = @result
|
154
|
+
runner.faults = @faults.sort.map(&:last).flatten(1)
|
155
|
+
|
156
|
+
runner.output_report(elapsed)
|
157
|
+
end
|
158
|
+
|
159
|
+
def number_of_workers
|
160
|
+
workers = number_of_cores
|
161
|
+
workers -= 1 if workers > 4 # reserve one core for DB
|
162
|
+
workers
|
163
|
+
end
|
164
|
+
|
165
|
+
def number_of_cores
|
166
|
+
if RUBY_PLATFORM =~ /linux/
|
167
|
+
cores = File.read('/proc/cpuinfo').split("\n\n").map do |data|
|
168
|
+
values = data.split("\n").map { |line| line.split(/\s*:/, 2) }
|
169
|
+
attrs = Hash[*values.flatten]
|
170
|
+
['physical id', 'core id'].map { |key| attrs[key] }.join("/")
|
171
|
+
end
|
172
|
+
cores.uniq.count
|
173
|
+
elsif RUBY_PLATFORM =~ /darwin/
|
174
|
+
`/usr/bin/hwprefs cpu_count`.to_i
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
require 'rails_parallel/object_socket'
|
2
|
+
require 'rails_parallel/runner/test_runner'
|
3
|
+
|
4
|
+
module RailsParallel
|
5
|
+
class Runner
|
6
|
+
class Schema
|
7
|
+
include Forks
|
8
|
+
|
9
|
+
def initialize(file)
|
10
|
+
@file = file
|
11
|
+
end
|
12
|
+
|
13
|
+
def load_main_db
|
14
|
+
if load_db(1)
|
15
|
+
failed = 0
|
16
|
+
ObjectSpace.each_object(Class) do |klass|
|
17
|
+
next unless klass < ActiveRecord::Base
|
18
|
+
|
19
|
+
klass.reset_column_information
|
20
|
+
begin
|
21
|
+
klass.columns
|
22
|
+
rescue StandardError => e
|
23
|
+
failed += 1
|
24
|
+
raise e if failed > 3
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def load_db(number)
|
31
|
+
update_db_config(number)
|
32
|
+
if schema_loaded?
|
33
|
+
reconnect
|
34
|
+
false
|
35
|
+
else
|
36
|
+
schema_load
|
37
|
+
true
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def reconnect(override = {})
|
44
|
+
ActiveRecord::Base.establish_connection(@dbconfig.merge(override))
|
45
|
+
ActiveRecord::Base.connection
|
46
|
+
end
|
47
|
+
|
48
|
+
def update_db_config(number)
|
49
|
+
config = ActiveRecord::Base.configurations[Rails.env]
|
50
|
+
config['database'] += "_#{number}" unless number == 1
|
51
|
+
@dbconfig = config.with_indifferent_access
|
52
|
+
end
|
53
|
+
|
54
|
+
def schema_load
|
55
|
+
dbname = @dbconfig[:database]
|
56
|
+
mysql_args = ['-u', 'root']
|
57
|
+
|
58
|
+
connection = reconnect(:database => nil)
|
59
|
+
connection.execute("DROP DATABASE IF EXISTS #{dbname}")
|
60
|
+
connection.execute("CREATE DATABASE #{dbname}")
|
61
|
+
|
62
|
+
File.open(@file) do |fh|
|
63
|
+
pid = fork do
|
64
|
+
STDIN.reopen(fh)
|
65
|
+
exec(*['mysql', mysql_args, dbname].flatten)
|
66
|
+
end
|
67
|
+
wait_for(pid)
|
68
|
+
end
|
69
|
+
|
70
|
+
reconnect
|
71
|
+
sm_table = ActiveRecord::Migrator.schema_migrations_table_name
|
72
|
+
ActiveRecord::Base.connection.execute("INSERT INTO #{sm_table} (version) VALUES ('#{@file}')")
|
73
|
+
end
|
74
|
+
|
75
|
+
def schema_loaded?
|
76
|
+
begin
|
77
|
+
ActiveRecord::Base.establish_connection(@dbconfig)
|
78
|
+
ActiveRecord::Base.connection
|
79
|
+
rescue StandardError
|
80
|
+
return false
|
81
|
+
end
|
82
|
+
|
83
|
+
begin
|
84
|
+
sm_table = ActiveRecord::Migrator.schema_migrations_table_name
|
85
|
+
migrated = ActiveRecord::Base.connection.select_values("SELECT version FROM #{sm_table}")
|
86
|
+
migrated.include?(@file)
|
87
|
+
rescue ActiveRecord::StatementInvalid
|
88
|
+
false
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'test/unit/ui/console/testrunner'
|
2
|
+
|
3
|
+
module RailsParallel
|
4
|
+
class Runner
|
5
|
+
class TestRunner < Test::Unit::UI::Console::TestRunner
|
6
|
+
attr_accessor :result, :faults
|
7
|
+
|
8
|
+
def initialize(suite, output_level = Test::Unit::UI::PROGRESS_ONLY)
|
9
|
+
super(suite, output_level)
|
10
|
+
end
|
11
|
+
|
12
|
+
def output_report(elapsed)
|
13
|
+
finished(elapsed)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'rails_parallel/runner/parent'
|
2
|
+
require 'rails_parallel/object_socket'
|
3
|
+
|
4
|
+
module RailsParallel
|
5
|
+
class Runner
|
6
|
+
def self.launch(socket)
|
7
|
+
Runner.new(socket).run
|
8
|
+
end
|
9
|
+
|
10
|
+
def initialize(socket)
|
11
|
+
@socket = socket
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
prepare
|
16
|
+
|
17
|
+
@socket << :ready
|
18
|
+
@socket.each_object do |obj|
|
19
|
+
break if obj == :shutdown
|
20
|
+
run_suite(obj)
|
21
|
+
@socket << :done << :ready
|
22
|
+
end
|
23
|
+
rescue EOFError
|
24
|
+
# shutdown
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def prepare
|
30
|
+
$LOAD_PATH << 'test'
|
31
|
+
require 'test_helper'
|
32
|
+
end
|
33
|
+
|
34
|
+
def run_suite(params)
|
35
|
+
Parent.new(params).run
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'redis'
|
3
|
+
require 'active_support/core_ext/enumerable'
|
4
|
+
|
5
|
+
module RailsParallel
|
6
|
+
class Timings
|
7
|
+
TIMING_COUNT = 10
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@cache = Redis.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def record(test_name, class_name, time)
|
14
|
+
key = key_for(test_name, class_name)
|
15
|
+
@cache.lpush(key, time)
|
16
|
+
@cache.ltrim(key, 0, TIMING_COUNT - 1)
|
17
|
+
end
|
18
|
+
|
19
|
+
def fetch(test_name, class_name)
|
20
|
+
key = key_for(test_name, class_name)
|
21
|
+
times = @cache.lrange(key, 0, TIMING_COUNT - 1).map(&:to_f)
|
22
|
+
return 0 if times.empty?
|
23
|
+
times.sum / times.count
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def key_for(test_name, class_name)
|
29
|
+
"timings-#{test_name}-#{class_name}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "rails_parallel/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "rails_parallel"
|
7
|
+
s.version = RailsParallel::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Adrian Irving-Beer"]
|
10
|
+
s.email = ["adrian@shopify.com"]
|
11
|
+
s.homepage = ""
|
12
|
+
s.summary = %q{Runs multiple Rails tests concurrently}
|
13
|
+
s.description = %q{rails_parallel runs your Rails tests by forking off a worker and running multiple tests concurrently. It makes heavy use of forking to reduce memory footprint (assuming copy-on-write), only loads your Rails environment once, and automatically scales to the number of cores available. Designed to work with MySQL only. For best results, run MySQL on a tmpfs or a RAM disk.}
|
14
|
+
|
15
|
+
s.rubyforge_project = "rails_parallel"
|
16
|
+
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
+
s.require_paths = ["lib"]
|
21
|
+
|
22
|
+
s.add_dependency 'redis', '~> 2.2.0'
|
23
|
+
s.add_dependency 'rails', '~> 3.0'
|
24
|
+
s.add_dependency 'rake', '~> 0.9.2'
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rails_parallel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 27
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 0
|
10
|
+
version: 0.1.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Adrian Irving-Beer
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-08-05 00:00:00 -04:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: redis
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 7
|
30
|
+
segments:
|
31
|
+
- 2
|
32
|
+
- 2
|
33
|
+
- 0
|
34
|
+
version: 2.2.0
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rails
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 7
|
46
|
+
segments:
|
47
|
+
- 3
|
48
|
+
- 0
|
49
|
+
version: "3.0"
|
50
|
+
type: :runtime
|
51
|
+
version_requirements: *id002
|
52
|
+
- !ruby/object:Gem::Dependency
|
53
|
+
name: rake
|
54
|
+
prerelease: false
|
55
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
56
|
+
none: false
|
57
|
+
requirements:
|
58
|
+
- - ~>
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
hash: 63
|
61
|
+
segments:
|
62
|
+
- 0
|
63
|
+
- 9
|
64
|
+
- 2
|
65
|
+
version: 0.9.2
|
66
|
+
type: :runtime
|
67
|
+
version_requirements: *id003
|
68
|
+
description: rails_parallel runs your Rails tests by forking off a worker and running multiple tests concurrently. It makes heavy use of forking to reduce memory footprint (assuming copy-on-write), only loads your Rails environment once, and automatically scales to the number of cores available. Designed to work with MySQL only. For best results, run MySQL on a tmpfs or a RAM disk.
|
69
|
+
email:
|
70
|
+
- adrian@shopify.com
|
71
|
+
executables:
|
72
|
+
- rails_parallel_worker
|
73
|
+
extensions: []
|
74
|
+
|
75
|
+
extra_rdoc_files: []
|
76
|
+
|
77
|
+
files:
|
78
|
+
- .gitignore
|
79
|
+
- Gemfile
|
80
|
+
- README.markdown
|
81
|
+
- Rakefile
|
82
|
+
- bin/rails_parallel_worker
|
83
|
+
- lib/rails_parallel.rb
|
84
|
+
- lib/rails_parallel/collector.rb
|
85
|
+
- lib/rails_parallel/forks.rb
|
86
|
+
- lib/rails_parallel/object_socket.rb
|
87
|
+
- lib/rails_parallel/rake.rb
|
88
|
+
- lib/rails_parallel/runner.rb
|
89
|
+
- lib/rails_parallel/runner/child.rb
|
90
|
+
- lib/rails_parallel/runner/parent.rb
|
91
|
+
- lib/rails_parallel/runner/schema.rb
|
92
|
+
- lib/rails_parallel/runner/test_runner.rb
|
93
|
+
- lib/rails_parallel/timings.rb
|
94
|
+
- lib/rails_parallel/version.rb
|
95
|
+
- rails_parallel.gemspec
|
96
|
+
has_rdoc: true
|
97
|
+
homepage: ""
|
98
|
+
licenses: []
|
99
|
+
|
100
|
+
post_install_message:
|
101
|
+
rdoc_options: []
|
102
|
+
|
103
|
+
require_paths:
|
104
|
+
- lib
|
105
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
106
|
+
none: false
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
hash: 3
|
111
|
+
segments:
|
112
|
+
- 0
|
113
|
+
version: "0"
|
114
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
115
|
+
none: false
|
116
|
+
requirements:
|
117
|
+
- - ">="
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
hash: 3
|
120
|
+
segments:
|
121
|
+
- 0
|
122
|
+
version: "0"
|
123
|
+
requirements: []
|
124
|
+
|
125
|
+
rubyforge_project: rails_parallel
|
126
|
+
rubygems_version: 1.3.7
|
127
|
+
signing_key:
|
128
|
+
specification_version: 3
|
129
|
+
summary: Runs multiple Rails tests concurrently
|
130
|
+
test_files: []
|
131
|
+
|