rails_parallel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/README.markdown +33 -0
- data/Rakefile +2 -0
- data/bin/rails_parallel_worker +23 -0
- data/lib/rails_parallel/collector.rb +32 -0
- data/lib/rails_parallel/forks.rb +39 -0
- data/lib/rails_parallel/object_socket.rb +76 -0
- data/lib/rails_parallel/rake.rb +182 -0
- data/lib/rails_parallel/runner/child.rb +84 -0
- data/lib/rails_parallel/runner/parent.rb +179 -0
- data/lib/rails_parallel/runner/schema.rb +93 -0
- data/lib/rails_parallel/runner/test_runner.rb +17 -0
- data/lib/rails_parallel/runner.rb +38 -0
- data/lib/rails_parallel/timings.rb +32 -0
- data/lib/rails_parallel/version.rb +3 -0
- data/lib/rails_parallel.rb +3 -0
- data/rails_parallel.gemspec +25 -0
- metadata +131 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.markdown
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
rails_parallel
|
2
|
+
==============
|
3
|
+
|
4
|
+
rails_parallel makes your Rails tests scale with the number of CPU cores available.
|
5
|
+
|
6
|
+
It also speeds up the testing process in general, by making heavy use of forking to only have to load the Rails environment once.
|
7
|
+
|
8
|
+
Installation
|
9
|
+
------------
|
10
|
+
|
11
|
+
To load rails_parallel, require "rails_parallel/rake" early in your Rakefile. One possibility is to load it conditionally based on an environment variable:
|
12
|
+
|
13
|
+
require 'rails_parallel/rake' if ENV['PARALLEL']
|
14
|
+
|
15
|
+
You'll want to add a lib/tasks/rails_parallel.rake with at least the following:
|
16
|
+
|
17
|
+
# RailsParallel handles the DB schema.
|
18
|
+
Rake::Task['test:prepare'].clear_prerequisites if Object.const_get(:RailsParallel)
|
19
|
+
|
20
|
+
namespace :parallel do
|
21
|
+
# Run this task if you have non-test tasks to run first and you want the
|
22
|
+
# RailsParallel worker to start loading your environment earlier.
|
23
|
+
task :launch do
|
24
|
+
RailsParallel::Rake.launch
|
25
|
+
end
|
26
|
+
|
27
|
+
# RailsParallel runs this if it needs to reload the DB.
|
28
|
+
namespace :db do
|
29
|
+
task :setup => ['db:drop', 'db:create', 'db:schema:load']
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
This gem was designed as an internal project and currently makes certain assumptions about your project setup, such as the use of MySQL and a separate versioned schema (rather than db/schema.rb). These will become more generic in future versions.
|
data/Rakefile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
ENV['RAILS_ENV'] = 'test'
|
4
|
+
|
5
|
+
begin
|
6
|
+
puts 'RP: Loading RailsParallel.'
|
7
|
+
$LOAD_PATH << 'lib'
|
8
|
+
require 'rails_parallel/runner'
|
9
|
+
require 'rails_parallel/object_socket'
|
10
|
+
|
11
|
+
socket = ObjectSocket.new(IO.for_fd(ARGV.first.to_i))
|
12
|
+
socket << :started
|
13
|
+
|
14
|
+
puts 'RP: Loading Rails.'
|
15
|
+
require "#{ENV['RAILS_PARALLEL_ROOT']}/config/environment"
|
16
|
+
|
17
|
+
puts 'RP: Ready for testing.'
|
18
|
+
RailsParallel::Runner.launch(socket)
|
19
|
+
puts 'RP: Shutting down.'
|
20
|
+
Kernel.exit!(0)
|
21
|
+
rescue Interrupt, SignalException
|
22
|
+
Kernel.exit!(1)
|
23
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'test/unit/collector'
|
2
|
+
|
3
|
+
module RailsParallel
|
4
|
+
class Collector
|
5
|
+
include Test::Unit::Collector
|
6
|
+
|
7
|
+
NAME = 'collected from the ObjectSpace'
|
8
|
+
|
9
|
+
def prepare(timings, test_name)
|
10
|
+
@suites = {}
|
11
|
+
::ObjectSpace.each_object(Class) do |klass|
|
12
|
+
@suites[klass.name] = klass.suite if Test::Unit::TestCase > klass
|
13
|
+
end
|
14
|
+
|
15
|
+
@pending = @suites.keys.sort_by do |name|
|
16
|
+
[
|
17
|
+
0 - timings.fetch(test_name, name), # runtime, descending
|
18
|
+
0 - @suites[name].size, # no. of tests, descending
|
19
|
+
name
|
20
|
+
]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def next_suite
|
25
|
+
@pending.shift
|
26
|
+
end
|
27
|
+
|
28
|
+
def suite_for(name)
|
29
|
+
@suites[name]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module RailsParallel
|
2
|
+
module Forks
|
3
|
+
def fork_and_run
|
4
|
+
ActiveRecord::Base.connection.disconnect! if ActiveRecord::Base.connected?
|
5
|
+
|
6
|
+
fork do
|
7
|
+
begin
|
8
|
+
yield
|
9
|
+
Kernel.exit!(0)
|
10
|
+
rescue Interrupt, SignalException
|
11
|
+
Kernel.exit!(1)
|
12
|
+
rescue Exception => e
|
13
|
+
puts "Error: #{e}"
|
14
|
+
puts(*e.backtrace.map {|t| "\t#{t}"})
|
15
|
+
before_exit
|
16
|
+
Kernel.exit!(1)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def wait_for(pid, nonblock = false)
|
22
|
+
pid = Process.waitpid(pid, nonblock ? Process::WNOHANG : 0)
|
23
|
+
check_status($?) if pid
|
24
|
+
pid
|
25
|
+
end
|
26
|
+
|
27
|
+
def wait_any(nonblock = false)
|
28
|
+
wait_for(-1, nonblock)
|
29
|
+
end
|
30
|
+
|
31
|
+
def check_status(stat)
|
32
|
+
raise "error: #{stat.inspect}" unless stat.success?
|
33
|
+
end
|
34
|
+
|
35
|
+
def before_exit
|
36
|
+
# cleanup here (in children)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'socket'
|
3
|
+
|
4
|
+
class ObjectSocket
|
5
|
+
BLOCK_SIZE = 4096
|
6
|
+
|
7
|
+
attr_reader :socket
|
8
|
+
|
9
|
+
def self.pair
|
10
|
+
Socket.pair(Socket::AF_UNIX, Socket::SOCK_STREAM, 0).map { |s| new(s) }
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(socket)
|
14
|
+
@socket = socket
|
15
|
+
@buffer = ''
|
16
|
+
end
|
17
|
+
|
18
|
+
def nonblock=(val)
|
19
|
+
@nonblock = val
|
20
|
+
end
|
21
|
+
|
22
|
+
def close
|
23
|
+
@socket.close
|
24
|
+
end
|
25
|
+
|
26
|
+
def nonblocking(&block)
|
27
|
+
with_nonblock(true, &block)
|
28
|
+
end
|
29
|
+
def blocking(&block)
|
30
|
+
with_nonblock(false, &block)
|
31
|
+
end
|
32
|
+
|
33
|
+
def each_object(&block)
|
34
|
+
first = true
|
35
|
+
loop do
|
36
|
+
process_buffer(&block) if first
|
37
|
+
first = false
|
38
|
+
|
39
|
+
@buffer += @nonblock ? @socket.read_nonblock(BLOCK_SIZE) : @socket.readpartial(BLOCK_SIZE)
|
40
|
+
process_buffer(&block)
|
41
|
+
end
|
42
|
+
rescue Errno::EAGAIN
|
43
|
+
# end of nonblocking data
|
44
|
+
end
|
45
|
+
|
46
|
+
def next_object
|
47
|
+
each_object { |o| return o }
|
48
|
+
nil # no pending data in nonblock mode
|
49
|
+
end
|
50
|
+
|
51
|
+
def <<(obj)
|
52
|
+
data = Marshal.dump(obj)
|
53
|
+
@socket.syswrite [data.size, data].pack('Na*')
|
54
|
+
self # chainable
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def process_buffer
|
60
|
+
while @buffer.size >= 4
|
61
|
+
size = 4 + @buffer.unpack('N').first
|
62
|
+
break unless @buffer.size >= size
|
63
|
+
|
64
|
+
packet = @buffer.slice!(0, size)
|
65
|
+
yield Marshal.load(packet[4..-1])
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def with_nonblock(value)
|
70
|
+
old_value = @nonblock
|
71
|
+
@nonblock = value
|
72
|
+
return yield
|
73
|
+
ensure
|
74
|
+
@nonblock = old_value
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,182 @@
|
|
1
|
+
require 'rake/testtask'
|
2
|
+
require 'fcntl'
|
3
|
+
|
4
|
+
require 'rails_parallel/object_socket'
|
5
|
+
|
6
|
+
module RailsParallel
|
7
|
+
class Rake
|
8
|
+
include Singleton
|
9
|
+
|
10
|
+
SCHEMA_DIR = 'tmp/rails_parallel/schema'
|
11
|
+
|
12
|
+
def self.launch
|
13
|
+
instance.launch
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.run(name, ruby_opts, files)
|
17
|
+
instance.launch
|
18
|
+
instance.run(name, ruby_opts, files)
|
19
|
+
end
|
20
|
+
|
21
|
+
def launch
|
22
|
+
return if @pid
|
23
|
+
at_exit { shutdown }
|
24
|
+
|
25
|
+
create_test_db
|
26
|
+
|
27
|
+
my_socket, c_socket = ObjectSocket.pair
|
28
|
+
sock = c_socket.socket
|
29
|
+
sock.fcntl(Fcntl::F_SETFD, sock.fcntl(Fcntl::F_GETFD, 0) & ~Fcntl::FD_CLOEXEC)
|
30
|
+
|
31
|
+
@pid = fork do
|
32
|
+
my_socket.close
|
33
|
+
ENV['RAILS_PARALLEL_ROOT'] = Rails.root
|
34
|
+
exec('rails_parallel_worker', sock.fileno.to_s)
|
35
|
+
raise 'exec failed'
|
36
|
+
end
|
37
|
+
|
38
|
+
c_socket.close
|
39
|
+
@socket = my_socket
|
40
|
+
|
41
|
+
expect(:started)
|
42
|
+
end
|
43
|
+
|
44
|
+
def run(name, ruby_opts, files)
|
45
|
+
options = parse_options(ruby_opts)
|
46
|
+
schema = schema_file
|
47
|
+
|
48
|
+
expect(:ready)
|
49
|
+
@socket << {
|
50
|
+
:name => name,
|
51
|
+
:schema => schema,
|
52
|
+
:options => options,
|
53
|
+
:files => files.to_a
|
54
|
+
}
|
55
|
+
expect(:done)
|
56
|
+
end
|
57
|
+
|
58
|
+
def shutdown
|
59
|
+
if @pid
|
60
|
+
@socket << :shutdown
|
61
|
+
Process.waitpid(@pid)
|
62
|
+
@pid = nil
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def expect(want)
|
69
|
+
got = @socket.next_object
|
70
|
+
raise "Expected #{want}, got #{got}" unless want == got
|
71
|
+
end
|
72
|
+
|
73
|
+
def parse_options(ruby_opts)
|
74
|
+
ruby_opts.collect do |opt|
|
75
|
+
case opt
|
76
|
+
when /^-r/
|
77
|
+
[:require, $']
|
78
|
+
else
|
79
|
+
raise "Unhandled Ruby option: #{opt.inspect}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def create_test_db
|
85
|
+
dbconfig = YAML.load_file('config/database.yml')['test']
|
86
|
+
ActiveRecord::Base.establish_connection(dbconfig.merge('database' => nil))
|
87
|
+
ActiveRecord::Base.connection.execute("CREATE DATABASE IF NOT EXISTS #{dbconfig['database']}")
|
88
|
+
end
|
89
|
+
|
90
|
+
def schema_digest
|
91
|
+
files = FileList['db/schema.versioned.rb', 'db/migrate/*.rb'].sort
|
92
|
+
digest = Digest::MD5.new
|
93
|
+
files.each { |f| digest.update("#{f}|#{File.read(f)}|") }
|
94
|
+
digest.hexdigest
|
95
|
+
end
|
96
|
+
|
97
|
+
def schema_file
|
98
|
+
digest = schema_digest
|
99
|
+
basename = "#{digest}.sql"
|
100
|
+
schema = "#{SCHEMA_DIR}/#{basename}"
|
101
|
+
|
102
|
+
if File.exist? schema
|
103
|
+
puts "RP: Using cached schema: #{basename}"
|
104
|
+
else
|
105
|
+
puts 'RP: Building new schema ... '
|
106
|
+
|
107
|
+
silently { generate_schema(digest, schema) }
|
108
|
+
|
109
|
+
puts "RP: Generated new schema: #{basename}"
|
110
|
+
end
|
111
|
+
|
112
|
+
schema
|
113
|
+
end
|
114
|
+
|
115
|
+
def silently
|
116
|
+
File.open('/dev/null', 'w') do |fh|
|
117
|
+
$stdout = $stderr = fh
|
118
|
+
yield
|
119
|
+
end
|
120
|
+
ensure
|
121
|
+
$stdout = STDOUT
|
122
|
+
$stderr = STDERR
|
123
|
+
end
|
124
|
+
|
125
|
+
def generate_schema(digest, schema)
|
126
|
+
FileUtils.mkdir_p(SCHEMA_DIR)
|
127
|
+
Tempfile.open(["#{digest}.", ".sql"], SCHEMA_DIR) do |file|
|
128
|
+
::Rake::Task['parallel:db:setup'].invoke
|
129
|
+
sh "mysqldump --no-data -u root shopify_dev > #{file.path}"
|
130
|
+
|
131
|
+
raise 'No schema dumped' unless file.size > 0
|
132
|
+
File.rename(file.path, schema)
|
133
|
+
$schema_dump_file = nil
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
module Rake
|
140
|
+
class TestTask
|
141
|
+
@@patched = false
|
142
|
+
|
143
|
+
def initialize(name=:test)
|
144
|
+
if name.kind_of? Hash
|
145
|
+
@name = name.keys.first
|
146
|
+
@depends = name.values.first
|
147
|
+
else
|
148
|
+
@name = name
|
149
|
+
@depends = []
|
150
|
+
end
|
151
|
+
@full_name = [Rake.application.current_scope, @name].join(':')
|
152
|
+
|
153
|
+
@libs = ["lib"]
|
154
|
+
@pattern = nil
|
155
|
+
@options = nil
|
156
|
+
@test_files = nil
|
157
|
+
@verbose = false
|
158
|
+
@warning = false
|
159
|
+
@loader = :rake
|
160
|
+
@ruby_opts = []
|
161
|
+
yield self if block_given?
|
162
|
+
@pattern = 'test/test*.rb' if @pattern.nil? && @test_files.nil?
|
163
|
+
|
164
|
+
if !@@patched && self.class.name == 'TestTaskWithoutDescription'
|
165
|
+
TestTaskWithoutDescription.class_eval { def define; super(false); end }
|
166
|
+
@@patched = true
|
167
|
+
end
|
168
|
+
|
169
|
+
define
|
170
|
+
end
|
171
|
+
|
172
|
+
def define(describe = true)
|
173
|
+
lib_path = @libs.join(File::PATH_SEPARATOR)
|
174
|
+
desc "Run tests" + (@full_name == :test ? "" : " for #{@name}") if describe
|
175
|
+
task @name => @depends do
|
176
|
+
files = file_list.map {|f| f =~ /[\*\?\[\]]/ ? FileList[f] : f }.flatten(1)
|
177
|
+
RailsParallel::Rake.run(@full_name, ruby_opts, files)
|
178
|
+
end
|
179
|
+
self
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'rails_parallel/object_socket'
|
2
|
+
require 'rails_parallel/runner/test_runner'
|
3
|
+
|
4
|
+
module RailsParallel
|
5
|
+
class Runner
|
6
|
+
class Child
|
7
|
+
include Forks
|
8
|
+
|
9
|
+
attr_reader :socket, :pid, :number, :last_suite, :last_time
|
10
|
+
|
11
|
+
def initialize(number, schema, collector)
|
12
|
+
@number = number
|
13
|
+
@schema = schema
|
14
|
+
@collector = collector
|
15
|
+
@buffer = ''
|
16
|
+
@state = :waiting
|
17
|
+
end
|
18
|
+
|
19
|
+
def launch
|
20
|
+
parent_socket, child_socket = ObjectSocket.pair
|
21
|
+
|
22
|
+
@pid = fork_and_run do
|
23
|
+
parent_socket.close
|
24
|
+
@socket = child_socket
|
25
|
+
main_loop
|
26
|
+
end
|
27
|
+
|
28
|
+
child_socket.close
|
29
|
+
@socket = parent_socket
|
30
|
+
@socket.nonblock = true
|
31
|
+
end
|
32
|
+
|
33
|
+
def run_suite(name)
|
34
|
+
@last_suite = name
|
35
|
+
@last_time = Time.now
|
36
|
+
@socket << name
|
37
|
+
end
|
38
|
+
|
39
|
+
def finish
|
40
|
+
@socket << :finish
|
41
|
+
end
|
42
|
+
|
43
|
+
def close
|
44
|
+
@socket.close
|
45
|
+
end
|
46
|
+
|
47
|
+
def kill
|
48
|
+
Process.kill('KILL', @pid)
|
49
|
+
close rescue nil
|
50
|
+
end
|
51
|
+
|
52
|
+
def socket
|
53
|
+
@socket.socket
|
54
|
+
end
|
55
|
+
|
56
|
+
def poll
|
57
|
+
output = []
|
58
|
+
@socket.each_object { |obj| output << obj }
|
59
|
+
output
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def main_loop
|
65
|
+
@schema.load_db(@number)
|
66
|
+
|
67
|
+
@socket << :started << :ready
|
68
|
+
|
69
|
+
@socket.each_object do |obj|
|
70
|
+
break if obj == :finish
|
71
|
+
|
72
|
+
($rp_suites ||= []) << obj
|
73
|
+
suite = @collector.suite_for(obj)
|
74
|
+
runner = TestRunner.new(suite)
|
75
|
+
runner.start
|
76
|
+
|
77
|
+
@socket << [obj, runner.result, runner.faults] << :ready
|
78
|
+
end
|
79
|
+
|
80
|
+
@socket << :finished
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require 'rails_parallel/forks'
|
2
|
+
require 'rails_parallel/collector'
|
3
|
+
require 'rails_parallel/timings'
|
4
|
+
require 'rails_parallel/runner/child'
|
5
|
+
require 'rails_parallel/runner/schema'
|
6
|
+
require 'rails_parallel/runner/test_runner'
|
7
|
+
|
8
|
+
class Test::Unit::TestResult
|
9
|
+
attr_reader :failures, :errors
|
10
|
+
|
11
|
+
def append(other)
|
12
|
+
@run_count += other.run_count
|
13
|
+
@assertion_count += other.assertion_count
|
14
|
+
@failures += other.failures
|
15
|
+
@errors += other.errors
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
module RailsParallel
|
20
|
+
class Runner
|
21
|
+
class Parent
|
22
|
+
include Forks
|
23
|
+
|
24
|
+
def initialize(params)
|
25
|
+
@name = params[:name]
|
26
|
+
@schema = Schema.new(params[:schema])
|
27
|
+
@options = params[:options]
|
28
|
+
@files = params[:files]
|
29
|
+
@max_children = number_of_workers
|
30
|
+
|
31
|
+
@timings = Timings.new
|
32
|
+
|
33
|
+
@children = []
|
34
|
+
@launched = 0
|
35
|
+
@by_pid = {}
|
36
|
+
@by_socket = {}
|
37
|
+
|
38
|
+
@result = Test::Unit::TestResult.new
|
39
|
+
@faults = {}
|
40
|
+
end
|
41
|
+
|
42
|
+
def run
|
43
|
+
@schema.load_main_db
|
44
|
+
|
45
|
+
pid = fork_and_run do
|
46
|
+
status "RP: Preparing #{@name} ... "
|
47
|
+
handle_options
|
48
|
+
prepare
|
49
|
+
puts "ready."
|
50
|
+
|
51
|
+
puts "RP: Running #{@name}."
|
52
|
+
start = Time.now
|
53
|
+
begin
|
54
|
+
launch_next_child
|
55
|
+
monitor
|
56
|
+
ensure
|
57
|
+
@children.each(&:kill)
|
58
|
+
output_result(Time.now - start)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
wait_for(pid)
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def status(msg)
|
67
|
+
$stdout.print(msg)
|
68
|
+
$stdout.flush
|
69
|
+
end
|
70
|
+
|
71
|
+
def handle_options
|
72
|
+
@options.each do |opt, value|
|
73
|
+
case opt
|
74
|
+
when :require
|
75
|
+
value = 'rubygems' if value == 'ubygems'
|
76
|
+
status "#{value}, "
|
77
|
+
require value
|
78
|
+
else
|
79
|
+
raise "Unknown option type: #{opt}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def prepare
|
85
|
+
status "#{@files.count} test files ... "
|
86
|
+
@files.each { |f| load f }
|
87
|
+
@collector = Collector.new
|
88
|
+
@collector.prepare(@timings, @name)
|
89
|
+
end
|
90
|
+
|
91
|
+
def launch_next_child
|
92
|
+
return if @launched >= @max_children
|
93
|
+
return if @complete
|
94
|
+
|
95
|
+
child = Child.new(@launched += 1, @schema, @collector)
|
96
|
+
child.launch
|
97
|
+
|
98
|
+
@children << child
|
99
|
+
@by_pid[child.pid] = child
|
100
|
+
@by_socket[child.socket] = child
|
101
|
+
end
|
102
|
+
|
103
|
+
def monitor
|
104
|
+
until @children.empty?
|
105
|
+
watching = @children.map(&:socket)
|
106
|
+
IO.select(watching).first.each do |socket|
|
107
|
+
child = @by_socket[socket]
|
108
|
+
|
109
|
+
begin
|
110
|
+
child.poll.each do |packet|
|
111
|
+
case packet
|
112
|
+
when :started
|
113
|
+
launch_next_child
|
114
|
+
when :ready
|
115
|
+
@timings.record(@name, child.last_suite, Time.now - child.last_time) if child.last_suite
|
116
|
+
|
117
|
+
suite = @collector.next_suite
|
118
|
+
if suite
|
119
|
+
child.run_suite(suite)
|
120
|
+
else
|
121
|
+
@complete = true
|
122
|
+
child.finish
|
123
|
+
end
|
124
|
+
when :finished
|
125
|
+
close_child(child)
|
126
|
+
else
|
127
|
+
suite, result, faults = packet
|
128
|
+
@result.append(result)
|
129
|
+
@faults[suite] = faults
|
130
|
+
end
|
131
|
+
end
|
132
|
+
rescue EOFError
|
133
|
+
close_child(child)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
while pid = wait_any(true)
|
138
|
+
child = @by_pid[pid]
|
139
|
+
close_child(child) if child
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def close_child(child)
|
145
|
+
child.close rescue nil
|
146
|
+
@children.delete(child)
|
147
|
+
@by_socket.delete(child.socket)
|
148
|
+
@by_pid.delete(child.pid)
|
149
|
+
end
|
150
|
+
|
151
|
+
def output_result(elapsed)
|
152
|
+
runner = TestRunner.new(nil, Test::Unit::UI::NORMAL)
|
153
|
+
runner.result = @result
|
154
|
+
runner.faults = @faults.sort.map(&:last).flatten(1)
|
155
|
+
|
156
|
+
runner.output_report(elapsed)
|
157
|
+
end
|
158
|
+
|
159
|
+
def number_of_workers
|
160
|
+
workers = number_of_cores
|
161
|
+
workers -= 1 if workers > 4 # reserve one core for DB
|
162
|
+
workers
|
163
|
+
end
|
164
|
+
|
165
|
+
def number_of_cores
|
166
|
+
if RUBY_PLATFORM =~ /linux/
|
167
|
+
cores = File.read('/proc/cpuinfo').split("\n\n").map do |data|
|
168
|
+
values = data.split("\n").map { |line| line.split(/\s*:/, 2) }
|
169
|
+
attrs = Hash[*values.flatten]
|
170
|
+
['physical id', 'core id'].map { |key| attrs[key] }.join("/")
|
171
|
+
end
|
172
|
+
cores.uniq.count
|
173
|
+
elsif RUBY_PLATFORM =~ /darwin/
|
174
|
+
`/usr/bin/hwprefs cpu_count`.to_i
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
require 'rails_parallel/object_socket'
|
2
|
+
require 'rails_parallel/runner/test_runner'
|
3
|
+
|
4
|
+
module RailsParallel
|
5
|
+
class Runner
|
6
|
+
class Schema
|
7
|
+
include Forks
|
8
|
+
|
9
|
+
def initialize(file)
|
10
|
+
@file = file
|
11
|
+
end
|
12
|
+
|
13
|
+
def load_main_db
|
14
|
+
if load_db(1)
|
15
|
+
failed = 0
|
16
|
+
ObjectSpace.each_object(Class) do |klass|
|
17
|
+
next unless klass < ActiveRecord::Base
|
18
|
+
|
19
|
+
klass.reset_column_information
|
20
|
+
begin
|
21
|
+
klass.columns
|
22
|
+
rescue StandardError => e
|
23
|
+
failed += 1
|
24
|
+
raise e if failed > 3
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def load_db(number)
|
31
|
+
update_db_config(number)
|
32
|
+
if schema_loaded?
|
33
|
+
reconnect
|
34
|
+
false
|
35
|
+
else
|
36
|
+
schema_load
|
37
|
+
true
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def reconnect(override = {})
|
44
|
+
ActiveRecord::Base.establish_connection(@dbconfig.merge(override))
|
45
|
+
ActiveRecord::Base.connection
|
46
|
+
end
|
47
|
+
|
48
|
+
def update_db_config(number)
|
49
|
+
config = ActiveRecord::Base.configurations[Rails.env]
|
50
|
+
config['database'] += "_#{number}" unless number == 1
|
51
|
+
@dbconfig = config.with_indifferent_access
|
52
|
+
end
|
53
|
+
|
54
|
+
def schema_load
|
55
|
+
dbname = @dbconfig[:database]
|
56
|
+
mysql_args = ['-u', 'root']
|
57
|
+
|
58
|
+
connection = reconnect(:database => nil)
|
59
|
+
connection.execute("DROP DATABASE IF EXISTS #{dbname}")
|
60
|
+
connection.execute("CREATE DATABASE #{dbname}")
|
61
|
+
|
62
|
+
File.open(@file) do |fh|
|
63
|
+
pid = fork do
|
64
|
+
STDIN.reopen(fh)
|
65
|
+
exec(*['mysql', mysql_args, dbname].flatten)
|
66
|
+
end
|
67
|
+
wait_for(pid)
|
68
|
+
end
|
69
|
+
|
70
|
+
reconnect
|
71
|
+
sm_table = ActiveRecord::Migrator.schema_migrations_table_name
|
72
|
+
ActiveRecord::Base.connection.execute("INSERT INTO #{sm_table} (version) VALUES ('#{@file}')")
|
73
|
+
end
|
74
|
+
|
75
|
+
def schema_loaded?
|
76
|
+
begin
|
77
|
+
ActiveRecord::Base.establish_connection(@dbconfig)
|
78
|
+
ActiveRecord::Base.connection
|
79
|
+
rescue StandardError
|
80
|
+
return false
|
81
|
+
end
|
82
|
+
|
83
|
+
begin
|
84
|
+
sm_table = ActiveRecord::Migrator.schema_migrations_table_name
|
85
|
+
migrated = ActiveRecord::Base.connection.select_values("SELECT version FROM #{sm_table}")
|
86
|
+
migrated.include?(@file)
|
87
|
+
rescue ActiveRecord::StatementInvalid
|
88
|
+
false
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'test/unit/ui/console/testrunner'
|
2
|
+
|
3
|
+
module RailsParallel
|
4
|
+
class Runner
|
5
|
+
class TestRunner < Test::Unit::UI::Console::TestRunner
|
6
|
+
attr_accessor :result, :faults
|
7
|
+
|
8
|
+
def initialize(suite, output_level = Test::Unit::UI::PROGRESS_ONLY)
|
9
|
+
super(suite, output_level)
|
10
|
+
end
|
11
|
+
|
12
|
+
def output_report(elapsed)
|
13
|
+
finished(elapsed)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'rails_parallel/runner/parent'
|
2
|
+
require 'rails_parallel/object_socket'
|
3
|
+
|
4
|
+
module RailsParallel
|
5
|
+
class Runner
|
6
|
+
def self.launch(socket)
|
7
|
+
Runner.new(socket).run
|
8
|
+
end
|
9
|
+
|
10
|
+
def initialize(socket)
|
11
|
+
@socket = socket
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
prepare
|
16
|
+
|
17
|
+
@socket << :ready
|
18
|
+
@socket.each_object do |obj|
|
19
|
+
break if obj == :shutdown
|
20
|
+
run_suite(obj)
|
21
|
+
@socket << :done << :ready
|
22
|
+
end
|
23
|
+
rescue EOFError
|
24
|
+
# shutdown
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def prepare
|
30
|
+
$LOAD_PATH << 'test'
|
31
|
+
require 'test_helper'
|
32
|
+
end
|
33
|
+
|
34
|
+
def run_suite(params)
|
35
|
+
Parent.new(params).run
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'redis'
|
3
|
+
require 'active_support/core_ext/enumerable'
|
4
|
+
|
5
|
+
module RailsParallel
|
6
|
+
class Timings
|
7
|
+
TIMING_COUNT = 10
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@cache = Redis.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def record(test_name, class_name, time)
|
14
|
+
key = key_for(test_name, class_name)
|
15
|
+
@cache.lpush(key, time)
|
16
|
+
@cache.ltrim(key, 0, TIMING_COUNT - 1)
|
17
|
+
end
|
18
|
+
|
19
|
+
def fetch(test_name, class_name)
|
20
|
+
key = key_for(test_name, class_name)
|
21
|
+
times = @cache.lrange(key, 0, TIMING_COUNT - 1).map(&:to_f)
|
22
|
+
return 0 if times.empty?
|
23
|
+
times.sum / times.count
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def key_for(test_name, class_name)
|
29
|
+
"timings-#{test_name}-#{class_name}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "rails_parallel/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "rails_parallel"
|
7
|
+
s.version = RailsParallel::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Adrian Irving-Beer"]
|
10
|
+
s.email = ["adrian@shopify.com"]
|
11
|
+
s.homepage = ""
|
12
|
+
s.summary = %q{Runs multiple Rails tests concurrently}
|
13
|
+
s.description = %q{rails_parallel runs your Rails tests by forking off a worker and running multiple tests concurrently. It makes heavy use of forking to reduce memory footprint (assuming copy-on-write), only loads your Rails environment once, and automatically scales to the number of cores available. Designed to work with MySQL only. For best results, run MySQL on a tmpfs or a RAM disk.}
|
14
|
+
|
15
|
+
s.rubyforge_project = "rails_parallel"
|
16
|
+
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
+
s.require_paths = ["lib"]
|
21
|
+
|
22
|
+
s.add_dependency 'redis', '~> 2.2.0'
|
23
|
+
s.add_dependency 'rails', '~> 3.0'
|
24
|
+
s.add_dependency 'rake', '~> 0.9.2'
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rails_parallel
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 27
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 0
|
10
|
+
version: 0.1.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Adrian Irving-Beer
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-08-05 00:00:00 -04:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: redis
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 7
|
30
|
+
segments:
|
31
|
+
- 2
|
32
|
+
- 2
|
33
|
+
- 0
|
34
|
+
version: 2.2.0
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rails
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 7
|
46
|
+
segments:
|
47
|
+
- 3
|
48
|
+
- 0
|
49
|
+
version: "3.0"
|
50
|
+
type: :runtime
|
51
|
+
version_requirements: *id002
|
52
|
+
- !ruby/object:Gem::Dependency
|
53
|
+
name: rake
|
54
|
+
prerelease: false
|
55
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
56
|
+
none: false
|
57
|
+
requirements:
|
58
|
+
- - ~>
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
hash: 63
|
61
|
+
segments:
|
62
|
+
- 0
|
63
|
+
- 9
|
64
|
+
- 2
|
65
|
+
version: 0.9.2
|
66
|
+
type: :runtime
|
67
|
+
version_requirements: *id003
|
68
|
+
description: rails_parallel runs your Rails tests by forking off a worker and running multiple tests concurrently. It makes heavy use of forking to reduce memory footprint (assuming copy-on-write), only loads your Rails environment once, and automatically scales to the number of cores available. Designed to work with MySQL only. For best results, run MySQL on a tmpfs or a RAM disk.
|
69
|
+
email:
|
70
|
+
- adrian@shopify.com
|
71
|
+
executables:
|
72
|
+
- rails_parallel_worker
|
73
|
+
extensions: []
|
74
|
+
|
75
|
+
extra_rdoc_files: []
|
76
|
+
|
77
|
+
files:
|
78
|
+
- .gitignore
|
79
|
+
- Gemfile
|
80
|
+
- README.markdown
|
81
|
+
- Rakefile
|
82
|
+
- bin/rails_parallel_worker
|
83
|
+
- lib/rails_parallel.rb
|
84
|
+
- lib/rails_parallel/collector.rb
|
85
|
+
- lib/rails_parallel/forks.rb
|
86
|
+
- lib/rails_parallel/object_socket.rb
|
87
|
+
- lib/rails_parallel/rake.rb
|
88
|
+
- lib/rails_parallel/runner.rb
|
89
|
+
- lib/rails_parallel/runner/child.rb
|
90
|
+
- lib/rails_parallel/runner/parent.rb
|
91
|
+
- lib/rails_parallel/runner/schema.rb
|
92
|
+
- lib/rails_parallel/runner/test_runner.rb
|
93
|
+
- lib/rails_parallel/timings.rb
|
94
|
+
- lib/rails_parallel/version.rb
|
95
|
+
- rails_parallel.gemspec
|
96
|
+
has_rdoc: true
|
97
|
+
homepage: ""
|
98
|
+
licenses: []
|
99
|
+
|
100
|
+
post_install_message:
|
101
|
+
rdoc_options: []
|
102
|
+
|
103
|
+
require_paths:
|
104
|
+
- lib
|
105
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
106
|
+
none: false
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
hash: 3
|
111
|
+
segments:
|
112
|
+
- 0
|
113
|
+
version: "0"
|
114
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
115
|
+
none: false
|
116
|
+
requirements:
|
117
|
+
- - ">="
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
hash: 3
|
120
|
+
segments:
|
121
|
+
- 0
|
122
|
+
version: "0"
|
123
|
+
requirements: []
|
124
|
+
|
125
|
+
rubyforge_project: rails_parallel
|
126
|
+
rubygems_version: 1.3.7
|
127
|
+
signing_key:
|
128
|
+
specification_version: 3
|
129
|
+
summary: Runs multiple Rails tests concurrently
|
130
|
+
test_files: []
|
131
|
+
|