perfectqueue 0.7.14 → 0.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +5 -0
- data/lib/perfectqueue/backend/rdb.rb +43 -21
- data/lib/perfectqueue/version.rb +1 -1
- data/test/backend_test.rb +4 -4
- data/test/stress.rb +99 -0
- metadata +15 -13
data/ChangeLog
CHANGED
@@ -7,7 +7,9 @@ class RDBBackend < Backend
|
|
7
7
|
require 'sequel'
|
8
8
|
@uri = uri
|
9
9
|
@table = table
|
10
|
-
@db = Sequel.connect(@uri)
|
10
|
+
@db = Sequel.connect(@uri, :max_connections=>1)
|
11
|
+
@last_time = Time.now.to_i
|
12
|
+
@mutex = Mutex.new
|
11
13
|
#init_db(@uri.split('//',2)[0])
|
12
14
|
connect {
|
13
15
|
# connection test
|
@@ -16,15 +18,19 @@ class RDBBackend < Backend
|
|
16
18
|
SELECT id, timeout, data, created_at, resource
|
17
19
|
FROM `#{@table}`
|
18
20
|
LEFT JOIN (
|
19
|
-
SELECT resource AS res,
|
20
|
-
|
21
|
-
|
22
|
-
WHERE timeout > ? AND created_at IS NOT NULL
|
21
|
+
SELECT resource AS res, COUNT(1) AS running
|
22
|
+
FROM `#{@table}` AS T
|
23
|
+
WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
|
23
24
|
GROUP BY resource
|
24
|
-
) AS
|
25
|
+
) AS R ON resource = res
|
25
26
|
WHERE timeout <= ? AND (running IS NULL OR running < #{MAX_RESOURCE})
|
26
27
|
ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW}
|
27
28
|
SQL
|
29
|
+
# sqlite doesn't support SELECT ... FOR UPDATE but
|
30
|
+
# sqlite doesn't need it because the db is not shared
|
31
|
+
unless @uri.split('//',2)[0].to_s.include?('sqlite')
|
32
|
+
@sql << 'FOR UPDATE'
|
33
|
+
end
|
28
34
|
end
|
29
35
|
|
30
36
|
def create_tables
|
@@ -45,10 +51,29 @@ SQL
|
|
45
51
|
|
46
52
|
private
|
47
53
|
def connect(&block)
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
54
|
+
now = Time.now.to_i
|
55
|
+
@mutex.synchronize do
|
56
|
+
if now - @last_time > KEEPALIVE
|
57
|
+
@db.disconnect
|
58
|
+
end
|
59
|
+
@last_time = now
|
60
|
+
retry_count = 0
|
61
|
+
begin
|
62
|
+
block.call
|
63
|
+
rescue
|
64
|
+
# workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
|
65
|
+
err = ([$!] + $!.backtrace.map {|bt| " #{bt}" }).join("\n")
|
66
|
+
if $!.to_s.include?('try restarting transaction')
|
67
|
+
retry_count += 1
|
68
|
+
if retry_count < MAX_RETRY
|
69
|
+
STDERR.puts err + "\nretrying"
|
70
|
+
retry
|
71
|
+
else
|
72
|
+
STDERR.puts err + "\nabort"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
raise
|
76
|
+
end
|
52
77
|
end
|
53
78
|
end
|
54
79
|
|
@@ -59,24 +84,25 @@ SQL
|
|
59
84
|
}
|
60
85
|
end
|
61
86
|
|
62
|
-
MAX_SELECT_ROW =
|
87
|
+
MAX_SELECT_ROW = 8
|
63
88
|
MAX_RESOURCE = (ENV['PQ_MAX_RESOURCE'] || 4).to_i
|
89
|
+
KEEPALIVE = 10
|
90
|
+
MAX_RETRY = 10
|
64
91
|
|
65
92
|
def acquire(timeout, now=Time.now.to_i)
|
66
93
|
connect {
|
67
|
-
@db.
|
68
|
-
@db.run "LOCK TABLES `#{@table}` WRITE, T WRITE;" rescue nil # TODO mysql only
|
69
|
-
begin
|
94
|
+
@db.transaction do
|
70
95
|
while true
|
71
96
|
rows = 0
|
72
97
|
@db.fetch(@sql, now, now) {|row|
|
73
|
-
|
74
98
|
unless row[:created_at]
|
75
99
|
# finished/canceled task
|
76
100
|
@db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
|
77
101
|
|
78
102
|
else
|
79
|
-
|
103
|
+
## optimistic lock is not needed because the row is locked for update
|
104
|
+
#n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
|
105
|
+
n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", timeout, row[:id]].update
|
80
106
|
if n > 0
|
81
107
|
return row[:id], Task.new(row[:id], row[:created_at], row[:data], row[:resource])
|
82
108
|
end
|
@@ -84,12 +110,8 @@ SQL
|
|
84
110
|
|
85
111
|
rows += 1
|
86
112
|
}
|
87
|
-
if rows < MAX_SELECT_ROW
|
88
|
-
return nil
|
89
|
-
end
|
113
|
+
break nil if rows < MAX_SELECT_ROW
|
90
114
|
end
|
91
|
-
ensure
|
92
|
-
@db.run "UNLOCK TABLES T, `#{@table}`;" rescue nil # TODO mysql only
|
93
115
|
end
|
94
116
|
}
|
95
117
|
end
|
data/lib/perfectqueue/version.rb
CHANGED
data/test/backend_test.rb
CHANGED
@@ -171,13 +171,13 @@ class BackendTest < Test::Unit::TestCase
|
|
171
171
|
assert_not_equal nil, task
|
172
172
|
|
173
173
|
assert_nothing_raised do
|
174
|
-
db1.update(token, time+TIMEOUT)
|
174
|
+
db1.update(token, time+TIMEOUT+1)
|
175
175
|
end
|
176
176
|
|
177
177
|
token_, task_ = db2.acquire(time+TIMEOUT, time)
|
178
178
|
assert_equal nil, token_
|
179
179
|
|
180
|
-
token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT)
|
180
|
+
token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT+1)
|
181
181
|
assert_not_equal nil, task
|
182
182
|
assert_equal @key_prefix+'test1', task.id
|
183
183
|
assert_equal time, task.created_at
|
@@ -199,7 +199,7 @@ class BackendTest < Test::Unit::TestCase
|
|
199
199
|
assert_not_equal nil, task
|
200
200
|
|
201
201
|
assert_nothing_raised do
|
202
|
-
db1.update(token, time+TIMEOUT)
|
202
|
+
db1.update(token, time+TIMEOUT+1)
|
203
203
|
end
|
204
204
|
|
205
205
|
token_, task_ = db2.acquire(time+TIMEOUT, time)
|
@@ -224,7 +224,7 @@ class BackendTest < Test::Unit::TestCase
|
|
224
224
|
time = Time.now.to_i
|
225
225
|
|
226
226
|
5.times do |i|
|
227
|
-
ok = db1.submit(@key_prefix+'test'+i.to_s, 'data1', time, 'user1')
|
227
|
+
ok = db1.submit(@key_prefix+'test'+i.to_s, 'data1', time-i, 'user1')
|
228
228
|
assert_equal true, ok
|
229
229
|
end
|
230
230
|
ok = db1.submit(@key_prefix+'test5', 'data2', time, 'user2')
|
data/test/stress.rb
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
$LOAD_PATH << File.expand_path(File.dirname(__FILE__)+"/../lib")
|
2
|
+
require 'perfectqueue'
|
3
|
+
require 'perfectqueue/backend/rdb'
|
4
|
+
require 'perfectqueue/backend/simpledb'
|
5
|
+
|
6
|
+
class StressTest
|
7
|
+
def initialize(uri, table, npt, thread)
|
8
|
+
@db_proc = Proc.new do
|
9
|
+
PerfectQueue::RDBBackend.new(uri, table)
|
10
|
+
end
|
11
|
+
@db_proc.call.create_tables
|
12
|
+
@npt = npt
|
13
|
+
@thread = thread
|
14
|
+
end
|
15
|
+
|
16
|
+
class ThreadMain < Thread
|
17
|
+
def initialize(key_prefix, db, num, now)
|
18
|
+
@key_prefix = key_prefix
|
19
|
+
@db = db
|
20
|
+
@num = num
|
21
|
+
@now = now
|
22
|
+
super(&method(:run))
|
23
|
+
end
|
24
|
+
|
25
|
+
def run
|
26
|
+
@num.times {|i|
|
27
|
+
@db.submit("#{@key_prefix}-#{i}", "data", @now)
|
28
|
+
token, task = @db.acquire(@now+60)
|
29
|
+
if token == nil
|
30
|
+
puts "acquire failed"
|
31
|
+
next
|
32
|
+
end
|
33
|
+
@db.update(token, @now+70)
|
34
|
+
@db.finish(token, @now+80)
|
35
|
+
}
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def run
|
40
|
+
threads = []
|
41
|
+
key_prefix = "stress-#{'%08x'%rand(2**32)}"
|
42
|
+
now = Time.now
|
43
|
+
@thread.times {|i|
|
44
|
+
threads << ThreadMain.new("#{key_prefix}-#{i}", @db_proc.call, @npt, now.to_i)
|
45
|
+
}
|
46
|
+
threads.each {|t|
|
47
|
+
t.join
|
48
|
+
}
|
49
|
+
finish = Time.now
|
50
|
+
|
51
|
+
elapsed = finish - now
|
52
|
+
puts "#{elapsed} sec."
|
53
|
+
puts "#{@npt * @thread / elapsed} req/sec."
|
54
|
+
puts "#{elapsed / (@npt * @thread)} sec/req."
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
require 'optparse'
|
59
|
+
|
60
|
+
op = OptionParser.new
|
61
|
+
op.banner += " <uri> <table>"
|
62
|
+
|
63
|
+
num = 100
|
64
|
+
thread = 1
|
65
|
+
|
66
|
+
op.on('-n', '--num N', Integer) {|n|
|
67
|
+
num = n
|
68
|
+
}
|
69
|
+
op.on('-t', '--thread N', Integer) {|n|
|
70
|
+
thread = n
|
71
|
+
}
|
72
|
+
|
73
|
+
begin
|
74
|
+
op.parse!(ARGV)
|
75
|
+
|
76
|
+
if ARGV.length != 2
|
77
|
+
puts op.to_s
|
78
|
+
exit 1
|
79
|
+
end
|
80
|
+
|
81
|
+
uri = ARGV[0]
|
82
|
+
table = ARGV[1]
|
83
|
+
|
84
|
+
rescue
|
85
|
+
puts op.to_s
|
86
|
+
puts $!
|
87
|
+
exit 1
|
88
|
+
end
|
89
|
+
|
90
|
+
npt = num / thread
|
91
|
+
num = npt * thread
|
92
|
+
|
93
|
+
puts "num: #{num}"
|
94
|
+
puts "thread: #{thread}"
|
95
|
+
puts "num/thread: #{npt}"
|
96
|
+
|
97
|
+
t = StressTest.new(uri, table, npt, thread)
|
98
|
+
t.run
|
99
|
+
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.15
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-01-
|
12
|
+
date: 2012-01-11 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sequel
|
16
|
-
requirement: &
|
16
|
+
requirement: &13147240 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 3.26.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *13147240
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: aws-sdk
|
27
|
-
requirement: &
|
27
|
+
requirement: &13146400 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ~>
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
version: 1.1.1
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *13146400
|
36
36
|
description:
|
37
37
|
email: frsyuki@gmail.com
|
38
38
|
executables:
|
@@ -54,14 +54,15 @@ files:
|
|
54
54
|
- lib/perfectqueue/worker.rb
|
55
55
|
- ChangeLog
|
56
56
|
- README.rdoc
|
57
|
-
- test/backend_test.rb
|
58
57
|
- test/exec_test.rb
|
59
58
|
- test/test_helper.rb
|
59
|
+
- test/backend_test.rb
|
60
|
+
- test/stress.rb
|
61
|
+
- test/fail.sh
|
60
62
|
- test/cat.sh
|
61
63
|
- test/echo.sh
|
62
|
-
- test/fail.sh
|
63
|
-
- test/huge.sh
|
64
64
|
- test/success.sh
|
65
|
+
- test/huge.sh
|
65
66
|
homepage: https://github.com/treasure-data/perfectqueue
|
66
67
|
licenses: []
|
67
68
|
post_install_message:
|
@@ -82,16 +83,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
82
83
|
version: '0'
|
83
84
|
requirements: []
|
84
85
|
rubyforge_project:
|
85
|
-
rubygems_version: 1.8.
|
86
|
+
rubygems_version: 1.8.10
|
86
87
|
signing_key:
|
87
88
|
specification_version: 3
|
88
89
|
summary: Highly available distributed queue built on RDBMS or SimpleDB
|
89
90
|
test_files:
|
90
|
-
- test/backend_test.rb
|
91
91
|
- test/exec_test.rb
|
92
92
|
- test/test_helper.rb
|
93
|
+
- test/backend_test.rb
|
94
|
+
- test/stress.rb
|
95
|
+
- test/fail.sh
|
93
96
|
- test/cat.sh
|
94
97
|
- test/echo.sh
|
95
|
-
- test/fail.sh
|
96
|
-
- test/huge.sh
|
97
98
|
- test/success.sh
|
99
|
+
- test/huge.sh
|