perfectqueue 0.9.0 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +5 -0
- data/bin/stress +2 -1
- data/lib/perfectqueue/backend/rdb.rb +16 -6
- data/lib/perfectqueue/backend/rdb_compat.rb +19 -8
- data/lib/perfectqueue/version.rb +1 -1
- data/spec/rdb_backend_spec.rb +14 -0
- data/spec/rdb_compat_backend_spec.rb +12 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cec4a6907599be03eb95b756075c114c1a282218
|
4
|
+
data.tar.gz: 3cc969e794964142212690c24eef4aab1c45e847
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 99bbd684d883988af719392e15e37446eec1b7404d1c3a661875156fc4137ab2e6ddf090e706a0c13449a0fc17fcdba1e2d19d54fc3d8db17e018081f3cb4326
|
7
|
+
data.tar.gz: 8f2be0558cf06f6caf6664c7cf77aaddabb3470b632495bfdd0a8790cac72592072e58fd17051fb7a04459c8f5b369a77343f7d50135b8402e5c7ba40c52995d
|
data/ChangeLog
CHANGED
data/bin/stress
CHANGED
@@ -149,7 +149,7 @@ def insert(queue)
|
|
149
149
|
"params":{}}
|
150
150
|
begin
|
151
151
|
n = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
|
152
|
-
queue.submit("import.1/main.action_#{n}_#{rd.hex(
|
152
|
+
queue.submit("import.1/main.action_#{n}_#{rd.hex(20)}", 'user02', h, now: t)
|
153
153
|
rescue
|
154
154
|
p $!
|
155
155
|
sleep 1
|
@@ -165,6 +165,7 @@ def worker(queue)
|
|
165
165
|
t0 = t = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
|
166
166
|
begin
|
167
167
|
ary = queue.poll_multi(max_acquire: 11, now: t)
|
168
|
+
sleep 1
|
168
169
|
ary.each do |x|
|
169
170
|
x.finish!({})
|
170
171
|
end if ary
|
@@ -21,6 +21,8 @@ module PerfectQueue::Backend
|
|
21
21
|
host: u.host,
|
22
22
|
port: u.port ? u.port.to_i : 3306
|
23
23
|
}
|
24
|
+
@pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
|
25
|
+
options[:connect_timeout] = config.fetch(:connect_timeout, 3)
|
24
26
|
options[:sslca] = config[:sslca] if config[:sslca]
|
25
27
|
db_name = u.path.split('/')[1]
|
26
28
|
@db = Sequel.mysql2(db_name, options)
|
@@ -54,23 +56,31 @@ module PerfectQueue::Backend
|
|
54
56
|
end
|
55
57
|
|
56
58
|
private
|
57
|
-
def connect
|
59
|
+
def connect
|
60
|
+
tmax = Process.clock_gettime(Process::CLOCK_REALTIME, :second) + @pq_connect_timeout
|
58
61
|
@mutex.synchronize do
|
59
62
|
retry_count = 0
|
60
63
|
begin
|
61
|
-
|
64
|
+
yield
|
65
|
+
rescue Sequel::DatabaseConnectionError
|
66
|
+
if (retry_count += 1) < MAX_RETRY && tmax > Process.clock_gettime(Process::CLOCK_REALTIME, :second)
|
67
|
+
STDERR.puts "#{$!}\n retrying."
|
68
|
+
sleep 2
|
69
|
+
retry
|
70
|
+
end
|
71
|
+
STDERR.puts "#{$!}\n abort."
|
72
|
+
raise
|
62
73
|
rescue
|
63
74
|
# workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
|
64
75
|
if $!.to_s.include?('try restarting transaction')
|
65
|
-
err =
|
76
|
+
err = $!.backtrace.map{|bt| " #{bt}" }.unshift($!).join("\n")
|
66
77
|
retry_count += 1
|
67
78
|
if retry_count < MAX_RETRY
|
68
|
-
STDERR.puts err
|
79
|
+
STDERR.puts "#{err}\n retrying."
|
69
80
|
sleep 0.5
|
70
81
|
retry
|
71
|
-
else
|
72
|
-
STDERR.puts err + "\n abort."
|
73
82
|
end
|
83
|
+
STDERR.puts "#{err}\n abort."
|
74
84
|
end
|
75
85
|
raise
|
76
86
|
ensure
|
@@ -39,6 +39,7 @@ module PerfectQueue
|
|
39
39
|
def initialize(client, config)
|
40
40
|
super
|
41
41
|
|
42
|
+
@pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
|
42
43
|
url = config[:url]
|
43
44
|
@table = config[:table]
|
44
45
|
unless @table
|
@@ -46,7 +47,9 @@ module PerfectQueue
|
|
46
47
|
end
|
47
48
|
|
48
49
|
if /\Amysql2:/i =~ url
|
49
|
-
|
50
|
+
options = {max_connections: 1, sslca: config[:sslca]}
|
51
|
+
options[:connect_timeout] = config.fetch(:connect_timeout, 3)
|
52
|
+
@db = Sequel.connect(url, options)
|
50
53
|
if config.fetch(:use_connection_pooling, nil) != nil
|
51
54
|
@use_connection_pooling = !!config[:use_connection_pooling]
|
52
55
|
else
|
@@ -91,7 +94,7 @@ UPDATE `#{@table}`
|
|
91
94
|
FROM `#{@table}` FORCE INDEX (`index_#{@table}_on_timeout`)
|
92
95
|
WHERE #{EVENT_HORIZON} < timeout AND timeout <= :now
|
93
96
|
ORDER BY timeout ASC
|
94
|
-
LIMIT :max_acquire
|
97
|
+
LIMIT :max_acquire) AS t1 USING(id)
|
95
98
|
SET timeout=:next_timeout, owner=CONNECTION_ID()
|
96
99
|
SQL
|
97
100
|
@sql = <<SQL
|
@@ -110,12 +113,10 @@ UPDATE `#{@table}`
|
|
110
113
|
FROM `#{@table}` AS t1
|
111
114
|
WHERE timeout > :now AND resource IS NOT NULL
|
112
115
|
GROUP BY resource
|
113
|
-
FOR UPDATE
|
114
116
|
) AS t2 USING(resource)
|
115
117
|
WHERE #{EVENT_HORIZON} < timeout AND timeout <= :now AND IFNULL(max_running - running, 1) > 0
|
116
118
|
ORDER BY weight DESC, timeout ASC
|
117
119
|
LIMIT :max_acquire
|
118
|
-
FOR UPDATE
|
119
120
|
) AS t3 USING (id)
|
120
121
|
SET timeout = :next_timeout, owner = CONNECTION_ID()
|
121
122
|
SQL
|
@@ -257,6 +258,7 @@ SQL
|
|
257
258
|
if n <= 0
|
258
259
|
return nil
|
259
260
|
end
|
261
|
+
@table_unlock.call
|
260
262
|
|
261
263
|
tasks = []
|
262
264
|
@db.fetch(@sql, next_timeout) {|row|
|
@@ -329,7 +331,7 @@ SQL
|
|
329
331
|
end
|
330
332
|
|
331
333
|
protected
|
332
|
-
def connect_locked
|
334
|
+
def connect_locked
|
333
335
|
connect {
|
334
336
|
locked = false
|
335
337
|
|
@@ -339,7 +341,7 @@ SQL
|
|
339
341
|
locked = true
|
340
342
|
end
|
341
343
|
|
342
|
-
return
|
344
|
+
return yield
|
343
345
|
ensure
|
344
346
|
if @use_connection_pooling && locked
|
345
347
|
@table_unlock.call
|
@@ -348,16 +350,25 @@ SQL
|
|
348
350
|
}
|
349
351
|
end
|
350
352
|
|
351
|
-
def connect
|
353
|
+
def connect
|
352
354
|
now = Time.now.to_i
|
355
|
+
tmax = now + @pq_connect_timeout
|
353
356
|
@mutex.synchronize do
|
354
357
|
# keepalive_timeout
|
355
358
|
@db.disconnect if now - @last_time > KEEPALIVE
|
356
359
|
|
357
360
|
count = 0
|
358
361
|
begin
|
359
|
-
|
362
|
+
yield
|
360
363
|
@last_time = now
|
364
|
+
rescue Sequel::DatabaseConnectionError
|
365
|
+
if (count += 1) < MAX_RETRY && tmax > Time.now.to_i
|
366
|
+
STDERR.puts "#{$!}\n retrying."
|
367
|
+
sleep 2
|
368
|
+
retry
|
369
|
+
end
|
370
|
+
STDERR.puts "#{$!}\n abort."
|
371
|
+
raise
|
361
372
|
rescue
|
362
373
|
# workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
|
363
374
|
if $!.to_s.include?('try restarting transaction')
|
data/lib/perfectqueue/version.rb
CHANGED
data/spec/rdb_backend_spec.rb
CHANGED
@@ -69,5 +69,19 @@ describe Backend::RDBBackend do
|
|
69
69
|
end.to raise_error(RuntimeError)
|
70
70
|
end
|
71
71
|
end
|
72
|
+
context 'cannot connect' do
|
73
|
+
let (:uri){ 'mysql2://root:@nonexistent/perfectqueue_test' }
|
74
|
+
let (:db) do
|
75
|
+
Backend::RDBBackend.new(uri, table)
|
76
|
+
end
|
77
|
+
it 'raises Sequel::DatabaseConnectionError' do
|
78
|
+
allow(STDERR).to receive(:puts)
|
79
|
+
slept = 0
|
80
|
+
expect(db).to receive(:sleep).exactly(9).times{|n| slept += n }
|
81
|
+
expect(db.db).to receive(:connect).exactly(10).times.and_call_original
|
82
|
+
expect{ db.__send__(:connect){ db.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
|
83
|
+
expect(slept).to be < 30
|
84
|
+
end
|
85
|
+
end
|
72
86
|
end
|
73
87
|
end
|
@@ -338,6 +338,18 @@ describe Backend::RDBCompatBackend do
|
|
338
338
|
end
|
339
339
|
end.to raise_error(RuntimeError)
|
340
340
|
end
|
341
|
+
context 'cannot connect' do
|
342
|
+
let (:config){ {url: 'mysql2://root:@nonexistent/perfectqueue_test', table: table} }
|
343
|
+
it 'raises Sequel::DatabaseConnectionError' do
|
344
|
+
allow(STDERR).to receive(:puts)
|
345
|
+
d = Backend::RDBCompatBackend.new(client, config)
|
346
|
+
slept = 0
|
347
|
+
expect(d).to receive(:sleep).exactly(9).times{|n| slept += n }
|
348
|
+
expect(d.db).to receive(:connect).exactly(10).times.and_call_original
|
349
|
+
expect{ d.__send__(:connect){ d.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
|
350
|
+
expect(slept).to eq(18)
|
351
|
+
end
|
352
|
+
end
|
341
353
|
end
|
342
354
|
end
|
343
355
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sequel
|