perfectqueue 0.7.7 → 0.7.8
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +5 -0
- data/lib/perfectqueue/backend.rb +4 -3
- data/lib/perfectqueue/backend/null.rb +1 -1
- data/lib/perfectqueue/backend/rdb.rb +43 -47
- data/lib/perfectqueue/backend/simpledb.rb +6 -3
- data/lib/perfectqueue/command/perfectqueue.rb +4 -4
- data/lib/perfectqueue/version.rb +1 -1
- data/test/backend_test.rb +32 -2
- data/test/test_helper.rb +2 -0
- metadata +7 -7
data/ChangeLog
CHANGED
data/lib/perfectqueue/backend.rb
CHANGED
@@ -3,13 +3,14 @@ module PerfectQueue
|
|
3
3
|
|
4
4
|
|
5
5
|
class Task
|
6
|
-
def initialize(id, created_at, data)
|
6
|
+
def initialize(id, created_at, data, resource=nil)
|
7
7
|
@id = id
|
8
8
|
@created_at = created_at
|
9
9
|
@data = data
|
10
|
+
@resource = resource
|
10
11
|
end
|
11
12
|
|
12
|
-
attr_reader :id, :created_at, :data
|
13
|
+
attr_reader :id, :created_at, :data, :resource
|
13
14
|
end
|
14
15
|
|
15
16
|
|
@@ -39,7 +40,7 @@ class Backend
|
|
39
40
|
end
|
40
41
|
|
41
42
|
# => true (success) or nil (already exists)
|
42
|
-
def submit(id, data, time=Time.now.to_i)
|
43
|
+
def submit(id, data, time=Time.now.to_i, resource=nil)
|
43
44
|
end
|
44
45
|
|
45
46
|
def close
|
@@ -14,33 +14,23 @@ class RDBBackend < Backend
|
|
14
14
|
}
|
15
15
|
end
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
# sql << " timeout INT NOT NULL,"
|
33
|
-
# sql << " data BLOB NOT NULL,"
|
34
|
-
# sql << " created_at INT,"
|
35
|
-
# sql << " PRIMARY KEY (id)"
|
36
|
-
# sql << ");"
|
37
|
-
# end
|
38
|
-
# # TODO index
|
39
|
-
# connect {
|
40
|
-
# @db.run sql
|
41
|
-
# }
|
42
|
-
#end
|
17
|
+
def create_tables
|
18
|
+
sql = ''
|
19
|
+
sql << "CREATE TABLE IF NOT EXISTS `#{@table}` ("
|
20
|
+
sql << " id VARCHAR(256) NOT NULL,"
|
21
|
+
sql << " timeout INT NOT NULL,"
|
22
|
+
sql << " data BLOB NOT NULL,"
|
23
|
+
sql << " created_at INT,"
|
24
|
+
sql << " resource VARCHAR(256),"
|
25
|
+
sql << " PRIMARY KEY (id)"
|
26
|
+
sql << ");"
|
27
|
+
# TODO index
|
28
|
+
connect {
|
29
|
+
@db.run sql
|
30
|
+
}
|
31
|
+
end
|
43
32
|
|
33
|
+
private
|
44
34
|
def connect(&block)
|
45
35
|
begin
|
46
36
|
block.call
|
@@ -51,35 +41,41 @@ class RDBBackend < Backend
|
|
51
41
|
|
52
42
|
public
|
53
43
|
def list(&block)
|
54
|
-
@db.fetch("SELECT id, timeout, data, created_at FROM `#{@table}` WHERE created_at IS NOT NULL ORDER BY timeout ASC;") {|row|
|
55
|
-
yield row[:id], row[:created_at], row[:data], row[:timeout]
|
44
|
+
@db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` WHERE created_at IS NOT NULL ORDER BY timeout ASC;") {|row|
|
45
|
+
yield row[:id], row[:created_at], row[:data], row[:timeout], row[:resource]
|
56
46
|
}
|
57
47
|
end
|
58
48
|
|
59
49
|
MAX_SELECT_ROW = 4
|
50
|
+
MAX_RESOURCE = 4
|
60
51
|
|
61
52
|
def acquire(timeout, now=Time.now.to_i)
|
62
53
|
connect {
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
54
|
+
@db.run "LOCK TABLES `#{@table}` WRITE;" rescue nil # TODO mysql only
|
55
|
+
begin
|
56
|
+
while true
|
57
|
+
rows = 0
|
58
|
+
@db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` AS T WHERE timeout <= ? AND (resource IS NULL OR (SELECT count(1) FROM `#{@table}` WHERE timeout > ? AND resource = T.resource) < #{MAX_RESOURCE}) ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW};", now, now) {|row|
|
59
|
+
|
60
|
+
unless row[:created_at]
|
61
|
+
# finished/canceled task
|
62
|
+
@db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
|
63
|
+
|
64
|
+
else
|
65
|
+
n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?;", timeout, row[:id], row[:timeout]].update
|
66
|
+
if n > 0
|
67
|
+
return row[:id], Task.new(row[:id], row[:created_at], row[:data], row[:resource])
|
68
|
+
end
|
75
69
|
end
|
76
|
-
end
|
77
70
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
71
|
+
rows += 1
|
72
|
+
}
|
73
|
+
if rows < MAX_SELECT_ROW
|
74
|
+
return nil
|
75
|
+
end
|
82
76
|
end
|
77
|
+
ensure
|
78
|
+
@db.run "UNLOCK TABLES `#{@table}`;" rescue nil # TODO mysql only
|
83
79
|
end
|
84
80
|
}
|
85
81
|
end
|
@@ -105,10 +101,10 @@ class RDBBackend < Backend
|
|
105
101
|
finish(id, delete_timeout, now)
|
106
102
|
end
|
107
103
|
|
108
|
-
def submit(id, data, time=Time.now.to_i)
|
104
|
+
def submit(id, data, time=Time.now.to_i, resource=nil)
|
109
105
|
connect {
|
110
106
|
begin
|
111
|
-
n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at) VALUES (?, ?, ?, ?);", id, time, data, time].insert
|
107
|
+
n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at, resource) VALUES (?, ?, ?, ?, ?);", id, time, data, time, resource].insert
|
112
108
|
return true
|
113
109
|
rescue Sequel::DatabaseError
|
114
110
|
return nil
|
@@ -26,6 +26,7 @@ class SimpleDBBackend < Backend
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def list(&block)
|
29
|
+
# TODO support resource limit
|
29
30
|
@domain.items.select('timeout', 'data', 'created_at',
|
30
31
|
:where => "created_at != '' AND timeout > '#{int_encode(0)}'",
|
31
32
|
:order => [:timeout, :asc],
|
@@ -36,7 +37,7 @@ class SimpleDBBackend < Backend
|
|
36
37
|
created_at = int_decode(attrs['created_at'].first)
|
37
38
|
data = attrs['data'].first
|
38
39
|
timeout = int_decode(attrs['timeout'].first)
|
39
|
-
yield id, created_at, data, timeout
|
40
|
+
yield id, created_at, data, timeout, nil
|
40
41
|
}
|
41
42
|
end
|
42
43
|
|
@@ -44,6 +45,7 @@ class SimpleDBBackend < Backend
|
|
44
45
|
|
45
46
|
def acquire(timeout, now=Time.now.to_i)
|
46
47
|
while true
|
48
|
+
# TODO support resource limit
|
47
49
|
rows = 0
|
48
50
|
@domain.items.select('timeout', 'data', 'created_at',
|
49
51
|
:where => "timeout <= '#{int_encode(now)}'",
|
@@ -66,7 +68,7 @@ class SimpleDBBackend < Backend
|
|
66
68
|
|
67
69
|
data = attrs['data'].first
|
68
70
|
|
69
|
-
return [id,salt], Task.new(id, created_at, data)
|
71
|
+
return [id,salt], Task.new(id, created_at, data, nil)
|
70
72
|
end
|
71
73
|
|
72
74
|
rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
|
@@ -111,7 +113,8 @@ class SimpleDBBackend < Backend
|
|
111
113
|
finish(token, delete_timeout, now)
|
112
114
|
end
|
113
115
|
|
114
|
-
def submit(id, data, time=Time.now.to_i)
|
116
|
+
def submit(id, data, time=Time.now.to_i, resource=nil)
|
117
|
+
# TODO support resource limit
|
115
118
|
begin
|
116
119
|
@domain.items[id].attributes.replace('timeout'=>int_encode(time), 'created_at'=>int_encode(time), 'data'=>data,
|
117
120
|
:unless=>'timeout')
|
@@ -270,12 +270,12 @@ backend = backend_proc.call
|
|
270
270
|
|
271
271
|
case type
|
272
272
|
when :list
|
273
|
-
format = "%26s %26s %26s %s"
|
274
|
-
puts format % ["id", "created_at", "timeout", "data"]
|
273
|
+
format = "%26s %26s %26s %26s %s"
|
274
|
+
puts format % ["id", "created_at", "timeout", "resource", "data"]
|
275
275
|
time_format = "%Y-%m-%d %H:%M:%S %z"
|
276
276
|
n = 0
|
277
|
-
backend.list {|id,created_at,data,timeout|
|
278
|
-
puts format % [id, Time.at(created_at).strftime(time_format), Time.at(timeout).strftime(time_format), data]
|
277
|
+
backend.list {|id,created_at,data,timeout,resource|
|
278
|
+
puts format % [id, Time.at(created_at).strftime(time_format), Time.at(timeout).strftime(time_format), resource, data]
|
279
279
|
n += 1
|
280
280
|
}
|
281
281
|
puts "#{n} entries."
|
data/lib/perfectqueue/version.rb
CHANGED
data/test/backend_test.rb
CHANGED
@@ -15,8 +15,10 @@ class BackendTest < Test::Unit::TestCase
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def open_backend
|
18
|
-
PerfectQueue::SimpleDBBackend.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], 'perfectqueue-test-1').use_consistent_read
|
19
|
-
|
18
|
+
#PerfectQueue::SimpleDBBackend.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], 'perfectqueue-test-1').use_consistent_read
|
19
|
+
db = PerfectQueue::RDBBackend.new(DB_URI, "perfectdb_test")
|
20
|
+
db.create_tables
|
21
|
+
db
|
20
22
|
end
|
21
23
|
|
22
24
|
it 'acquire' do
|
@@ -213,5 +215,33 @@ class BackendTest < Test::Unit::TestCase
|
|
213
215
|
assert_equal time, task.created_at
|
214
216
|
assert_equal 'data1', task.data
|
215
217
|
end
|
218
|
+
|
219
|
+
it 'resource' do
|
220
|
+
clean_backend
|
221
|
+
|
222
|
+
db1 = open_backend
|
223
|
+
|
224
|
+
time = Time.now.to_i
|
225
|
+
|
226
|
+
5.times do |i|
|
227
|
+
ok = db1.submit(@key_prefix+'test'+i.to_s, 'data1', time, 'user1')
|
228
|
+
assert_equal true, ok
|
229
|
+
end
|
230
|
+
ok = db1.submit(@key_prefix+'test5', 'data2', time, 'user2')
|
231
|
+
assert_equal true, ok
|
232
|
+
|
233
|
+
4.times do
|
234
|
+
token, task = db1.acquire(time+TIMEOUT, time)
|
235
|
+
assert_not_equal nil, task
|
236
|
+
assert_equal "user1", task.resource
|
237
|
+
end
|
238
|
+
|
239
|
+
token, task = db1.acquire(time+TIMEOUT, time)
|
240
|
+
assert_not_equal nil, task
|
241
|
+
assert_equal "user2", task.resource
|
242
|
+
|
243
|
+
token, task = db1.acquire(time+TIMEOUT, time)
|
244
|
+
assert_equal nil, task
|
245
|
+
end
|
216
246
|
end
|
217
247
|
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.8
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-12-
|
12
|
+
date: 2011-12-17 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sequel
|
16
|
-
requirement: &
|
16
|
+
requirement: &70113160282360 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 3.26.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70113160282360
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: aws-sdk
|
27
|
-
requirement: &
|
27
|
+
requirement: &70113160281640 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ~>
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
version: 1.1.1
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70113160281640
|
36
36
|
description:
|
37
37
|
email: frsyuki@gmail.com
|
38
38
|
executables:
|
@@ -82,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
82
82
|
version: '0'
|
83
83
|
requirements: []
|
84
84
|
rubyforge_project:
|
85
|
-
rubygems_version: 1.8.
|
85
|
+
rubygems_version: 1.8.12
|
86
86
|
signing_key:
|
87
87
|
specification_version: 3
|
88
88
|
summary: Highly available distributed queue built on RDBMS or SimpleDB
|