dbq 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +110 -0
- data/Rakefile +31 -0
- data/dbq.gemspec +27 -0
- data/lib/dbq.rb +8 -0
- data/lib/dbq/basic_queue.rb +44 -0
- data/lib/dbq/ordered_queue.rb +63 -0
- data/lib/dbq/queue.rb +28 -0
- data/lib/dbq/version.rb +3 -0
- data/spec/integration/postgres/connnected_spec.rb +7 -0
- data/spec/integration/postgres/ordered_queue_spec.rb +184 -0
- data/spec/integration/postgres/queue_spec.rb +89 -0
- data/spec/lib/dbq/queue_spec.rb +5 -0
- data/spec/lib/dbq/version_spec.rb +5 -0
- data/spec/migrate/create_test_ordered_queues.rb +13 -0
- data/spec/migrate/create_test_queues.rb +10 -0
- data/spec/spec_helper.rb +20 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 945cacb9b92fb902509f1fcc529711486252c550
|
4
|
+
data.tar.gz: 9c889fab71094d0aacc02d74a391fe786c549379
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ea1df7b2ebe4d1c617fc8069216fd756d82e7e3e7e28093532864a146b98acacf634ddabaeb9426422feb99a820e0120fe230734668e63e94386dca6d4cb2471
|
7
|
+
data.tar.gz: 502452a1f3ac28affe23e61367e2cc9ecbd49ebb7cec6ea5bcb0fd9c965dee06c8014eb163a4bbf81c72c0dc122bf992642bee40193608f0eb7f1143e70a9858
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Tyler Hartland
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
# DBQ
|
2
|
+
|
3
|
+
DuraBle Queues
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'dbq'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install dbq
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
### Queue
|
22
|
+
|
23
|
+
Create a table with the following column and index:
|
24
|
+
|
25
|
+
* checked_out_at (timestamp, indexed)
|
26
|
+
|
27
|
+
Add any data columns you want.
|
28
|
+
|
29
|
+
Create an ActiveRecord model for your table and include DBQ:Queue:
|
30
|
+
|
31
|
+
```
|
32
|
+
class MyQueue < ActiveRecord::Base
|
33
|
+
include DBQ::Queue
|
34
|
+
end
|
35
|
+
```
|
36
|
+
|
37
|
+
Push some data onto your queue (push is, for now, synonymous with create!):
|
38
|
+
|
39
|
+
```MyQueue.push(my_data: 'some data')```
|
40
|
+
|
41
|
+
Pop your data back off your queue:
|
42
|
+
|
43
|
+
```MyQueue.pop.my_data #=> 'some data'```
|
44
|
+
|
45
|
+
### OrderedQueue
|
46
|
+
|
47
|
+
Create a table with the following columns:
|
48
|
+
|
49
|
+
* checked_out_at (timestamp, indexed)
|
50
|
+
* my_ordered_attr1 (any type, indexed)
|
51
|
+
* my_ordered_attr2 (any type, indexed)
|
52
|
+
* ...
|
53
|
+
|
54
|
+
Again, add any data fields you want.
|
55
|
+
|
56
|
+
Create an ActiveRecord model for your table and include DBQ:OrderedQueue. Be sure to specify which fields should enforce order (detailed explanation below). I'd recommend validating presence on those fields as well.
|
57
|
+
|
58
|
+
```
|
59
|
+
class MyOrderedQueue < ActiveRecord::Base
|
60
|
+
include DBQ::OrderedQueue
|
61
|
+
validates :my_ordered_attr1, :my_ordered_attr2, presence: true
|
62
|
+
enforces_order_on :my_ordered_attr1, :my_ordered_attr2
|
63
|
+
end
|
64
|
+
```
|
65
|
+
|
66
|
+
Push some data onto your queue:
|
67
|
+
|
68
|
+
```MyOrderedQueue.push(my_ordered_attr1: 'attr1', my_ordered_attr2: 'attr2', my_data 'some data')```
|
69
|
+
|
70
|
+
Pop your data back off your queue:
|
71
|
+
|
72
|
+
```MyOrderedQueue.pop.my_data #=> 'some data'```
|
73
|
+
|
74
|
+
What's the point!?
|
75
|
+
|
76
|
+
DBQ::OrderedQueue will enforce processing order for records which have matching ordered_attrs. If a 'sibling' record is checked out, its siblings will not come off the queue until the first sibling's pop is committed. Here's an example:
|
77
|
+
|
78
|
+
Two sibling records exist (using one ordered attr for brevity):
|
79
|
+
|
80
|
+
```
|
81
|
+
MyOrderedQueue.push(ordered: 1, data: 'first item')
|
82
|
+
MyOrderedQueue.push(ordered: 1, data: 'second item')
|
83
|
+
```
|
84
|
+
|
85
|
+
One process/thread pops the first item (in a transaction!):
|
86
|
+
|
87
|
+
```do_some_slow_transactional_work(MyOrderedQueue.pop) ```
|
88
|
+
|
89
|
+
Before the first transaction commits, DBQ::OrderedQueue restricts access to the sibling item:
|
90
|
+
|
91
|
+
```MyOrderedQueue.pop #=> nil```
|
92
|
+
|
93
|
+
After the first transaction commits, DBQ::OrderedQueue allows the second item to be popped:
|
94
|
+
|
95
|
+
```MyOrderedQueue.pop.data #=> 'second item'```
|
96
|
+
|
97
|
+
If the first transaction rolls back, the first item will come off the queue again:
|
98
|
+
|
99
|
+
```MyOrderedQueue.pop.data #=> 'first item'```
|
100
|
+
|
101
|
+
If the process gets killed or the OS crashes while the first item is checked out, you may need to manually check the item back in by setting checked_out_at to null.
|
102
|
+
|
103
|
+
|
104
|
+
## Contributing
|
105
|
+
|
106
|
+
1. Fork it ( http://github.com/<my-github-username>/dbq/fork )
|
107
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
108
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
109
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
110
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require 'dbq'
|
3
|
+
|
4
|
+
namespace :db do
|
5
|
+
desc "Seed job schedules"
|
6
|
+
namespace :test do
|
7
|
+
task :prepare do
|
8
|
+
puts `dropdb dbq_test -e`
|
9
|
+
puts `createdb dbq_test -e`
|
10
|
+
|
11
|
+
# only testing on postgres for now
|
12
|
+
db = URI.parse('postgres://localhost/dbq_test')
|
13
|
+
DB_NAME = db.path[1..-1]
|
14
|
+
|
15
|
+
ActiveRecord::Base.establish_connection(
|
16
|
+
:adapter => db.scheme == 'postgres' ? 'postgresql' : db.scheme,
|
17
|
+
:host => db.host,
|
18
|
+
:port => db.port,
|
19
|
+
:username => db.user,
|
20
|
+
:password => db.password,
|
21
|
+
:database => DB_NAME,
|
22
|
+
:encoding => 'utf8'
|
23
|
+
)
|
24
|
+
|
25
|
+
Dir['spec/migrate/*.rb'].each { |f| require File.absolute_path(f) }
|
26
|
+
migrations = Migrate.constants.map { |c| Migrate.const_get(c) }
|
27
|
+
ActiveRecord::Migration.run(*migrations)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
data/dbq.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'dbq/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "dbq"
|
8
|
+
spec.version = DBQ::VERSION
|
9
|
+
spec.authors = ["TH"]
|
10
|
+
spec.email = ["tyler.hartland@code42.com"]
|
11
|
+
spec.summary = %q{Durable queues which commit with database transactions.}
|
12
|
+
# spec.description = %q{TODO: Write a longer description. Optional.}
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.5"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
spec.add_development_dependency "rspec"
|
24
|
+
spec.add_development_dependency "pg"
|
25
|
+
|
26
|
+
spec.add_runtime_dependency 'activerecord'
|
27
|
+
end
|
data/lib/dbq.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
module DBQ
|
2
|
+
module BasicQueue
|
3
|
+
def self.included(receiver)
|
4
|
+
receiver.after_rollback :check_in!
|
5
|
+
receiver.extend ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def pop
|
10
|
+
item = check_out_item
|
11
|
+
item.try(:destroy)
|
12
|
+
item
|
13
|
+
end
|
14
|
+
|
15
|
+
def push(opts)
|
16
|
+
create!(opts)
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def check_out_item
|
22
|
+
raise 'Do not include BasicQueue directly. Maybe you wanted DBQ::Queue?'
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def check_out!
|
27
|
+
self.update_attributes(checked_out_at: Time.now)
|
28
|
+
end
|
29
|
+
|
30
|
+
def check_in!
|
31
|
+
# necessary if we were frozen by a rolled back destroy call
|
32
|
+
self.class.update(id, checked_out_at: nil)
|
33
|
+
end
|
34
|
+
|
35
|
+
def data=(new_data)
|
36
|
+
self.wrapped_data ||= {}
|
37
|
+
wrapped_data['data'] = new_data
|
38
|
+
end
|
39
|
+
|
40
|
+
def data
|
41
|
+
wrapped_data['data'] if wrapped_data
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module DBQ
|
2
|
+
module OrderedQueue
|
3
|
+
def self.included(receiver)
|
4
|
+
receiver.include BasicQueue
|
5
|
+
receiver.extend ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def enforces_order_on(*attrs)
|
10
|
+
@check_out_query = nil
|
11
|
+
@enforce_order_on = attrs
|
12
|
+
# check indexes and warn if they do not exist?
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def check_out_item
|
18
|
+
# a separate thread means a separate connection/transaction/commit
|
19
|
+
thread = Thread.new do
|
20
|
+
item = nil
|
21
|
+
connection_pool.with_connection do
|
22
|
+
transaction do
|
23
|
+
item = find_by_sql(check_out_query).first
|
24
|
+
item.try(:check_out!)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
item
|
28
|
+
end
|
29
|
+
thread.abort_on_exception = true
|
30
|
+
thread.value
|
31
|
+
end
|
32
|
+
|
33
|
+
def check_out_query
|
34
|
+
@check_out_query ||= <<-SQL
|
35
|
+
select * from #{table_name}
|
36
|
+
where #{table_name}.id not in (
|
37
|
+
select #{table_name}.id
|
38
|
+
from #{table_name}
|
39
|
+
inner join #{table_name} as self_join on
|
40
|
+
#{join_clause}
|
41
|
+
where self_join.checked_out_at is not null
|
42
|
+
)
|
43
|
+
and #{table_name}.checked_out_at is null
|
44
|
+
order by #{table_name}.id asc
|
45
|
+
limit 1
|
46
|
+
for update;
|
47
|
+
SQL
|
48
|
+
end
|
49
|
+
|
50
|
+
# possibly add check/warn for recommended indexes
|
51
|
+
# def indexes
|
52
|
+
# connection.indexes(self)
|
53
|
+
# end
|
54
|
+
|
55
|
+
def join_clause
|
56
|
+
@enforce_order_on.inject([]) do |items, ordered_attr|
|
57
|
+
items << "#{table_name}.#{ordered_attr} = self_join.#{ordered_attr}"
|
58
|
+
items
|
59
|
+
end.join(' and ')
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/lib/dbq/queue.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
module DBQ
|
2
|
+
module Queue
|
3
|
+
def self.included(receiver)
|
4
|
+
receiver.include BasicQueue
|
5
|
+
receiver.extend ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
private
|
10
|
+
|
11
|
+
def check_out_item
|
12
|
+
thread = Thread.new do
|
13
|
+
item = nil
|
14
|
+
connection_pool.with_connection do
|
15
|
+
transaction do
|
16
|
+
item = where(checked_out_at: nil)
|
17
|
+
.order(id: :asc).limit(1).lock(true).first
|
18
|
+
item.try(:check_out!)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
item
|
22
|
+
end
|
23
|
+
thread.abort_on_exception = true
|
24
|
+
thread.value
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/dbq/version.rb
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class TestOrderedQueue < ActiveRecord::Base
|
4
|
+
include DBQ::OrderedQueue
|
5
|
+
enforces_order_on :fk1, :fk2
|
6
|
+
end
|
7
|
+
|
8
|
+
describe TestOrderedQueue do
|
9
|
+
let(:klass) { TestOrderedQueue }
|
10
|
+
|
11
|
+
before do
|
12
|
+
# klass.logger = Logger.new(STDOUT)
|
13
|
+
klass.delete_all
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
klass.delete_all
|
18
|
+
end
|
19
|
+
|
20
|
+
context 'a single item exists' do
|
21
|
+
let(:value) { {'expected' => 'changed_attrs'} }
|
22
|
+
|
23
|
+
before do
|
24
|
+
klass.push(data: value)
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'can pop the item back out' do
|
28
|
+
expect(klass.pop.data).to eq value
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'destroys the item' do
|
32
|
+
expect { klass.pop }.to change { klass.count }.from(1).to(0)
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'the transaction is rolled back' do
|
36
|
+
it 'does not destroy the item' do
|
37
|
+
expect {
|
38
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
39
|
+
}.not_to change {
|
40
|
+
klass.count
|
41
|
+
}.from(1)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'resets checked_out_at to nil' do
|
45
|
+
expect {
|
46
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
47
|
+
}.not_to change {
|
48
|
+
klass.first.checked_out_at
|
49
|
+
}.from(nil)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'items are not order constrained' do
|
55
|
+
before do
|
56
|
+
klass.push(data: 1)
|
57
|
+
klass.push(data: 2)
|
58
|
+
klass.push(data: 3)
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'multiple items are popped simultaneously' do
|
62
|
+
it 'only pops each item once' do
|
63
|
+
result_threads = [
|
64
|
+
simultaneous_pop,
|
65
|
+
simultaneous_pop,
|
66
|
+
simultaneous_pop
|
67
|
+
]
|
68
|
+
values = result_threads.map(&:value)
|
69
|
+
data = values.map(&:data)
|
70
|
+
expect(data.sort).to eq [ 1, 2, 3 ]
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
context 'order enforcement matches partially' do
|
77
|
+
before do
|
78
|
+
klass.push(fk1: 1, fk2: 1, data: 1)
|
79
|
+
klass.push(fk1: 1, fk2: 2, data: 2)
|
80
|
+
klass.push(fk1: 1, fk2: 3, data: 3)
|
81
|
+
end
|
82
|
+
|
83
|
+
context 'multiple items are popped simultaneously' do
|
84
|
+
it 'only pops each item once' do
|
85
|
+
result_threads = [
|
86
|
+
simultaneous_pop,
|
87
|
+
simultaneous_pop,
|
88
|
+
simultaneous_pop
|
89
|
+
]
|
90
|
+
values = result_threads.map(&:value)
|
91
|
+
data = values.map(&:data)
|
92
|
+
expect(data.sort).to eq [ 1, 2, 3 ]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
context 'order enforcement matches' do
|
98
|
+
before do
|
99
|
+
klass.push(fk1: 1, fk2: 1, data: 1)
|
100
|
+
klass.push(fk1: 1, fk2: 1, data: 2)
|
101
|
+
klass.push(fk1: 1, fk2: 1, data: 3)
|
102
|
+
end
|
103
|
+
|
104
|
+
it 'pops items in order' do
|
105
|
+
expect([klass.pop.data, klass.pop.data, klass.pop.data]).to eq [ 1, 2, 3 ]
|
106
|
+
end
|
107
|
+
|
108
|
+
context 'multiple items are popped simultaneously' do
|
109
|
+
it 'only pops each item once' do
|
110
|
+
result_threads = [
|
111
|
+
simultaneous_pop,
|
112
|
+
simultaneous_pop,
|
113
|
+
simultaneous_pop
|
114
|
+
]
|
115
|
+
values = result_threads.map(&:value).compact.sort
|
116
|
+
# ids = values.map(&:id)
|
117
|
+
expect(values.uniq).to eq values
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
context 'an item is being processed' do
|
122
|
+
it 'does not return order-enfoced items' do
|
123
|
+
thread = slow_threaded_pop # work on an item
|
124
|
+
sleep 0.05 # wait for threaded pop to go first
|
125
|
+
begin
|
126
|
+
expect(klass.count).to eq 3
|
127
|
+
expect(klass.first.checked_out_at).not_to be_nil
|
128
|
+
expect(klass.last.checked_out_at).to be_nil
|
129
|
+
expect(klass.pop).to be_nil
|
130
|
+
ensure
|
131
|
+
thread.kill
|
132
|
+
end
|
133
|
+
end
|
134
|
+
context 'a non-order-enforced item also exists' do
|
135
|
+
before do
|
136
|
+
klass.push(fk1: 1, fk2: 4, data: 4)
|
137
|
+
end
|
138
|
+
|
139
|
+
it 'returns the non-order-enforced item' do
|
140
|
+
thread = slow_threaded_pop # work on an item
|
141
|
+
sleep 0.05 # wait for threaded pop to go first
|
142
|
+
begin
|
143
|
+
expect(klass.count).to eq 4
|
144
|
+
expect(klass.first.checked_out_at).not_to be_nil
|
145
|
+
expect(klass.last.checked_out_at).to be_nil
|
146
|
+
expect(klass.pop.data).to eq 4
|
147
|
+
ensure
|
148
|
+
thread.kill
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def simultaneous_pop(at=Time.now + 0.1)
|
158
|
+
thread = Thread.new do
|
159
|
+
result = nil
|
160
|
+
klass.connection_pool.with_connection do
|
161
|
+
klass.transaction do
|
162
|
+
sleep_for = at - Time.now
|
163
|
+
sleep sleep_for if sleep_for > 0
|
164
|
+
result = klass.pop
|
165
|
+
end
|
166
|
+
end
|
167
|
+
result
|
168
|
+
end
|
169
|
+
thread.abort_on_exception = true
|
170
|
+
thread
|
171
|
+
end
|
172
|
+
|
173
|
+
def slow_threaded_pop
|
174
|
+
thread = Thread.new do
|
175
|
+
klass.connection_pool.with_connection do
|
176
|
+
klass.transaction do
|
177
|
+
klass.pop
|
178
|
+
sleep 2
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
thread.abort_on_exception = true
|
183
|
+
thread
|
184
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class TestQueue < ActiveRecord::Base
|
4
|
+
include DBQ::Queue
|
5
|
+
end
|
6
|
+
|
7
|
+
describe TestQueue do
|
8
|
+
let(:klass) { TestQueue }
|
9
|
+
|
10
|
+
before do
|
11
|
+
klass.connection_pool.with_connection do
|
12
|
+
klass.delete_all
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
klass.connection_pool.with_connection do
|
18
|
+
klass.delete_all
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context 'a single item exists' do
|
23
|
+
let(:value) { {'expected' => 'changed_attrs'} }
|
24
|
+
|
25
|
+
before do
|
26
|
+
klass.push(data: value)
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'can pop the item back out' do
|
30
|
+
expect(klass.pop.data).to eq value
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'destroys the item' do
|
34
|
+
expect { klass.pop.data }.to change { klass.count }.from(1).to(0)
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'the transaction is rolled back' do
|
38
|
+
it 'does not destroy the item' do
|
39
|
+
expect {
|
40
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
41
|
+
}.not_to change {
|
42
|
+
klass.count
|
43
|
+
}.from(1)
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'resets checked_out_at to nil' do
|
47
|
+
expect {
|
48
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
49
|
+
}.not_to change {
|
50
|
+
klass.first.checked_out_at
|
51
|
+
}.from(nil)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'multiple items are popped simultaneously' do
|
57
|
+
before do
|
58
|
+
klass.push(data: 1)
|
59
|
+
klass.push(data: 2)
|
60
|
+
klass.push(data: 3)
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'only pops each item once' do
|
64
|
+
result_threads = [
|
65
|
+
simultaneous_pop,
|
66
|
+
simultaneous_pop,
|
67
|
+
simultaneous_pop
|
68
|
+
]
|
69
|
+
results = result_threads.map(&:value).map(&:data)
|
70
|
+
expect(results.sort).to eq [ 1, 2, 3 ]
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def simultaneous_pop(at=Time.now + 0.1)
|
76
|
+
thread = Thread.new do
|
77
|
+
result = nil
|
78
|
+
klass.connection_pool.with_connection do
|
79
|
+
klass.transaction do
|
80
|
+
sleep_for = at - Time.now
|
81
|
+
sleep sleep_for if sleep_for > 0
|
82
|
+
result = klass.pop
|
83
|
+
end
|
84
|
+
end
|
85
|
+
result
|
86
|
+
end
|
87
|
+
thread.abort_on_exception = true
|
88
|
+
thread
|
89
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'dbq'
|
2
|
+
require 'pg'
|
3
|
+
|
4
|
+
RSpec.configure do |config|
|
5
|
+
config.order = 'random'
|
6
|
+
end
|
7
|
+
|
8
|
+
# only testing on postgres for now
|
9
|
+
db = URI.parse('postgres://localhost/dbq_test')
|
10
|
+
DB_NAME = db.path[1..-1]
|
11
|
+
|
12
|
+
ActiveRecord::Base.establish_connection(
|
13
|
+
:adapter => db.scheme == 'postgres' ? 'postgresql' : db.scheme,
|
14
|
+
:host => db.host,
|
15
|
+
:port => db.port,
|
16
|
+
:username => db.user,
|
17
|
+
:password => db.password,
|
18
|
+
:database => DB_NAME,
|
19
|
+
:encoding => 'utf8'
|
20
|
+
)
|
metadata
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: dbq
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- TH
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-04-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.5'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.5'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pg
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: activerecord
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description:
|
84
|
+
email:
|
85
|
+
- tyler.hartland@code42.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- ".gitignore"
|
91
|
+
- ".rspec"
|
92
|
+
- Gemfile
|
93
|
+
- LICENSE.txt
|
94
|
+
- README.md
|
95
|
+
- Rakefile
|
96
|
+
- dbq.gemspec
|
97
|
+
- lib/dbq.rb
|
98
|
+
- lib/dbq/basic_queue.rb
|
99
|
+
- lib/dbq/ordered_queue.rb
|
100
|
+
- lib/dbq/queue.rb
|
101
|
+
- lib/dbq/version.rb
|
102
|
+
- spec/integration/postgres/connnected_spec.rb
|
103
|
+
- spec/integration/postgres/ordered_queue_spec.rb
|
104
|
+
- spec/integration/postgres/queue_spec.rb
|
105
|
+
- spec/lib/dbq/queue_spec.rb
|
106
|
+
- spec/lib/dbq/version_spec.rb
|
107
|
+
- spec/migrate/create_test_ordered_queues.rb
|
108
|
+
- spec/migrate/create_test_queues.rb
|
109
|
+
- spec/spec_helper.rb
|
110
|
+
homepage: ''
|
111
|
+
licenses:
|
112
|
+
- MIT
|
113
|
+
metadata: {}
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
require_paths:
|
117
|
+
- lib
|
118
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - ">="
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project:
|
130
|
+
rubygems_version: 2.1.11
|
131
|
+
signing_key:
|
132
|
+
specification_version: 4
|
133
|
+
summary: Durable queues which commit with database transactions.
|
134
|
+
test_files:
|
135
|
+
- spec/integration/postgres/connnected_spec.rb
|
136
|
+
- spec/integration/postgres/ordered_queue_spec.rb
|
137
|
+
- spec/integration/postgres/queue_spec.rb
|
138
|
+
- spec/lib/dbq/queue_spec.rb
|
139
|
+
- spec/lib/dbq/version_spec.rb
|
140
|
+
- spec/migrate/create_test_ordered_queues.rb
|
141
|
+
- spec/migrate/create_test_queues.rb
|
142
|
+
- spec/spec_helper.rb
|
143
|
+
has_rdoc:
|