dbq 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +110 -0
- data/Rakefile +31 -0
- data/dbq.gemspec +27 -0
- data/lib/dbq.rb +8 -0
- data/lib/dbq/basic_queue.rb +44 -0
- data/lib/dbq/ordered_queue.rb +63 -0
- data/lib/dbq/queue.rb +28 -0
- data/lib/dbq/version.rb +3 -0
- data/spec/integration/postgres/connnected_spec.rb +7 -0
- data/spec/integration/postgres/ordered_queue_spec.rb +184 -0
- data/spec/integration/postgres/queue_spec.rb +89 -0
- data/spec/lib/dbq/queue_spec.rb +5 -0
- data/spec/lib/dbq/version_spec.rb +5 -0
- data/spec/migrate/create_test_ordered_queues.rb +13 -0
- data/spec/migrate/create_test_queues.rb +10 -0
- data/spec/spec_helper.rb +20 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 945cacb9b92fb902509f1fcc529711486252c550
|
4
|
+
data.tar.gz: 9c889fab71094d0aacc02d74a391fe786c549379
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ea1df7b2ebe4d1c617fc8069216fd756d82e7e3e7e28093532864a146b98acacf634ddabaeb9426422feb99a820e0120fe230734668e63e94386dca6d4cb2471
|
7
|
+
data.tar.gz: 502452a1f3ac28affe23e61367e2cc9ecbd49ebb7cec6ea5bcb0fd9c965dee06c8014eb163a4bbf81c72c0dc122bf992642bee40193608f0eb7f1143e70a9858
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Tyler Hartland
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
# DBQ
|
2
|
+
|
3
|
+
DuraBle Queues
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'dbq'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install dbq
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
### Queue
|
22
|
+
|
23
|
+
Create a table with the following column and index:
|
24
|
+
|
25
|
+
* checked_out_at (timestamp, indexed)
|
26
|
+
|
27
|
+
Add any data columns you want.
|
28
|
+
|
29
|
+
Create an ActiveRecord model for your table and include DBQ:Queue:
|
30
|
+
|
31
|
+
```
|
32
|
+
class MyQueue < ActiveRecord::Base
|
33
|
+
include DBQ::Queue
|
34
|
+
end
|
35
|
+
```
|
36
|
+
|
37
|
+
Push some data onto your queue (push is, for now, synonymous with create!):
|
38
|
+
|
39
|
+
```MyQueue.push(my_data: 'some data')```
|
40
|
+
|
41
|
+
Pop your data back off your queue:
|
42
|
+
|
43
|
+
```MyQueue.pop.my_data #=> 'some data'```
|
44
|
+
|
45
|
+
### OrderedQueue
|
46
|
+
|
47
|
+
Create a table with the following columns:
|
48
|
+
|
49
|
+
* checked_out_at (timestamp, indexed)
|
50
|
+
* my_ordered_attr1 (any type, indexed)
|
51
|
+
* my_ordered_attr2 (any type, indexed)
|
52
|
+
* ...
|
53
|
+
|
54
|
+
Again, add any data fields you want.
|
55
|
+
|
56
|
+
Create an ActiveRecord model for your table and include DBQ:OrderedQueue. Be sure to specify which fields should enforce order (detailed explanation below). I'd recommend validating presence on those fields as well.
|
57
|
+
|
58
|
+
```
|
59
|
+
class MyOrderedQueue < ActiveRecord::Base
|
60
|
+
include DBQ::OrderedQueue
|
61
|
+
validates :my_ordered_attr1, :my_ordered_attr2, presence: true
|
62
|
+
enforces_order_on :my_ordered_attr1, :my_ordered_attr2
|
63
|
+
end
|
64
|
+
```
|
65
|
+
|
66
|
+
Push some data onto your queue:
|
67
|
+
|
68
|
+
```MyOrderedQueue.push(my_ordered_attr1: 'attr1', my_ordered_attr2: 'attr2', my_data 'some data')```
|
69
|
+
|
70
|
+
Pop your data back off your queue:
|
71
|
+
|
72
|
+
```MyOrderedQueue.pop.my_data #=> 'some data'```
|
73
|
+
|
74
|
+
What's the point!?
|
75
|
+
|
76
|
+
DBQ::OrderedQueue will enforce processing order for records which have matching ordered_attrs. If a 'sibling' record is checked out, its siblings will not come off the queue until the first sibling's pop is committed. Here's an example:
|
77
|
+
|
78
|
+
Two sibling records exist (using one ordered attr for brevity):
|
79
|
+
|
80
|
+
```
|
81
|
+
MyOrderedQueue.push(ordered: 1, data: 'first item')
|
82
|
+
MyOrderedQueue.push(ordered: 1, data: 'second item')
|
83
|
+
```
|
84
|
+
|
85
|
+
One process/thread pops the first item (in a transaction!):
|
86
|
+
|
87
|
+
```do_some_slow_transactional_work(MyOrderedQueue.pop) ```
|
88
|
+
|
89
|
+
Before the first transaction commits, DBQ::OrderedQueue restricts access to the sibling item:
|
90
|
+
|
91
|
+
```MyOrderedQueue.pop #=> nil```
|
92
|
+
|
93
|
+
After the first transaction commits, DBQ::OrderedQueue allows the second item to be popped:
|
94
|
+
|
95
|
+
```MyOrderedQueue.pop.data #=> 'second item'```
|
96
|
+
|
97
|
+
If the first transaction rolls back, the first item will come off the queue again:
|
98
|
+
|
99
|
+
```MyOrderedQueue.pop.data #=> 'first item'```
|
100
|
+
|
101
|
+
If the process gets killed or the OS crashes while the first item is checked out, you may need to manually check the item back in by setting checked_out_at to null.
|
102
|
+
|
103
|
+
|
104
|
+
## Contributing
|
105
|
+
|
106
|
+
1. Fork it ( http://github.com/<my-github-username>/dbq/fork )
|
107
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
108
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
109
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
110
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require 'dbq'
|
3
|
+
|
4
|
+
namespace :db do
|
5
|
+
desc "Seed job schedules"
|
6
|
+
namespace :test do
|
7
|
+
task :prepare do
|
8
|
+
puts `dropdb dbq_test -e`
|
9
|
+
puts `createdb dbq_test -e`
|
10
|
+
|
11
|
+
# only testing on postgres for now
|
12
|
+
db = URI.parse('postgres://localhost/dbq_test')
|
13
|
+
DB_NAME = db.path[1..-1]
|
14
|
+
|
15
|
+
ActiveRecord::Base.establish_connection(
|
16
|
+
:adapter => db.scheme == 'postgres' ? 'postgresql' : db.scheme,
|
17
|
+
:host => db.host,
|
18
|
+
:port => db.port,
|
19
|
+
:username => db.user,
|
20
|
+
:password => db.password,
|
21
|
+
:database => DB_NAME,
|
22
|
+
:encoding => 'utf8'
|
23
|
+
)
|
24
|
+
|
25
|
+
Dir['spec/migrate/*.rb'].each { |f| require File.absolute_path(f) }
|
26
|
+
migrations = Migrate.constants.map { |c| Migrate.const_get(c) }
|
27
|
+
ActiveRecord::Migration.run(*migrations)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
data/dbq.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'dbq/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "dbq"
|
8
|
+
spec.version = DBQ::VERSION
|
9
|
+
spec.authors = ["TH"]
|
10
|
+
spec.email = ["tyler.hartland@code42.com"]
|
11
|
+
spec.summary = %q{Durable queues which commit with database transactions.}
|
12
|
+
# spec.description = %q{TODO: Write a longer description. Optional.}
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.5"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
spec.add_development_dependency "rspec"
|
24
|
+
spec.add_development_dependency "pg"
|
25
|
+
|
26
|
+
spec.add_runtime_dependency 'activerecord'
|
27
|
+
end
|
data/lib/dbq.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
module DBQ
|
2
|
+
module BasicQueue
|
3
|
+
def self.included(receiver)
|
4
|
+
receiver.after_rollback :check_in!
|
5
|
+
receiver.extend ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def pop
|
10
|
+
item = check_out_item
|
11
|
+
item.try(:destroy)
|
12
|
+
item
|
13
|
+
end
|
14
|
+
|
15
|
+
def push(opts)
|
16
|
+
create!(opts)
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def check_out_item
|
22
|
+
raise 'Do not include BasicQueue directly. Maybe you wanted DBQ::Queue?'
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def check_out!
|
27
|
+
self.update_attributes(checked_out_at: Time.now)
|
28
|
+
end
|
29
|
+
|
30
|
+
def check_in!
|
31
|
+
# necessary if we were frozen by a rolled back destroy call
|
32
|
+
self.class.update(id, checked_out_at: nil)
|
33
|
+
end
|
34
|
+
|
35
|
+
def data=(new_data)
|
36
|
+
self.wrapped_data ||= {}
|
37
|
+
wrapped_data['data'] = new_data
|
38
|
+
end
|
39
|
+
|
40
|
+
def data
|
41
|
+
wrapped_data['data'] if wrapped_data
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module DBQ
|
2
|
+
module OrderedQueue
|
3
|
+
def self.included(receiver)
|
4
|
+
receiver.include BasicQueue
|
5
|
+
receiver.extend ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def enforces_order_on(*attrs)
|
10
|
+
@check_out_query = nil
|
11
|
+
@enforce_order_on = attrs
|
12
|
+
# check indexes and warn if they do not exist?
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def check_out_item
|
18
|
+
# a separate thread means a separate connection/transaction/commit
|
19
|
+
thread = Thread.new do
|
20
|
+
item = nil
|
21
|
+
connection_pool.with_connection do
|
22
|
+
transaction do
|
23
|
+
item = find_by_sql(check_out_query).first
|
24
|
+
item.try(:check_out!)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
item
|
28
|
+
end
|
29
|
+
thread.abort_on_exception = true
|
30
|
+
thread.value
|
31
|
+
end
|
32
|
+
|
33
|
+
def check_out_query
|
34
|
+
@check_out_query ||= <<-SQL
|
35
|
+
select * from #{table_name}
|
36
|
+
where #{table_name}.id not in (
|
37
|
+
select #{table_name}.id
|
38
|
+
from #{table_name}
|
39
|
+
inner join #{table_name} as self_join on
|
40
|
+
#{join_clause}
|
41
|
+
where self_join.checked_out_at is not null
|
42
|
+
)
|
43
|
+
and #{table_name}.checked_out_at is null
|
44
|
+
order by #{table_name}.id asc
|
45
|
+
limit 1
|
46
|
+
for update;
|
47
|
+
SQL
|
48
|
+
end
|
49
|
+
|
50
|
+
# possibly add check/warn for recommended indexes
|
51
|
+
# def indexes
|
52
|
+
# connection.indexes(self)
|
53
|
+
# end
|
54
|
+
|
55
|
+
def join_clause
|
56
|
+
@enforce_order_on.inject([]) do |items, ordered_attr|
|
57
|
+
items << "#{table_name}.#{ordered_attr} = self_join.#{ordered_attr}"
|
58
|
+
items
|
59
|
+
end.join(' and ')
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/lib/dbq/queue.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
module DBQ
|
2
|
+
module Queue
|
3
|
+
def self.included(receiver)
|
4
|
+
receiver.include BasicQueue
|
5
|
+
receiver.extend ClassMethods
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
private
|
10
|
+
|
11
|
+
def check_out_item
|
12
|
+
thread = Thread.new do
|
13
|
+
item = nil
|
14
|
+
connection_pool.with_connection do
|
15
|
+
transaction do
|
16
|
+
item = where(checked_out_at: nil)
|
17
|
+
.order(id: :asc).limit(1).lock(true).first
|
18
|
+
item.try(:check_out!)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
item
|
22
|
+
end
|
23
|
+
thread.abort_on_exception = true
|
24
|
+
thread.value
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/dbq/version.rb
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class TestOrderedQueue < ActiveRecord::Base
|
4
|
+
include DBQ::OrderedQueue
|
5
|
+
enforces_order_on :fk1, :fk2
|
6
|
+
end
|
7
|
+
|
8
|
+
describe TestOrderedQueue do
|
9
|
+
let(:klass) { TestOrderedQueue }
|
10
|
+
|
11
|
+
before do
|
12
|
+
# klass.logger = Logger.new(STDOUT)
|
13
|
+
klass.delete_all
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
klass.delete_all
|
18
|
+
end
|
19
|
+
|
20
|
+
context 'a single item exists' do
|
21
|
+
let(:value) { {'expected' => 'changed_attrs'} }
|
22
|
+
|
23
|
+
before do
|
24
|
+
klass.push(data: value)
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'can pop the item back out' do
|
28
|
+
expect(klass.pop.data).to eq value
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'destroys the item' do
|
32
|
+
expect { klass.pop }.to change { klass.count }.from(1).to(0)
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'the transaction is rolled back' do
|
36
|
+
it 'does not destroy the item' do
|
37
|
+
expect {
|
38
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
39
|
+
}.not_to change {
|
40
|
+
klass.count
|
41
|
+
}.from(1)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'resets checked_out_at to nil' do
|
45
|
+
expect {
|
46
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
47
|
+
}.not_to change {
|
48
|
+
klass.first.checked_out_at
|
49
|
+
}.from(nil)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'items are not order constrained' do
|
55
|
+
before do
|
56
|
+
klass.push(data: 1)
|
57
|
+
klass.push(data: 2)
|
58
|
+
klass.push(data: 3)
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'multiple items are popped simultaneously' do
|
62
|
+
it 'only pops each item once' do
|
63
|
+
result_threads = [
|
64
|
+
simultaneous_pop,
|
65
|
+
simultaneous_pop,
|
66
|
+
simultaneous_pop
|
67
|
+
]
|
68
|
+
values = result_threads.map(&:value)
|
69
|
+
data = values.map(&:data)
|
70
|
+
expect(data.sort).to eq [ 1, 2, 3 ]
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
context 'order enforcement matches partially' do
|
77
|
+
before do
|
78
|
+
klass.push(fk1: 1, fk2: 1, data: 1)
|
79
|
+
klass.push(fk1: 1, fk2: 2, data: 2)
|
80
|
+
klass.push(fk1: 1, fk2: 3, data: 3)
|
81
|
+
end
|
82
|
+
|
83
|
+
context 'multiple items are popped simultaneously' do
|
84
|
+
it 'only pops each item once' do
|
85
|
+
result_threads = [
|
86
|
+
simultaneous_pop,
|
87
|
+
simultaneous_pop,
|
88
|
+
simultaneous_pop
|
89
|
+
]
|
90
|
+
values = result_threads.map(&:value)
|
91
|
+
data = values.map(&:data)
|
92
|
+
expect(data.sort).to eq [ 1, 2, 3 ]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
context 'order enforcement matches' do
|
98
|
+
before do
|
99
|
+
klass.push(fk1: 1, fk2: 1, data: 1)
|
100
|
+
klass.push(fk1: 1, fk2: 1, data: 2)
|
101
|
+
klass.push(fk1: 1, fk2: 1, data: 3)
|
102
|
+
end
|
103
|
+
|
104
|
+
it 'pops items in order' do
|
105
|
+
expect([klass.pop.data, klass.pop.data, klass.pop.data]).to eq [ 1, 2, 3 ]
|
106
|
+
end
|
107
|
+
|
108
|
+
context 'multiple items are popped simultaneously' do
|
109
|
+
it 'only pops each item once' do
|
110
|
+
result_threads = [
|
111
|
+
simultaneous_pop,
|
112
|
+
simultaneous_pop,
|
113
|
+
simultaneous_pop
|
114
|
+
]
|
115
|
+
values = result_threads.map(&:value).compact.sort
|
116
|
+
# ids = values.map(&:id)
|
117
|
+
expect(values.uniq).to eq values
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
context 'an item is being processed' do
|
122
|
+
it 'does not return order-enfoced items' do
|
123
|
+
thread = slow_threaded_pop # work on an item
|
124
|
+
sleep 0.05 # wait for threaded pop to go first
|
125
|
+
begin
|
126
|
+
expect(klass.count).to eq 3
|
127
|
+
expect(klass.first.checked_out_at).not_to be_nil
|
128
|
+
expect(klass.last.checked_out_at).to be_nil
|
129
|
+
expect(klass.pop).to be_nil
|
130
|
+
ensure
|
131
|
+
thread.kill
|
132
|
+
end
|
133
|
+
end
|
134
|
+
context 'a non-order-enforced item also exists' do
|
135
|
+
before do
|
136
|
+
klass.push(fk1: 1, fk2: 4, data: 4)
|
137
|
+
end
|
138
|
+
|
139
|
+
it 'returns the non-order-enforced item' do
|
140
|
+
thread = slow_threaded_pop # work on an item
|
141
|
+
sleep 0.05 # wait for threaded pop to go first
|
142
|
+
begin
|
143
|
+
expect(klass.count).to eq 4
|
144
|
+
expect(klass.first.checked_out_at).not_to be_nil
|
145
|
+
expect(klass.last.checked_out_at).to be_nil
|
146
|
+
expect(klass.pop.data).to eq 4
|
147
|
+
ensure
|
148
|
+
thread.kill
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def simultaneous_pop(at=Time.now + 0.1)
|
158
|
+
thread = Thread.new do
|
159
|
+
result = nil
|
160
|
+
klass.connection_pool.with_connection do
|
161
|
+
klass.transaction do
|
162
|
+
sleep_for = at - Time.now
|
163
|
+
sleep sleep_for if sleep_for > 0
|
164
|
+
result = klass.pop
|
165
|
+
end
|
166
|
+
end
|
167
|
+
result
|
168
|
+
end
|
169
|
+
thread.abort_on_exception = true
|
170
|
+
thread
|
171
|
+
end
|
172
|
+
|
173
|
+
def slow_threaded_pop
|
174
|
+
thread = Thread.new do
|
175
|
+
klass.connection_pool.with_connection do
|
176
|
+
klass.transaction do
|
177
|
+
klass.pop
|
178
|
+
sleep 2
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
thread.abort_on_exception = true
|
183
|
+
thread
|
184
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class TestQueue < ActiveRecord::Base
|
4
|
+
include DBQ::Queue
|
5
|
+
end
|
6
|
+
|
7
|
+
describe TestQueue do
|
8
|
+
let(:klass) { TestQueue }
|
9
|
+
|
10
|
+
before do
|
11
|
+
klass.connection_pool.with_connection do
|
12
|
+
klass.delete_all
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
klass.connection_pool.with_connection do
|
18
|
+
klass.delete_all
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context 'a single item exists' do
|
23
|
+
let(:value) { {'expected' => 'changed_attrs'} }
|
24
|
+
|
25
|
+
before do
|
26
|
+
klass.push(data: value)
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'can pop the item back out' do
|
30
|
+
expect(klass.pop.data).to eq value
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'destroys the item' do
|
34
|
+
expect { klass.pop.data }.to change { klass.count }.from(1).to(0)
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'the transaction is rolled back' do
|
38
|
+
it 'does not destroy the item' do
|
39
|
+
expect {
|
40
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
41
|
+
}.not_to change {
|
42
|
+
klass.count
|
43
|
+
}.from(1)
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'resets checked_out_at to nil' do
|
47
|
+
expect {
|
48
|
+
klass.transaction { klass.pop; raise ActiveRecord::Rollback }
|
49
|
+
}.not_to change {
|
50
|
+
klass.first.checked_out_at
|
51
|
+
}.from(nil)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'multiple items are popped simultaneously' do
|
57
|
+
before do
|
58
|
+
klass.push(data: 1)
|
59
|
+
klass.push(data: 2)
|
60
|
+
klass.push(data: 3)
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'only pops each item once' do
|
64
|
+
result_threads = [
|
65
|
+
simultaneous_pop,
|
66
|
+
simultaneous_pop,
|
67
|
+
simultaneous_pop
|
68
|
+
]
|
69
|
+
results = result_threads.map(&:value).map(&:data)
|
70
|
+
expect(results.sort).to eq [ 1, 2, 3 ]
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def simultaneous_pop(at=Time.now + 0.1)
|
76
|
+
thread = Thread.new do
|
77
|
+
result = nil
|
78
|
+
klass.connection_pool.with_connection do
|
79
|
+
klass.transaction do
|
80
|
+
sleep_for = at - Time.now
|
81
|
+
sleep sleep_for if sleep_for > 0
|
82
|
+
result = klass.pop
|
83
|
+
end
|
84
|
+
end
|
85
|
+
result
|
86
|
+
end
|
87
|
+
thread.abort_on_exception = true
|
88
|
+
thread
|
89
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'dbq'
|
2
|
+
require 'pg'
|
3
|
+
|
4
|
+
RSpec.configure do |config|
|
5
|
+
config.order = 'random'
|
6
|
+
end
|
7
|
+
|
8
|
+
# only testing on postgres for now
|
9
|
+
db = URI.parse('postgres://localhost/dbq_test')
|
10
|
+
DB_NAME = db.path[1..-1]
|
11
|
+
|
12
|
+
ActiveRecord::Base.establish_connection(
|
13
|
+
:adapter => db.scheme == 'postgres' ? 'postgresql' : db.scheme,
|
14
|
+
:host => db.host,
|
15
|
+
:port => db.port,
|
16
|
+
:username => db.user,
|
17
|
+
:password => db.password,
|
18
|
+
:database => DB_NAME,
|
19
|
+
:encoding => 'utf8'
|
20
|
+
)
|
metadata
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: dbq
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- TH
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-04-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.5'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.5'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pg
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: activerecord
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description:
|
84
|
+
email:
|
85
|
+
- tyler.hartland@code42.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- ".gitignore"
|
91
|
+
- ".rspec"
|
92
|
+
- Gemfile
|
93
|
+
- LICENSE.txt
|
94
|
+
- README.md
|
95
|
+
- Rakefile
|
96
|
+
- dbq.gemspec
|
97
|
+
- lib/dbq.rb
|
98
|
+
- lib/dbq/basic_queue.rb
|
99
|
+
- lib/dbq/ordered_queue.rb
|
100
|
+
- lib/dbq/queue.rb
|
101
|
+
- lib/dbq/version.rb
|
102
|
+
- spec/integration/postgres/connnected_spec.rb
|
103
|
+
- spec/integration/postgres/ordered_queue_spec.rb
|
104
|
+
- spec/integration/postgres/queue_spec.rb
|
105
|
+
- spec/lib/dbq/queue_spec.rb
|
106
|
+
- spec/lib/dbq/version_spec.rb
|
107
|
+
- spec/migrate/create_test_ordered_queues.rb
|
108
|
+
- spec/migrate/create_test_queues.rb
|
109
|
+
- spec/spec_helper.rb
|
110
|
+
homepage: ''
|
111
|
+
licenses:
|
112
|
+
- MIT
|
113
|
+
metadata: {}
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
require_paths:
|
117
|
+
- lib
|
118
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - ">="
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project:
|
130
|
+
rubygems_version: 2.1.11
|
131
|
+
signing_key:
|
132
|
+
specification_version: 4
|
133
|
+
summary: Durable queues which commit with database transactions.
|
134
|
+
test_files:
|
135
|
+
- spec/integration/postgres/connnected_spec.rb
|
136
|
+
- spec/integration/postgres/ordered_queue_spec.rb
|
137
|
+
- spec/integration/postgres/queue_spec.rb
|
138
|
+
- spec/lib/dbq/queue_spec.rb
|
139
|
+
- spec/lib/dbq/version_spec.rb
|
140
|
+
- spec/migrate/create_test_ordered_queues.rb
|
141
|
+
- spec/migrate/create_test_queues.rb
|
142
|
+
- spec/spec_helper.rb
|
143
|
+
has_rdoc:
|