parallel_queue 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.rspec +1 -0
- data/.rvmrc +1 -0
- data/Gemfile +12 -0
- data/Gemfile.lock +48 -0
- data/LICENSE +21 -0
- data/README.md +89 -0
- data/Rakefile +50 -0
- data/VERSION +1 -0
- data/dequeue_demo.rb +23 -0
- data/enqueue_demo.rb +19 -0
- data/lib/parallel_queue.rb +124 -0
- data/parallel_queue.gemspec +72 -0
- data/spec/parallel_queue_spec.rb +259 -0
- data/spec/spec_helper.rb +17 -0
- metadata +144 -0
data/.document
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm use 1.9.3@parallel_queue
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
binding_of_caller (0.6.7)
|
5
|
+
coderay (1.0.6)
|
6
|
+
diff-lcs (1.1.3)
|
7
|
+
git (1.2.5)
|
8
|
+
jeweler (1.8.3)
|
9
|
+
bundler (~> 1.0)
|
10
|
+
git (>= 1.2.5)
|
11
|
+
rake
|
12
|
+
rdoc
|
13
|
+
json (1.6.6)
|
14
|
+
method_source (0.7.1)
|
15
|
+
pry (0.9.9.3)
|
16
|
+
coderay (~> 1.0.5)
|
17
|
+
method_source (~> 0.7.1)
|
18
|
+
slop (>= 2.4.4, < 3)
|
19
|
+
pry-nav (0.2.1)
|
20
|
+
pry (~> 0.9.9)
|
21
|
+
pry-stack_explorer (0.4.2)
|
22
|
+
binding_of_caller (~> 0.6.2)
|
23
|
+
pry (~> 0.9.9)
|
24
|
+
rake (0.9.2.2)
|
25
|
+
rdoc (3.12)
|
26
|
+
json (~> 1.4)
|
27
|
+
redis (2.2.2)
|
28
|
+
rspec (2.9.0)
|
29
|
+
rspec-core (~> 2.9.0)
|
30
|
+
rspec-expectations (~> 2.9.0)
|
31
|
+
rspec-mocks (~> 2.9.0)
|
32
|
+
rspec-core (2.9.0)
|
33
|
+
rspec-expectations (2.9.1)
|
34
|
+
diff-lcs (~> 1.1.3)
|
35
|
+
rspec-mocks (2.9.0)
|
36
|
+
slop (2.4.4)
|
37
|
+
|
38
|
+
PLATFORMS
|
39
|
+
ruby
|
40
|
+
|
41
|
+
DEPENDENCIES
|
42
|
+
bundler
|
43
|
+
jeweler
|
44
|
+
pry
|
45
|
+
pry-nav
|
46
|
+
pry-stack_explorer
|
47
|
+
redis (>= 2.2.0)
|
48
|
+
rspec
|
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
Copyright (c) 2011 Daniel W. Nelson
|
3
|
+
|
4
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
a copy of this software and associated documentation files (the
|
6
|
+
"Software"), to deal in the Software without restriction, including
|
7
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be
|
13
|
+
included in all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
ParallelQueue
|
2
|
+
==========
|
3
|
+
|
4
|
+
ParallelQueue provides a thread safe, Redis backed, parallel queue abstraction. Motivation for creating this was for queueing messages based on ID so that very chatty message emitters don't prevent messages from others from being processed.
|
5
|
+
|
6
|
+
|
7
|
+
|
8
|
+
Usage
|
9
|
+
=====
|
10
|
+
|
11
|
+
In your Gemfile:
|
12
|
+
|
13
|
+
gem 'parallel_queue'
|
14
|
+
|
15
|
+
Example:
|
16
|
+
|
17
|
+
$ irb
|
18
|
+
require 'redis'
|
19
|
+
require 'parallel_queue'
|
20
|
+
redis = Redis.new(:host => '127.0.0.1', :port => '6379')
|
21
|
+
queue = ParallelQueue.new(redis, 'my_object_message_queue')
|
22
|
+
queue.enqueue('123', 'hello world')
|
23
|
+
queue.enqueue('peanuts', 'Chalie Brown')
|
24
|
+
queue.enqueue('peanuts', 'Snoopy')
|
25
|
+
|
26
|
+
puts queue.queue_count
|
27
|
+
- 2
|
28
|
+
|
29
|
+
queue.dequeue_each do |item|
|
30
|
+
puts item
|
31
|
+
end
|
32
|
+
- 'hello world'
|
33
|
+
- 'Charlie Brown'
|
34
|
+
|
35
|
+
puts queue.queue_count
|
36
|
+
- 1
|
37
|
+
|
38
|
+
queue.dequeue_each do |item|
|
39
|
+
puts item
|
40
|
+
end
|
41
|
+
- 'Snoopy'
|
42
|
+
|
43
|
+
puts queue.queue_count
|
44
|
+
- 0
|
45
|
+
|
46
|
+
queue.dequeue_each do |item|
|
47
|
+
puts item
|
48
|
+
end
|
49
|
+
|
50
|
+
All items are queued and returned as strings.
|
51
|
+
|
52
|
+
|
53
|
+
|
54
|
+
Testing in multiple threads
|
55
|
+
===========================
|
56
|
+
|
57
|
+
A couple files are provided to make it easy to test this in multiple threads. Start by opening three terminals (I use the [byobu](https://launchpad.net/byobu) flavor of [screen](http://www.gnu.org/software/screen/)) and cd-ing to the root of this project.
|
58
|
+
|
59
|
+
Terminal 1:
|
60
|
+
|
61
|
+
$ ruby dequeue_demo.rb d1.txt
|
62
|
+
|
63
|
+
Terminal 2:
|
64
|
+
|
65
|
+
$ ruby dequeue_demo.rb d2.txt
|
66
|
+
|
67
|
+
Terminal 3 (run the command in terminal 3 after starting 1 & 2):
|
68
|
+
|
69
|
+
$ ruby enqueue_demo.rb
|
70
|
+
After a few seconds have passed, press control-c in Terminal 3.
|
71
|
+
Terminals 1 & 2 will automatically stop when they finish processing
|
72
|
+
the data enqueued by Terminal 3.
|
73
|
+
|
74
|
+
To check for any common values between d1.txt and d2.txt (there should be none):
|
75
|
+
|
76
|
+
$ comm -1 -2 d1.txt d2.txt
|
77
|
+
|
78
|
+
|
79
|
+
|
80
|
+
Contributing to parallel_queue
|
81
|
+
==========================
|
82
|
+
|
83
|
+
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
|
84
|
+
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
|
85
|
+
* Fork the project
|
86
|
+
* Start a feature/bugfix branch
|
87
|
+
* Commit and push until you are happy with your contribution
|
88
|
+
* Make sure to add specs for it. This is important so I don't break it in a future version unintentionally.
|
89
|
+
* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
|
data/Rakefile
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
begin
|
6
|
+
Bundler.setup(:default, :development)
|
7
|
+
rescue Bundler::BundlerError => e
|
8
|
+
$stderr.puts e.message
|
9
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
10
|
+
exit e.status_code
|
11
|
+
end
|
12
|
+
require 'rake'
|
13
|
+
|
14
|
+
require 'jeweler'
|
15
|
+
Jeweler::Tasks.new do |gem|
|
16
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
17
|
+
gem.name = "parallel_queue"
|
18
|
+
gem.homepage = "http://github.com/populr/parallel_queue"
|
19
|
+
gem.license = "MIT"
|
20
|
+
gem.summary = %Q{A thread safe, Redis backed, parallel queue abstraction}
|
21
|
+
gem.description = %Q{Motivation for creating this was for queueing messages based on ID so that very chatty message emitters don't prevent messages from others from being processed}
|
22
|
+
gem.email = "daniel@populr.me"
|
23
|
+
gem.authors = ["Daniel Nelson"]
|
24
|
+
# dependencies defined in Gemfile
|
25
|
+
gem.files.exclude 'spec'
|
26
|
+
end
|
27
|
+
Jeweler::RubygemsDotOrgTasks.new
|
28
|
+
|
29
|
+
require 'rspec/core'
|
30
|
+
require 'rspec/core/rake_task'
|
31
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
32
|
+
spec.pattern = FileList['spec/**/*_spec.rb']
|
33
|
+
end
|
34
|
+
|
35
|
+
RSpec::Core::RakeTask.new(:rcov) do |spec|
|
36
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
37
|
+
spec.rcov = true
|
38
|
+
end
|
39
|
+
|
40
|
+
task :default => :spec
|
41
|
+
|
42
|
+
require 'rake/rdoctask'
|
43
|
+
Rake::RDocTask.new do |rdoc|
|
44
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
45
|
+
|
46
|
+
rdoc.rdoc_dir = 'rdoc'
|
47
|
+
rdoc.title = "parallel_queue #{version}"
|
48
|
+
rdoc.rdoc_files.include('README*')
|
49
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
50
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/dequeue_demo.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__) + '/lib')
|
3
|
+
require 'redis'
|
4
|
+
require 'parallel_queue'
|
5
|
+
require 'pry'
|
6
|
+
require 'pry-nav'
|
7
|
+
require 'pry-stack_explorer'
|
8
|
+
|
9
|
+
redis = Redis.new(:host => '127.0.0.1', :port => '6379')
|
10
|
+
queue = ParallelQueue.new(redis, 'demo_queue')
|
11
|
+
|
12
|
+
filename = ARGV[0]
|
13
|
+
return puts "File name required" if filename.nil?
|
14
|
+
|
15
|
+
array = []
|
16
|
+
while array.empty? || queue.queue_count > 0
|
17
|
+
queue.dequeue_each() do |item|
|
18
|
+
array << item.to_i
|
19
|
+
puts "dequeue: #{item}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
File.open(filename, 'w') { |f| f.write(array.sort.join("\n")) }
|
data/enqueue_demo.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__) + '/lib')
|
3
|
+
require 'redis'
|
4
|
+
require 'parallel_queue'
|
5
|
+
require 'pry'
|
6
|
+
require 'pry-nav'
|
7
|
+
require 'pry-stack_explorer'
|
8
|
+
|
9
|
+
redis = Redis.new(:host => '127.0.0.1', :port => '6379')
|
10
|
+
queue = ParallelQueue.new(redis, 'demo_queue')
|
11
|
+
|
12
|
+
counter = 0
|
13
|
+
ids = ['abc', 'def', 'ghi', 'jkl', 'mno', 'pqrs']
|
14
|
+
while true
|
15
|
+
id = ids.sample
|
16
|
+
queue.enqueue(id, counter)
|
17
|
+
puts "enqueue: #{counter}"
|
18
|
+
counter += 1
|
19
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
class ParallelQueue
|
2
|
+
|
3
|
+
def initialize(redis, queue_name, options = {})
|
4
|
+
@redis = redis
|
5
|
+
@queue_name = queue_name
|
6
|
+
@maxlength = options[:maxlength] || nil
|
7
|
+
@lock_name = 'lock.' + @queue_name
|
8
|
+
@current_queue_index = 0
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
def delete_queue(id)
|
13
|
+
@redis.lrem(list_of_queue_names, 1, id)
|
14
|
+
@redis.del(queue_from_id(id))
|
15
|
+
end
|
16
|
+
|
17
|
+
def empty?
|
18
|
+
queue_count == 0
|
19
|
+
end
|
20
|
+
|
21
|
+
def queue_count
|
22
|
+
@redis.llen(list_of_queue_names)
|
23
|
+
end
|
24
|
+
|
25
|
+
# <tt>:item</tt>:: A string
|
26
|
+
#
|
27
|
+
def enqueue(id, item)
|
28
|
+
queue = queue_from_id(id)
|
29
|
+
@redis.rpush(queue, item)
|
30
|
+
@redis.ltrim(queue, -@maxlength, - 1) if @maxlength
|
31
|
+
@redis.lrem(list_of_queue_names, 1, id)
|
32
|
+
@redis.rpush(list_of_queue_names, id)
|
33
|
+
end
|
34
|
+
|
35
|
+
def dequeue
|
36
|
+
if acquire_lock || break_lock
|
37
|
+
current_id = @redis.lindex(list_of_queue_names, current_queue_index)
|
38
|
+
|
39
|
+
# pop from the current queue
|
40
|
+
current_queue = queue_from_id(current_id)
|
41
|
+
item = @redis.lpop(current_queue)
|
42
|
+
delete_queue(current_id) if @redis.llen(current_queue) == 0
|
43
|
+
|
44
|
+
increment_current_queue_index
|
45
|
+
release_lock
|
46
|
+
item
|
47
|
+
else # couldn't acquire or break the lock. wait and try again
|
48
|
+
sleep 0.01
|
49
|
+
dequeue
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def dequeue_each(&block)
|
54
|
+
return if queue_count == 0
|
55
|
+
self.current_queue_index = 0
|
56
|
+
|
57
|
+
begin
|
58
|
+
item = dequeue
|
59
|
+
yield(item) unless item.nil?
|
60
|
+
end while current_queue_index > 0 && current_queue_index < queue_count
|
61
|
+
end
|
62
|
+
|
63
|
+
def delete_all!
|
64
|
+
if acquire_lock || break_lock
|
65
|
+
while !empty?
|
66
|
+
delete_queue(@redis.lpop(list_of_queue_names))
|
67
|
+
end
|
68
|
+
@redis.del(list_of_queue_names)
|
69
|
+
self.current_queue_index = 0
|
70
|
+
release_lock
|
71
|
+
else # couldn't acquire or break the lock. wait and try again
|
72
|
+
sleep 0.01
|
73
|
+
delete_all!
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
|
78
|
+
def acquire_lock # :nodoc:
|
79
|
+
@redis.setnx(@lock_name, new_lock_expiration)
|
80
|
+
end
|
81
|
+
|
82
|
+
def release_lock # :nodoc:
|
83
|
+
@redis.del(@lock_name)
|
84
|
+
end
|
85
|
+
|
86
|
+
def break_lock # :nodoc:
|
87
|
+
previous = @redis.getset(@lock_name, new_lock_expiration)
|
88
|
+
previous.nil? || Time.at(previous.to_i) <= Time.now
|
89
|
+
end
|
90
|
+
|
91
|
+
protected
|
92
|
+
|
93
|
+
def increment_current_queue_index
|
94
|
+
self.current_queue_index = current_queue_index + 1
|
95
|
+
end
|
96
|
+
|
97
|
+
def current_queue_index # :nodoc:
|
98
|
+
return 0 if queue_count == 0
|
99
|
+
@current_queue_index % queue_count
|
100
|
+
end
|
101
|
+
|
102
|
+
def current_queue_index=(index) # :nodoc:
|
103
|
+
@current_queue_index = index
|
104
|
+
end
|
105
|
+
|
106
|
+
|
107
|
+
private
|
108
|
+
|
109
|
+
def list_of_queue_names
|
110
|
+
"#{@queue_name}_qs"
|
111
|
+
end
|
112
|
+
|
113
|
+
def queue_from_id(id)
|
114
|
+
"#{@queue_name}_q_#{id}"
|
115
|
+
end
|
116
|
+
|
117
|
+
|
118
|
+
LOCK_DURATION = 1
|
119
|
+
|
120
|
+
def new_lock_expiration
|
121
|
+
(Time.now + LOCK_DURATION).to_i
|
122
|
+
end
|
123
|
+
|
124
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "parallel_queue"
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Daniel Nelson"]
|
12
|
+
s.date = "2012-05-11"
|
13
|
+
s.description = "Motivation for creating this was for queueing messages based on ID so that very chatty message emitters don't prevent messages from others from being processed"
|
14
|
+
s.email = "daniel@populr.me"
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE",
|
17
|
+
"README.md"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".document",
|
21
|
+
".rspec",
|
22
|
+
".rvmrc",
|
23
|
+
"Gemfile",
|
24
|
+
"Gemfile.lock",
|
25
|
+
"LICENSE",
|
26
|
+
"README.md",
|
27
|
+
"Rakefile",
|
28
|
+
"VERSION",
|
29
|
+
"dequeue_demo.rb",
|
30
|
+
"enqueue_demo.rb",
|
31
|
+
"lib/parallel_queue.rb",
|
32
|
+
"parallel_queue.gemspec",
|
33
|
+
"spec/parallel_queue_spec.rb",
|
34
|
+
"spec/spec_helper.rb"
|
35
|
+
]
|
36
|
+
s.homepage = "http://github.com/populr/parallel_queue"
|
37
|
+
s.licenses = ["MIT"]
|
38
|
+
s.require_paths = ["lib"]
|
39
|
+
s.rubygems_version = "1.8.10"
|
40
|
+
s.summary = "A thread safe, Redis backed, parallel queue abstraction"
|
41
|
+
|
42
|
+
if s.respond_to? :specification_version then
|
43
|
+
s.specification_version = 3
|
44
|
+
|
45
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
46
|
+
s.add_runtime_dependency(%q<redis>, [">= 2.2.0"])
|
47
|
+
s.add_development_dependency(%q<rspec>, [">= 0"])
|
48
|
+
s.add_development_dependency(%q<bundler>, [">= 0"])
|
49
|
+
s.add_development_dependency(%q<jeweler>, [">= 0"])
|
50
|
+
s.add_development_dependency(%q<pry>, [">= 0"])
|
51
|
+
s.add_development_dependency(%q<pry-nav>, [">= 0"])
|
52
|
+
s.add_development_dependency(%q<pry-stack_explorer>, [">= 0"])
|
53
|
+
else
|
54
|
+
s.add_dependency(%q<redis>, [">= 2.2.0"])
|
55
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
56
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
57
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
58
|
+
s.add_dependency(%q<pry>, [">= 0"])
|
59
|
+
s.add_dependency(%q<pry-nav>, [">= 0"])
|
60
|
+
s.add_dependency(%q<pry-stack_explorer>, [">= 0"])
|
61
|
+
end
|
62
|
+
else
|
63
|
+
s.add_dependency(%q<redis>, [">= 2.2.0"])
|
64
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
65
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
66
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
67
|
+
s.add_dependency(%q<pry>, [">= 0"])
|
68
|
+
s.add_dependency(%q<pry-nav>, [">= 0"])
|
69
|
+
s.add_dependency(%q<pry-stack_explorer>, [">= 0"])
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
@@ -0,0 +1,259 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
describe "ParallelQueue" do
|
4
|
+
before(:each) do
|
5
|
+
@redis = Redis.new(:host => '127.0.0.1', :port => '6379')
|
6
|
+
@redis.del("demo_queue_qs")
|
7
|
+
@redis.del("demo_queue_q_abc")
|
8
|
+
@redis.del("demo_queue_q_peanuts")
|
9
|
+
@redis.del("demo_queue_q_123")
|
10
|
+
@queue = ParallelQueue.new(@redis, 'demo_queue')
|
11
|
+
end
|
12
|
+
|
13
|
+
after(:each) do
|
14
|
+
@queue.delete_all!
|
15
|
+
end
|
16
|
+
|
17
|
+
describe "clear" do
|
18
|
+
it "should" do
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe "#enqueue" do
|
23
|
+
context "when the specified id does not yet have a queue" do
|
24
|
+
it "should create a new queue" do
|
25
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
26
|
+
@queue.queue_count.should == 1
|
27
|
+
|
28
|
+
@queue.enqueue('123', 'hello world')
|
29
|
+
@queue.queue_count.should == 2
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context "when the specified id already has a queue" do
|
34
|
+
it "should not create a new queue" do
|
35
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
36
|
+
@queue.enqueue('peanuts', 'Charlie Brown')
|
37
|
+
@queue.queue_count.should == 1
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
context "when a maxlength is present, and that max is exceeded" do
|
42
|
+
it "should discard the oldest item" do
|
43
|
+
@queue = ParallelQueue.new(@redis, 'demo_queue', :maxlength => 3)
|
44
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
45
|
+
@queue.enqueue('peanuts', 'Woodstock')
|
46
|
+
@queue.enqueue('peanuts', 'Charlie Brown')
|
47
|
+
@queue.enqueue('peanuts', 'Lucy')
|
48
|
+
|
49
|
+
results = []
|
50
|
+
while !@queue.empty?
|
51
|
+
results << @queue.dequeue
|
52
|
+
end
|
53
|
+
|
54
|
+
results.should include('Woodstock')
|
55
|
+
results.should include('Charlie Brown')
|
56
|
+
results.should include('Lucy')
|
57
|
+
results.should_not include('Snoopy')
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
describe "#dequeue" do
|
63
|
+
it "sequential dequeues should iterate through and dequeue from the current queues" do
|
64
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
65
|
+
@queue.enqueue('peanuts', 'Charlie Brown')
|
66
|
+
@queue.enqueue('123', 'hello')
|
67
|
+
@queue.enqueue('123', 'world')
|
68
|
+
results = []
|
69
|
+
results << @queue.dequeue
|
70
|
+
results << @queue.dequeue
|
71
|
+
results.should include('Snoopy')
|
72
|
+
results.should include('hello')
|
73
|
+
end
|
74
|
+
|
75
|
+
context "when the queue from which the item was removed not empty" do
|
76
|
+
it "should not delete the empty queue" do
|
77
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
78
|
+
@queue.enqueue('peanuts', 'Charlie Brown')
|
79
|
+
@queue.enqueue('123', 'hello')
|
80
|
+
@queue.enqueue('123', 'world')
|
81
|
+
|
82
|
+
@queue.dequeue
|
83
|
+
@queue.queue_count.should == 2
|
84
|
+
@queue.dequeue
|
85
|
+
@queue.queue_count.should == 2
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
context "when the queue from which the item was removed is empty" do
|
90
|
+
it "should delete the empty queue" do
|
91
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
92
|
+
@queue.enqueue('123', 'hello world')
|
93
|
+
|
94
|
+
@queue.dequeue
|
95
|
+
@queue.queue_count.should == 1
|
96
|
+
@queue.dequeue
|
97
|
+
@queue.queue_count.should == 0
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
describe "#dequeue_each" do
|
103
|
+
it "should call dequeue on every queue, passing the response to a block" do
|
104
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
105
|
+
@queue.enqueue('peanuts', 'Charlie Brown')
|
106
|
+
@queue.enqueue('123', 'hello')
|
107
|
+
|
108
|
+
results = []
|
109
|
+
@queue.dequeue_each do |item|
|
110
|
+
results << item
|
111
|
+
end
|
112
|
+
results.should have(2).items
|
113
|
+
results.should include('Snoopy')
|
114
|
+
results.should include('hello')
|
115
|
+
|
116
|
+
results = []
|
117
|
+
@queue.dequeue_each do |item|
|
118
|
+
results << item
|
119
|
+
end
|
120
|
+
results.should eq(['Charlie Brown'])
|
121
|
+
end
|
122
|
+
|
123
|
+
context "when the situation arises in which another thread deletes a queue so that when we run dequeue, we receive nil" do
|
124
|
+
it "should not yield to the block" do
|
125
|
+
@queue.stub(:queue_count).and_return(1, 0)
|
126
|
+
results = []
|
127
|
+
@queue.dequeue_each do |item|
|
128
|
+
results << item
|
129
|
+
end
|
130
|
+
results.should have(0).items
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
|
136
|
+
describe "#delete_queue" do
|
137
|
+
it "should delete an entire queue, even if it is not empty" do
|
138
|
+
@queue.enqueue('123', 'hello world')
|
139
|
+
@queue.should_not be_empty
|
140
|
+
@queue.delete_queue('123')
|
141
|
+
@queue.should be_empty
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
describe "#delete_all!" do
|
146
|
+
it "should remove all queues" do
|
147
|
+
@queue.enqueue('123', 'hello world')
|
148
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
149
|
+
@queue.should_not be_empty
|
150
|
+
@queue.delete_all!
|
151
|
+
@queue.should be_empty
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
describe "#acquire_lock" do
|
156
|
+
after(:each) do
|
157
|
+
@queue.release_lock
|
158
|
+
end
|
159
|
+
|
160
|
+
context "when the lock is not already taken" do
|
161
|
+
it "should return true" do
|
162
|
+
@queue.acquire_lock.should be_true
|
163
|
+
end
|
164
|
+
|
165
|
+
it "should lock the resource" do
|
166
|
+
@queue.acquire_lock
|
167
|
+
@queue.acquire_lock.should be_false
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
context "when the lock is already taken" do
|
172
|
+
it "should return false" do
|
173
|
+
@queue.acquire_lock
|
174
|
+
@queue.acquire_lock.should be_false
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
describe "#break_lock" do
|
180
|
+
after(:each) do
|
181
|
+
@queue.release_lock
|
182
|
+
end
|
183
|
+
|
184
|
+
context "when the lock is not already taken" do
|
185
|
+
it "should be true" do
|
186
|
+
@queue.break_lock.should be_true
|
187
|
+
end
|
188
|
+
|
189
|
+
it "should lock the resource" do
|
190
|
+
@queue.break_lock
|
191
|
+
@queue.acquire_lock.should be_false
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
context "when the lock is taken and not yet expired" do
|
196
|
+
it "should be false" do
|
197
|
+
@queue.acquire_lock
|
198
|
+
@queue.break_lock.should be_false
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
context "when the lock is taken, but expired" do
|
203
|
+
it "should be true" do
|
204
|
+
@queue.acquire_lock
|
205
|
+
sleep(3)
|
206
|
+
@queue.break_lock.should be_true
|
207
|
+
end
|
208
|
+
|
209
|
+
it "should lock the resource" do
|
210
|
+
@queue.acquire_lock
|
211
|
+
sleep(3)
|
212
|
+
@queue.break_lock
|
213
|
+
@queue.acquire_lock.should be_false
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
it "should be able to break an expired lock acquired through breaking a lock" do
|
218
|
+
@queue.break_lock
|
219
|
+
sleep(3)
|
220
|
+
@queue.break_lock.should be_true
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
describe "#queue_count" do
|
225
|
+
context "when the queue is empty" do
|
226
|
+
it "should be 0" do
|
227
|
+
@queue.queue_count.should == 0
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
context "when the queue has N elements, regardless of whether or not they are ready to be dequeued" do
|
232
|
+
it "should match the number of elements" do
|
233
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
234
|
+
@queue.queue_count.should == 1
|
235
|
+
|
236
|
+
@queue.enqueue('123', 'hello')
|
237
|
+
@queue.queue_count.should == 2
|
238
|
+
|
239
|
+
@queue.enqueue('abc', 'a')
|
240
|
+
@queue.queue_count.should == 3
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
describe "#empty?" do
|
246
|
+
context "when the queue is empty" do
|
247
|
+
it "should be true" do
|
248
|
+
@queue.should be_empty
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
context "when the queue has an element that is not ready to be dequeued" do
|
253
|
+
it "should be false" do
|
254
|
+
@queue.enqueue('peanuts', 'Snoopy')
|
255
|
+
@queue.should_not be_empty
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
3
|
+
require 'rspec'
|
4
|
+
require 'parallel_queue'
|
5
|
+
require 'bundler'
|
6
|
+
Bundler.require
|
7
|
+
require 'pry'
|
8
|
+
require 'pry-nav'
|
9
|
+
require 'pry-stack_explorer'
|
10
|
+
|
11
|
+
# Requires supporting files with custom matchers and macros, etc,
|
12
|
+
# in ./support/ and its subdirectories.
|
13
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
14
|
+
|
15
|
+
RSpec.configure do |config|
|
16
|
+
|
17
|
+
end
|
metadata
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: parallel_queue
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Daniel Nelson
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-05-11 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: redis
|
16
|
+
requirement: &2161456900 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 2.2.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *2161456900
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: rspec
|
27
|
+
requirement: &2161489940 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :development
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2161489940
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: bundler
|
38
|
+
requirement: &2161488860 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :development
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *2161488860
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: jeweler
|
49
|
+
requirement: &2161488220 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :development
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *2161488220
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: pry
|
60
|
+
requirement: &2161487480 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ! '>='
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '0'
|
66
|
+
type: :development
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *2161487480
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry-nav
|
71
|
+
requirement: &2161486340 !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: *2161486340
|
80
|
+
- !ruby/object:Gem::Dependency
|
81
|
+
name: pry-stack_explorer
|
82
|
+
requirement: &2161483620 !ruby/object:Gem::Requirement
|
83
|
+
none: false
|
84
|
+
requirements:
|
85
|
+
- - ! '>='
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '0'
|
88
|
+
type: :development
|
89
|
+
prerelease: false
|
90
|
+
version_requirements: *2161483620
|
91
|
+
description: Motivation for creating this was for queueing messages based on ID so
|
92
|
+
that very chatty message emitters don't prevent messages from others from being
|
93
|
+
processed
|
94
|
+
email: daniel@populr.me
|
95
|
+
executables: []
|
96
|
+
extensions: []
|
97
|
+
extra_rdoc_files:
|
98
|
+
- LICENSE
|
99
|
+
- README.md
|
100
|
+
files:
|
101
|
+
- .document
|
102
|
+
- .rspec
|
103
|
+
- .rvmrc
|
104
|
+
- Gemfile
|
105
|
+
- Gemfile.lock
|
106
|
+
- LICENSE
|
107
|
+
- README.md
|
108
|
+
- Rakefile
|
109
|
+
- VERSION
|
110
|
+
- dequeue_demo.rb
|
111
|
+
- enqueue_demo.rb
|
112
|
+
- lib/parallel_queue.rb
|
113
|
+
- parallel_queue.gemspec
|
114
|
+
- spec/parallel_queue_spec.rb
|
115
|
+
- spec/spec_helper.rb
|
116
|
+
homepage: http://github.com/populr/parallel_queue
|
117
|
+
licenses:
|
118
|
+
- MIT
|
119
|
+
post_install_message:
|
120
|
+
rdoc_options: []
|
121
|
+
require_paths:
|
122
|
+
- lib
|
123
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
124
|
+
none: false
|
125
|
+
requirements:
|
126
|
+
- - ! '>='
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '0'
|
129
|
+
segments:
|
130
|
+
- 0
|
131
|
+
hash: -1593146586268589980
|
132
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
133
|
+
none: false
|
134
|
+
requirements:
|
135
|
+
- - ! '>='
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
requirements: []
|
139
|
+
rubyforge_project:
|
140
|
+
rubygems_version: 1.8.10
|
141
|
+
signing_key:
|
142
|
+
specification_version: 3
|
143
|
+
summary: A thread safe, Redis backed, parallel queue abstraction
|
144
|
+
test_files: []
|