girl_friday 0.9.4 → 0.9.5
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +2 -1
- data/.rvmrc +2 -2
- data/.travis.yml +1 -1
- data/Gemfile +1 -0
- data/History.md +10 -0
- data/README.md +17 -2
- data/Rakefile +1 -1
- data/examples/backends/mongo_persistence.rb +55 -0
- data/examples/backends/using_mongo.rb +22 -0
- data/lib/girl_friday/batch.rb +1 -0
- data/lib/girl_friday/error_handler.rb +15 -12
- data/lib/girl_friday/persistence.rb +1 -1
- data/lib/girl_friday/version.rb +1 -1
- data/lib/girl_friday/work_queue.rb +46 -39
- data/lib/girl_friday.rb +37 -10
- data/test/helper.rb +14 -4
- data/test/test_batch.rb +22 -3
- data/test/test_girl_friday.rb +21 -183
- data/test/test_girl_friday_immediately.rb +3 -30
- data/test/test_girl_friday_queue.rb +250 -0
- metadata +29 -20
- data/lib/girl_friday/timed_queue.rb +0 -48
- data/test/timed_queue.rb +0 -48
data/.gitignore
CHANGED
data/.rvmrc
CHANGED
@@ -1,3 +1,3 @@
|
|
1
1
|
rvm use jruby@gf --create
|
2
|
-
export RBXOPT=-Xrbc.db
|
3
|
-
export JRUBY_OPTS='--1.9
|
2
|
+
export RBXOPT=-Xrbc.db=".rbxdb"
|
3
|
+
export JRUBY_OPTS='--1.9'
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/History.md
CHANGED
@@ -1,6 +1,16 @@
|
|
1
1
|
Changes
|
2
2
|
================
|
3
3
|
|
4
|
+
0.9.5
|
5
|
+
---------
|
6
|
+
|
7
|
+
* Refactor thread usage so Queues can be properly shutdown and GC'd [GH-30]
|
8
|
+
* Use WeakRefs instead of ObjectSpace, as that plays better on JRuby.
|
9
|
+
* Can now pass a [connection\_pool](https://github.com/mperham/connection_pool) in as a Redis instance.
|
10
|
+
* Switch Redis.new to Redis.connect so a :url option can be passed in.
|
11
|
+
Nice for using on Heroku with Redis To Go.
|
12
|
+
* Allow stacking of error handlers, fixes GH-11
|
13
|
+
|
4
14
|
0.9.4
|
5
15
|
---------
|
6
16
|
|
data/README.md
CHANGED
@@ -32,20 +32,30 @@ In your Rails app, create a `config/initializers/girl_friday.rb` which defines y
|
|
32
32
|
EMAIL_QUEUE = GirlFriday::WorkQueue.new(:user_email, :size => 3) do |msg|
|
33
33
|
UserMailer.registration_email(msg).deliver
|
34
34
|
end
|
35
|
+
|
35
36
|
IMAGE_QUEUE = GirlFriday::WorkQueue.new(:image_crawler, :size => 7) do |msg|
|
36
37
|
ImageCrawler.process(msg)
|
37
38
|
end
|
38
39
|
|
39
|
-
SCRAPE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :host => 'host' }] do |msg|
|
40
|
+
SCRAPE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :host => 'host' }]) do |msg|
|
40
41
|
Page.scrape(msg)
|
41
42
|
end
|
42
43
|
|
43
|
-
TRANSCODE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => $redis }] do |msg|
|
44
|
+
TRANSCODE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => $redis }]) do |msg|
|
44
45
|
VideoProcessor.transcode(msg)
|
45
46
|
end
|
46
47
|
|
47
48
|
:size is the number of workers to spin up and defaults to 5. Keep in mind, ActiveRecord defaults to a connection pool size of 5 so if your workers are accessing the database you'll want to ensure that the connection pool is large enough by modifying `config/database.yml`.
|
48
49
|
|
50
|
+
You can use a connection pool to share a set of Redis connections with
|
51
|
+
other threads and GirlFriday queues using the `connection\_pool` gem:
|
52
|
+
|
53
|
+
require 'connection_pool'
|
54
|
+
redis_pool = ConnectionPool.new(:size => 5, :timeout => 5) { Redis.new }
|
55
|
+
CLEAN_FILTER_QUEUE = GirlFriday::WorkQueue.new(:clean_filter, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis_pool}]) do |msg|
|
56
|
+
Filter.clean(msg)
|
57
|
+
end
|
58
|
+
|
49
59
|
In your controller action or model, you can call `#push(msg)`
|
50
60
|
|
51
61
|
EMAIL_QUEUE.push(:email => @user.email, :name => @user.name)
|
@@ -56,6 +66,11 @@ Your message processing block should **not** access any instance data or variabl
|
|
56
66
|
|
57
67
|
You can call `GirlFriday::WorkQueue.immediate!` to process jobs immediately, which is helpful when testing. `GirlFriday::WorkQueue.queue!` will revert this & jobs will be processed by actors.
|
58
68
|
|
69
|
+
Queues are not garbage collected until they are shutdown, even if you
|
70
|
+
have no reference to them. Make sure you call `WorkQueue#shutdown` if you are
|
71
|
+
dynamically creating them so you don't leak memory. `GirlFriday.shutdown!` will shut down all
|
72
|
+
running queues in the process.
|
73
|
+
|
59
74
|
More Detail
|
60
75
|
--------------------
|
61
76
|
|
data/Rakefile
CHANGED
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'mongo'
|
2
|
+
|
3
|
+
# Add index on created_at
|
4
|
+
|
5
|
+
module GirlFriday
|
6
|
+
module Store
|
7
|
+
class Mongo
|
8
|
+
def initialize(name, options)
|
9
|
+
@opts = options
|
10
|
+
@key = "girl_friday-#{name}-#{environment}"
|
11
|
+
end
|
12
|
+
|
13
|
+
def push(work)
|
14
|
+
val = Marshal.dump(work)
|
15
|
+
collection.insert({ "work" => val })
|
16
|
+
end
|
17
|
+
alias_method :<<, :push
|
18
|
+
|
19
|
+
def pop
|
20
|
+
begin
|
21
|
+
val = collection.find_and_modify(:sort => [['$natural', 1]],
|
22
|
+
:remove => true)
|
23
|
+
Marshal.load(val["work"]) if val
|
24
|
+
rescue
|
25
|
+
# rescuing
|
26
|
+
# Database command 'findandmodify' failed: {"errmsg"=>"No matching object found", "ok"=>0.0}
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def size
|
31
|
+
collection.size
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def environment
|
37
|
+
ENV['RACK_ENV'] || ENV['RAILS_ENV'] || 'none'
|
38
|
+
end
|
39
|
+
|
40
|
+
def connection
|
41
|
+
@connection ||= (@opts.delete(:mongo) || ::Mongo::Connection.new("localhost", 27017, :pool_size => 5))
|
42
|
+
end
|
43
|
+
|
44
|
+
def db
|
45
|
+
db = @opts.delete(:db) || "girl_friday"
|
46
|
+
@db ||= connection[db]
|
47
|
+
end
|
48
|
+
|
49
|
+
def collection
|
50
|
+
@collection ||= db[@key]
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'rubygems' # for rbx
|
2
|
+
here = File.dirname(__FILE__)
|
3
|
+
$LOAD_PATH.unshift File.expand_path(here + '/../../lib')
|
4
|
+
$LOAD_PATH.unshift here
|
5
|
+
require 'girl_friday'
|
6
|
+
require 'mongo_persistence'
|
7
|
+
|
8
|
+
class Foo
|
9
|
+
def initialize(msg)
|
10
|
+
puts msg.inspect
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
PUTS_QUEUE = GirlFriday::WorkQueue.new(:puts_mongo, :store => GirlFriday::Store::Mongo) do |msg|
|
15
|
+
Foo.new(msg)
|
16
|
+
end
|
17
|
+
|
18
|
+
1.upto(100) do |i|
|
19
|
+
PUTS_QUEUE << { :what => i }
|
20
|
+
end
|
21
|
+
|
22
|
+
sleep 2
|
data/lib/girl_friday/batch.rb
CHANGED
@@ -1,22 +1,25 @@
|
|
1
1
|
module GirlFriday
|
2
|
-
|
3
|
-
|
4
|
-
$stderr.puts(ex)
|
5
|
-
$stderr.puts(ex.backtrace.join("\n"))
|
6
|
-
end
|
7
|
-
|
2
|
+
module ErrorHandler
|
3
|
+
|
8
4
|
def self.default
|
9
|
-
|
5
|
+
handlers = [Stderr]
|
6
|
+
handlers << Hoptoad if defined?(HoptoadNotifier)
|
7
|
+
handlers
|
8
|
+
end
|
9
|
+
|
10
|
+
class Stderr
|
11
|
+
def handle(ex)
|
12
|
+
$stderr.puts(ex)
|
13
|
+
$stderr.puts(ex.backtrace.join("\n"))
|
14
|
+
end
|
10
15
|
end
|
11
|
-
end
|
12
|
-
end
|
13
16
|
|
14
|
-
module GirlFriday
|
15
|
-
class ErrorHandler
|
16
17
|
class Hoptoad
|
17
18
|
def handle(ex)
|
18
19
|
HoptoadNotifier.notify_or_ignore(ex)
|
19
20
|
end
|
20
21
|
end
|
22
|
+
Airbrake = Hoptoad
|
23
|
+
|
21
24
|
end
|
22
|
-
end
|
25
|
+
end
|
data/lib/girl_friday/version.rb
CHANGED
@@ -11,14 +11,16 @@ module GirlFriday
|
|
11
11
|
@name = name.to_s
|
12
12
|
@size = options[:size] || 5
|
13
13
|
@processor = block
|
14
|
-
@
|
14
|
+
@error_handlers = (Array(options[:error_handler]) || ErrorHandler.default).map(&:new)
|
15
15
|
|
16
16
|
@shutdown = false
|
17
17
|
@busy_workers = []
|
18
18
|
@created_at = Time.now.to_i
|
19
19
|
@total_processed = @total_errors = @total_queued = 0
|
20
20
|
@persister = (options[:store] || Store::InMemory).new(name, (options[:store_config] || []))
|
21
|
+
@weakref = WeakRef.new(self)
|
21
22
|
start
|
23
|
+
GirlFriday.add_queue @weakref
|
22
24
|
end
|
23
25
|
|
24
26
|
def self.immediate!
|
@@ -37,13 +39,8 @@ module GirlFriday
|
|
37
39
|
result
|
38
40
|
end
|
39
41
|
|
40
|
-
|
41
|
-
|
42
|
-
alias_method :push_async, :push_immediately
|
43
|
-
else
|
44
|
-
def push_async(work, &block)
|
45
|
-
@supervisor << Work[work, block]
|
46
|
-
end
|
42
|
+
def push_async(work, &block)
|
43
|
+
@supervisor << Work[work, block]
|
47
44
|
end
|
48
45
|
alias_method :push, :push_async
|
49
46
|
alias_method :<<, :push_async
|
@@ -88,18 +85,17 @@ module GirlFriday
|
|
88
85
|
else
|
89
86
|
@busy_workers.delete(who.this)
|
90
87
|
ready_workers << who.this
|
91
|
-
shutdown_complete if @shutdown && @busy_workers.size == 0
|
92
88
|
end
|
93
89
|
rescue => ex
|
94
90
|
# Redis network error? Log and ignore.
|
95
|
-
@
|
91
|
+
@error_handlers.each { |handler| handler.handle(ex) }
|
96
92
|
end
|
97
93
|
|
98
94
|
def shutdown_complete
|
99
95
|
begin
|
100
96
|
@when_shutdown.call(self) if @when_shutdown
|
101
97
|
rescue Exception => ex
|
102
|
-
@
|
98
|
+
@error_handlers.each { |handler| handler.handle(ex) }
|
103
99
|
end
|
104
100
|
end
|
105
101
|
|
@@ -115,7 +111,7 @@ module GirlFriday
|
|
115
111
|
end
|
116
112
|
rescue => ex
|
117
113
|
# Redis network error? Log and ignore.
|
118
|
-
@
|
114
|
+
@error_handlers.each { |handler| handler.handle(ex) }
|
119
115
|
end
|
120
116
|
|
121
117
|
def ready_workers
|
@@ -133,39 +129,19 @@ module GirlFriday
|
|
133
129
|
@supervisor = Actor.spawn do
|
134
130
|
supervisor = Actor.current
|
135
131
|
@work_loop = Proc.new do
|
136
|
-
|
132
|
+
while !@shutdown do
|
137
133
|
work = Actor.receive
|
138
|
-
|
139
|
-
|
140
|
-
|
134
|
+
if !@shutdown
|
135
|
+
result = @processor.call(work.msg)
|
136
|
+
work.callback.call(result) if work.callback
|
137
|
+
supervisor << Ready[Actor.current]
|
138
|
+
end
|
141
139
|
end
|
142
140
|
end
|
143
141
|
|
144
142
|
Actor.trap_exit = true
|
145
143
|
begin
|
146
|
-
|
147
|
-
Actor.receive do |f|
|
148
|
-
f.when(Ready) do |who|
|
149
|
-
on_ready(who)
|
150
|
-
end
|
151
|
-
f.when(Work) do |work|
|
152
|
-
on_work(work)
|
153
|
-
end
|
154
|
-
f.when(Shutdown) do |stop|
|
155
|
-
@shutdown = true
|
156
|
-
@when_shutdown = stop.callback
|
157
|
-
shutdown_complete if @shutdown && @busy_workers.size == 0
|
158
|
-
end
|
159
|
-
f.when(Actor::DeadActorError) do |exit|
|
160
|
-
# TODO Provide current message contents as error context
|
161
|
-
@total_errors += 1
|
162
|
-
@busy_workers.delete(exit.actor)
|
163
|
-
ready_workers << Actor.spawn_link(&@work_loop)
|
164
|
-
@error_handler.handle(exit.reason)
|
165
|
-
end
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
144
|
+
supervisor_loop
|
169
145
|
rescue Exception => ex
|
170
146
|
$stderr.print "Fatal error in girl_friday: supervisor for #{name} died.\n"
|
171
147
|
$stderr.print("#{ex}\n")
|
@@ -185,6 +161,37 @@ module GirlFriday
|
|
185
161
|
end
|
186
162
|
end
|
187
163
|
|
164
|
+
def supervisor_loop
|
165
|
+
loop do
|
166
|
+
Actor.receive do |f|
|
167
|
+
f.when(Ready) do |who|
|
168
|
+
on_ready(who)
|
169
|
+
end
|
170
|
+
f.when(Work) do |work|
|
171
|
+
on_work(work)
|
172
|
+
end
|
173
|
+
f.when(Shutdown) do |stop|
|
174
|
+
@shutdown = true
|
175
|
+
@when_shutdown = stop.callback
|
176
|
+
@busy_workers.each { |w| w << stop }
|
177
|
+
ready_workers.each { |w| w << stop }
|
178
|
+
shutdown_complete
|
179
|
+
GirlFriday.remove_queue @weakref
|
180
|
+
return
|
181
|
+
end
|
182
|
+
f.when(Actor::DeadActorError) do |ex|
|
183
|
+
if !@shutdown
|
184
|
+
# TODO Provide current message contents as error context
|
185
|
+
@total_errors += 1
|
186
|
+
@busy_workers.delete(ex.actor)
|
187
|
+
ready_workers << Actor.spawn_link(&@work_loop)
|
188
|
+
@error_handlers.each { |handler| handler.handle(ex.reason) }
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
188
195
|
end
|
189
196
|
Queue = WorkQueue
|
190
197
|
end
|
data/lib/girl_friday.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'weakref'
|
1
2
|
require 'thread'
|
2
3
|
begin
|
3
4
|
# Rubinius
|
@@ -16,12 +17,34 @@ require 'girl_friday/batch'
|
|
16
17
|
|
17
18
|
module GirlFriday
|
18
19
|
|
20
|
+
@@lock = Mutex.new
|
21
|
+
|
22
|
+
def self.add_queue(ref)
|
23
|
+
@@lock.synchronize do
|
24
|
+
@queues ||= []
|
25
|
+
@queues.reject! { |q| !q.weakref_alive? }
|
26
|
+
@queues << ref
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.remove_queue(ref)
|
31
|
+
@@lock.synchronize do
|
32
|
+
@queues.delete ref
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
19
36
|
def self.queues
|
20
|
-
|
37
|
+
@queues || []
|
21
38
|
end
|
22
39
|
|
23
40
|
def self.status
|
24
|
-
queues.inject({})
|
41
|
+
queues.inject({}) do |memo, queue|
|
42
|
+
begin
|
43
|
+
memo = memo.merge(queue.__getobj__.status)
|
44
|
+
rescue WeakRef::RefError
|
45
|
+
end
|
46
|
+
memo
|
47
|
+
end
|
25
48
|
end
|
26
49
|
|
27
50
|
##
|
@@ -33,7 +56,7 @@ module GirlFriday
|
|
33
56
|
# Note that shutdown! just works with existing queues. If you create a
|
34
57
|
# new queue, it will act as normal.
|
35
58
|
def self.shutdown!(timeout=30)
|
36
|
-
qs = queues
|
59
|
+
qs = queues.select { |q| q.weakref_alive? }
|
37
60
|
count = qs.size
|
38
61
|
|
39
62
|
if count > 0
|
@@ -41,7 +64,15 @@ module GirlFriday
|
|
41
64
|
var = ConditionVariable.new
|
42
65
|
|
43
66
|
qs.each do |q|
|
44
|
-
q.
|
67
|
+
next if !q.weakref_alive?
|
68
|
+
begin
|
69
|
+
q.__getobj__.shutdown do |queue|
|
70
|
+
m.synchronize do
|
71
|
+
count -= 1
|
72
|
+
var.signal if count == 0
|
73
|
+
end
|
74
|
+
end
|
75
|
+
rescue WeakRef::RefError
|
45
76
|
m.synchronize do
|
46
77
|
count -= 1
|
47
78
|
var.signal if count == 0
|
@@ -52,19 +83,15 @@ module GirlFriday
|
|
52
83
|
m.synchronize do
|
53
84
|
var.wait(m, timeout)
|
54
85
|
end
|
55
|
-
#puts "girl_friday shutdown complete"
|
56
86
|
end
|
57
87
|
count
|
58
88
|
end
|
59
89
|
|
60
90
|
end
|
61
91
|
|
62
|
-
|
63
|
-
|
92
|
+
|
93
|
+
unless defined?($testing)
|
64
94
|
at_exit do
|
65
95
|
GirlFriday.shutdown!
|
66
96
|
end
|
67
|
-
rescue RuntimeError
|
68
|
-
$stderr.puts "[warn] girl_friday will not shut down cleanly, pass -X+O to JRuby to enable ObjectSpace"
|
69
97
|
end
|
70
|
-
|
data/test/helper.rb
CHANGED
@@ -1,19 +1,29 @@
|
|
1
1
|
$testing = true
|
2
|
+
puts RUBY_DESCRIPTION
|
3
|
+
|
4
|
+
at_exit do
|
5
|
+
if Thread.list.size > 1
|
6
|
+
Thread.list.each do |thread|
|
7
|
+
next if thread.status == 'run'
|
8
|
+
puts "WARNING: lingering threads found. All threads should be shutdown and garbage collected."
|
9
|
+
p [thread, thread['name']]
|
10
|
+
# puts thread.backtrace.join("\n")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
2
14
|
|
3
15
|
# require 'simplecov'
|
4
16
|
# SimpleCov.start do
|
5
17
|
# add_filter "/actor.rb"
|
6
18
|
# end
|
7
19
|
|
8
|
-
# rbx is 1.8-mode for another month...
|
9
20
|
require 'rubygems'
|
21
|
+
require 'minitest/spec'
|
10
22
|
require 'minitest/autorun'
|
11
|
-
require '
|
23
|
+
require 'connection_pool'
|
12
24
|
require 'girl_friday'
|
13
25
|
require 'flexmock/minitest'
|
14
26
|
|
15
|
-
puts RUBY_DESCRIPTION
|
16
|
-
|
17
27
|
class MiniTest::Unit::TestCase
|
18
28
|
|
19
29
|
def async_test(time=0.5)
|
data/test/test_batch.rb
CHANGED
@@ -3,7 +3,7 @@ require 'helper'
|
|
3
3
|
class TestBatch < MiniTest::Unit::TestCase
|
4
4
|
|
5
5
|
def test_simple_batch_operation
|
6
|
-
work = [
|
6
|
+
work = [0.5] * 10
|
7
7
|
a = Time.now
|
8
8
|
batch = GirlFriday::Batch.new(work, :size => 10) do |msg|
|
9
9
|
sleep msg
|
@@ -14,10 +14,29 @@ class TestBatch < MiniTest::Unit::TestCase
|
|
14
14
|
assert_in_delta(0.0, (b - a), 0.1)
|
15
15
|
|
16
16
|
# asking for the results should block
|
17
|
-
results = batch.results(
|
17
|
+
results = batch.results(1.0)
|
18
18
|
c = Time.now
|
19
|
-
assert_in_delta(
|
19
|
+
assert_in_delta(0.5, (c - b), 0.1)
|
20
|
+
|
20
21
|
assert_equal 10, results.size
|
21
22
|
assert_kind_of Time, results[0]
|
22
23
|
end
|
24
|
+
|
25
|
+
def test_batch_timeout
|
26
|
+
work = [0.1] * 4
|
27
|
+
work[2] = 0.4
|
28
|
+
batch = GirlFriday::Batch.new(work, :size => 4) do |msg|
|
29
|
+
sleep msg
|
30
|
+
'x'
|
31
|
+
end
|
32
|
+
results = batch.results(0.3)
|
33
|
+
assert_equal 'x', results[0]
|
34
|
+
assert_equal 'x', results[1]
|
35
|
+
assert_nil results[2]
|
36
|
+
assert_equal 'x', results[3]
|
37
|
+
|
38
|
+
# Necessary to work around a Ruby 1.9.2 bug
|
39
|
+
# http://redmine.ruby-lang.org/issues/5342
|
40
|
+
sleep 0.1
|
41
|
+
end
|
23
42
|
end
|
data/test/test_girl_friday.rb
CHANGED
@@ -2,198 +2,36 @@ require 'helper'
|
|
2
2
|
|
3
3
|
class TestGirlFriday < MiniTest::Unit::TestCase
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
def test_should_process_messages
|
10
|
-
async_test do |cb|
|
11
|
-
queue = GirlFriday::WorkQueue.new('test') do |msg|
|
12
|
-
assert_equal 'foo', msg[:text]
|
13
|
-
cb.call
|
14
|
-
end
|
15
|
-
queue.push(:text => 'foo')
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def test_should_handle_worker_error
|
20
|
-
async_test do |cb|
|
21
|
-
TestErrorHandler.send(:define_method, :handle) do |ex|
|
22
|
-
assert_equal 'oops', ex.message
|
23
|
-
assert_equal 'RuntimeError', ex.class.name
|
24
|
-
cb.call
|
25
|
-
end
|
26
|
-
|
27
|
-
queue = GirlFriday::WorkQueue.new('test', :error_handler => TestErrorHandler) do |msg|
|
28
|
-
raise 'oops'
|
29
|
-
end
|
30
|
-
queue.push(:text => 'foo')
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def test_should_call_callback_when_complete
|
35
|
-
async_test do |cb|
|
36
|
-
queue = GirlFriday::WorkQueue.new('test', :size => 1) do |msg|
|
37
|
-
assert_equal 'foo', msg[:text]
|
38
|
-
'camel'
|
39
|
-
end
|
40
|
-
queue.push(:text => 'foo') do |result|
|
41
|
-
assert_equal 'camel', result
|
42
|
-
cb.call
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def test_should_provide_status
|
48
|
-
mutex = Mutex.new
|
49
|
-
total = 200
|
50
|
-
count = 0
|
51
|
-
incr = Proc.new do
|
52
|
-
mutex.synchronize do
|
53
|
-
count += 1
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
actual = nil
|
58
|
-
async_test do |cb|
|
59
|
-
queue = GirlFriday::WorkQueue.new('image_crawler', :size => 3) do |msg|
|
60
|
-
mycount = incr.call
|
61
|
-
actual = queue.status if mycount == 100
|
62
|
-
cb.call if mycount == total
|
63
|
-
end
|
64
|
-
total.times do |idx|
|
65
|
-
queue.push(:text => 'foo')
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
refute_nil actual
|
70
|
-
refute_nil actual['image_crawler']
|
71
|
-
metrics = actual['image_crawler']
|
72
|
-
assert metrics[:total_queued] > 0
|
73
|
-
assert metrics[:total_queued] <= total
|
74
|
-
assert_equal 3, metrics[:pool_size]
|
75
|
-
assert_equal 3, metrics[:busy]
|
76
|
-
assert_equal 0, metrics[:ready]
|
77
|
-
assert(metrics[:backlog] > 0)
|
78
|
-
assert(metrics[:total_processed] > 0)
|
79
|
-
end
|
80
|
-
|
81
|
-
def test_should_persist_with_redis
|
82
|
-
begin
|
83
|
-
require 'redis'
|
84
|
-
redis = Redis.new
|
85
|
-
redis.flushdb
|
86
|
-
rescue LoadError
|
87
|
-
return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
|
88
|
-
rescue Errno::ECONNREFUSED
|
89
|
-
return puts 'Skipping redis test, not running locally'
|
5
|
+
describe 'GirlFriday' do
|
6
|
+
after do
|
7
|
+
GirlFriday.shutdown!
|
90
8
|
end
|
91
9
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
mutex.synchronize do
|
97
|
-
count += 1
|
10
|
+
describe '.status' do
|
11
|
+
before do
|
12
|
+
q1 = GirlFriday::Queue.new(:q1) do; end
|
13
|
+
q2 = GirlFriday::Queue.new(:q2) do; end
|
98
14
|
end
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
cb.call if count == total
|
105
|
-
end
|
106
|
-
total.times do
|
107
|
-
queue.push(:text => 'foo')
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
def test_should_persist_with_redis_instance
|
113
|
-
begin
|
114
|
-
require 'redis'
|
115
|
-
redis = Redis.new
|
116
|
-
redis.flushdb
|
117
|
-
rescue LoadError
|
118
|
-
return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
|
119
|
-
rescue Errno::ECONNREFUSED
|
120
|
-
return puts 'Skipping redis test, not running locally accepting connections over UNIX socket'
|
121
|
-
end
|
122
|
-
|
123
|
-
mutex = Mutex.new
|
124
|
-
total = 100
|
125
|
-
count = 0
|
126
|
-
incr = Proc.new do
|
127
|
-
mutex.synchronize do
|
128
|
-
count += 1
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
async_test do |cb|
|
133
|
-
queue = GirlFriday::WorkQueue.new('test', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
|
134
|
-
incr.call
|
135
|
-
cb.call if count == total
|
136
|
-
end
|
137
|
-
total.times do
|
138
|
-
queue.push(:text => 'foo')
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
|
143
|
-
def test_should_allow_graceful_shutdown
|
144
|
-
mutex = Mutex.new
|
145
|
-
total = 100
|
146
|
-
count = 0
|
147
|
-
incr = Proc.new do
|
148
|
-
mutex.synchronize do
|
149
|
-
count += 1
|
15
|
+
it 'provides a status structure for each live queue' do
|
16
|
+
hash = GirlFriday.status
|
17
|
+
assert_kind_of Hash, hash
|
18
|
+
assert_equal 2, GirlFriday.queues.size
|
19
|
+
assert_equal 2, hash.size
|
150
20
|
end
|
151
21
|
end
|
152
22
|
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
23
|
+
describe '.shutdown!' do
|
24
|
+
before do
|
25
|
+
q1 = GirlFriday::Queue.new(:q1) do; end
|
26
|
+
q2 = GirlFriday::Queue.new(:q2) do; end
|
157
27
|
end
|
158
|
-
|
159
|
-
|
28
|
+
it 'provides a status structure for each live queue' do
|
29
|
+
a = Time.now
|
30
|
+
assert_equal 0, GirlFriday.shutdown!
|
31
|
+
assert_in_delta 0, Time.now - a, 0.1
|
32
|
+
assert_equal 0, GirlFriday.queues.size
|
160
33
|
end
|
161
|
-
|
162
|
-
count = GirlFriday.shutdown!
|
163
|
-
assert_equal 0, count
|
164
|
-
s = queue.status
|
165
|
-
assert_equal 0, s['shutdown'][:busy]
|
166
|
-
assert_equal 2, s['shutdown'][:ready]
|
167
|
-
assert(s['shutdown'][:backlog] > 0)
|
168
|
-
cb.call
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
|
-
def test_should_create_workers_lazily
|
173
|
-
async_test do |cb|
|
174
|
-
queue = GirlFriday::Queue.new('shutdown', :size => 2) do |msg|
|
175
|
-
assert_equal 1, queue.instance_variable_get(:@ready_workers).size
|
176
|
-
cb.call
|
177
|
-
end
|
178
|
-
refute queue.instance_variable_defined?(:@ready_workers)
|
179
|
-
# don't instantiate the worker threads until we actually put
|
180
|
-
# work onto the queue.
|
181
|
-
queue << 'empty msg'
|
182
|
-
end
|
183
|
-
end
|
184
|
-
|
185
|
-
def test_stubbing_girl_friday_with_flexmock
|
186
|
-
expected = Thread.current.to_s
|
187
|
-
actual = nil
|
188
|
-
processor = Proc.new do |msg|
|
189
|
-
actual = Thread.current.to_s
|
190
|
-
end
|
191
|
-
queue = GirlFriday::Queue.new('shutdown', :size => 2, &processor)
|
192
|
-
flexmock(queue).should_receive(:push).zero_or_more_times.and_return do |msg|
|
193
|
-
processor.call(msg)
|
194
34
|
end
|
195
|
-
queue.push 'hello world!'
|
196
|
-
assert_equal expected, actual
|
197
35
|
end
|
198
36
|
|
199
37
|
end
|
@@ -16,6 +16,7 @@ class TestGirlFridayImmediately < MiniTest::Unit::TestCase
|
|
16
16
|
end
|
17
17
|
assert_equal 42, queue.push(:start => 41)
|
18
18
|
assert_equal 42, queue << { :start => 41 }
|
19
|
+
queue.shutdown
|
19
20
|
end
|
20
21
|
|
21
22
|
def test_should_process_immediately_with_callback
|
@@ -23,6 +24,7 @@ class TestGirlFridayImmediately < MiniTest::Unit::TestCase
|
|
23
24
|
msg[:start] + 1
|
24
25
|
end
|
25
26
|
assert_equal 43, queue.push(:start => 41) { |r| r + 1 }
|
27
|
+
queue.shutdown
|
26
28
|
end
|
27
29
|
|
28
30
|
def test_should_process_style_idempotently
|
@@ -45,35 +47,6 @@ class TestGirlFridayImmediately < MiniTest::Unit::TestCase
|
|
45
47
|
GirlFriday::WorkQueue.immediate!
|
46
48
|
assert_equal 3, queue.push(:start => 2)
|
47
49
|
assert_equal 3, queue << {:start => 2}
|
48
|
-
|
49
|
-
|
50
|
-
def test_should_process_immediately_when_rails_dev
|
51
|
-
# remove WorkQueue definitions first, since #push_async is defined
|
52
|
-
# as the class definition is executed, and Rails constant is not defined yet
|
53
|
-
[:Queue, :WorkQueue].each {|c| GirlFriday.send(:remove_const, c)}
|
54
|
-
|
55
|
-
# now top-level Rails constant will exist
|
56
|
-
Object.send(:include, RailsEnvironment)
|
57
|
-
|
58
|
-
# re-load class now that Rails constant is defined
|
59
|
-
$".delete_if {|f| f =~ %r{work_queue.rb}}
|
60
|
-
require 'girl_friday/work_queue'
|
61
|
-
|
62
|
-
# verify that we process immediately
|
63
|
-
queue = GirlFriday::WorkQueue.new('now') do |msg|
|
64
|
-
msg[:start] + 1
|
65
|
-
end
|
66
|
-
assert_equal 4, queue.push(:start => 3)
|
67
|
-
assert_equal 4, queue << {:start => 3}
|
50
|
+
queue.shutdown
|
68
51
|
end
|
69
52
|
end
|
70
|
-
|
71
|
-
module RailsEnvironment
|
72
|
-
class Rails
|
73
|
-
def self.method_missing(m, *args, &block)
|
74
|
-
# swallow every call and return self so that WorkQueue
|
75
|
-
# thinks Rails dev environment is loaded
|
76
|
-
self
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|
@@ -0,0 +1,250 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class TestGirlFriday < MiniTest::Unit::TestCase
|
4
|
+
|
5
|
+
class TestErrorHandler
|
6
|
+
include MiniTest::Assertions
|
7
|
+
end
|
8
|
+
|
9
|
+
def test_should_process_messages
|
10
|
+
async_test do |cb|
|
11
|
+
queue = GirlFriday::WorkQueue.new('process') do |msg|
|
12
|
+
assert_equal 'foo', msg[:text]
|
13
|
+
queue.shutdown do
|
14
|
+
cb.call
|
15
|
+
end
|
16
|
+
end
|
17
|
+
queue.push(:text => 'foo')
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_should_handle_worker_error
|
22
|
+
async_test do |cb|
|
23
|
+
queue = nil
|
24
|
+
TestErrorHandler.send(:define_method, :handle) do |ex|
|
25
|
+
assert_equal 'oops', ex.message
|
26
|
+
assert_equal 'RuntimeError', ex.class.name
|
27
|
+
queue.shutdown do
|
28
|
+
cb.call
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
queue = GirlFriday::WorkQueue.new('error', :error_handler => TestErrorHandler) do |msg|
|
33
|
+
raise 'oops'
|
34
|
+
end
|
35
|
+
queue.push(:text => 'foo')
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_should_call_callback_when_complete
|
40
|
+
async_test do |cb|
|
41
|
+
queue = GirlFriday::WorkQueue.new('callback', :size => 1) do |msg|
|
42
|
+
assert_equal 'foo', msg[:text]
|
43
|
+
'camel'
|
44
|
+
end
|
45
|
+
queue.push(:text => 'foo') do |result|
|
46
|
+
assert_equal 'camel', result
|
47
|
+
queue.shutdown do
|
48
|
+
cb.call
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def test_should_provide_status
|
55
|
+
mutex = Mutex.new
|
56
|
+
total = 200
|
57
|
+
count = 0
|
58
|
+
incr = Proc.new do
|
59
|
+
mutex.synchronize do
|
60
|
+
count += 1
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
actual = nil
|
65
|
+
async_test do |cb|
|
66
|
+
queue = GirlFriday::WorkQueue.new('status', :size => 3) do |msg|
|
67
|
+
mycount = incr.call
|
68
|
+
actual = queue.status if mycount == 100
|
69
|
+
queue.shutdown do
|
70
|
+
cb.call
|
71
|
+
end if mycount == total
|
72
|
+
end
|
73
|
+
total.times do |idx|
|
74
|
+
queue.push(:text => 'foo')
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
refute_nil actual
|
79
|
+
refute_nil actual['status']
|
80
|
+
metrics = actual['status']
|
81
|
+
assert metrics[:total_queued] > 0
|
82
|
+
assert metrics[:total_queued] <= total
|
83
|
+
assert_equal 3, metrics[:pool_size]
|
84
|
+
assert_equal 3, metrics[:busy]
|
85
|
+
assert_equal 0, metrics[:ready]
|
86
|
+
assert(metrics[:backlog] > 0)
|
87
|
+
assert(metrics[:total_processed] > 0)
|
88
|
+
end
|
89
|
+
|
90
|
+
def test_should_persist_with_redis
|
91
|
+
begin
|
92
|
+
require 'redis'
|
93
|
+
redis = Redis.new
|
94
|
+
redis.flushdb
|
95
|
+
rescue LoadError
|
96
|
+
return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
|
97
|
+
rescue Errno::ECONNREFUSED
|
98
|
+
return puts 'Skipping redis test, not running locally'
|
99
|
+
end
|
100
|
+
|
101
|
+
mutex = Mutex.new
|
102
|
+
total = 100
|
103
|
+
count = 0
|
104
|
+
incr = Proc.new do
|
105
|
+
mutex.synchronize do
|
106
|
+
count += 1
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
async_test(1.0) do |cb|
|
111
|
+
queue = GirlFriday::WorkQueue.new('redis', :size => 2, :store => GirlFriday::Store::Redis) do |msg|
|
112
|
+
incr.call
|
113
|
+
queue.shutdown do
|
114
|
+
cb.call
|
115
|
+
end if count == total
|
116
|
+
end
|
117
|
+
total.times do
|
118
|
+
queue.push(:text => 'foo')
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def test_should_persist_with_redis_instance
|
124
|
+
begin
|
125
|
+
require 'redis'
|
126
|
+
redis = Redis.new
|
127
|
+
redis.flushdb
|
128
|
+
rescue LoadError
|
129
|
+
return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
|
130
|
+
rescue Errno::ECONNREFUSED
|
131
|
+
return puts 'Skipping redis test, not running locally'
|
132
|
+
end
|
133
|
+
|
134
|
+
mutex = Mutex.new
|
135
|
+
total = 100
|
136
|
+
count = 0
|
137
|
+
incr = Proc.new do
|
138
|
+
mutex.synchronize do
|
139
|
+
count += 1
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
async_test(1.0) do |cb|
|
144
|
+
queue = GirlFriday::WorkQueue.new('redis-instance', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
|
145
|
+
incr.call
|
146
|
+
queue.shutdown do
|
147
|
+
cb.call
|
148
|
+
end if count == total
|
149
|
+
end
|
150
|
+
total.times do
|
151
|
+
queue.push(:text => 'foo')
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def test_should_persist_with_redis_connection_pool
|
157
|
+
begin
|
158
|
+
require 'redis'
|
159
|
+
require 'connection_pool'
|
160
|
+
redis = ConnectionPool.new(:size => 5, :timeout => 5){ Redis.new }
|
161
|
+
redis.flushdb
|
162
|
+
rescue LoadError
|
163
|
+
return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
|
164
|
+
rescue Errno::ECONNREFUSED
|
165
|
+
return puts 'Skipping redis test, not running locally'
|
166
|
+
end
|
167
|
+
|
168
|
+
mutex = Mutex.new
|
169
|
+
total = 100
|
170
|
+
count = 0
|
171
|
+
incr = Proc.new do
|
172
|
+
mutex.synchronize do
|
173
|
+
count += 1
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
async_test(1.0) do |cb|
|
178
|
+
queue = GirlFriday::WorkQueue.new('redis-pool', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
|
179
|
+
incr.call
|
180
|
+
queue.shutdown do
|
181
|
+
cb.call
|
182
|
+
end if count == total
|
183
|
+
end
|
184
|
+
total.times do
|
185
|
+
queue.push(:text => 'foo')
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def test_should_allow_graceful_shutdown
|
191
|
+
mutex = Mutex.new
|
192
|
+
total = 100
|
193
|
+
count = 0
|
194
|
+
incr = Proc.new do
|
195
|
+
mutex.synchronize do
|
196
|
+
count += 1
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
async_test do |cb|
|
201
|
+
queue = GirlFriday::WorkQueue.new('shutdown', :size => 2) do |msg|
|
202
|
+
incr.call
|
203
|
+
cb.call if count == total
|
204
|
+
end
|
205
|
+
total.times do
|
206
|
+
queue.push(:text => 'foo')
|
207
|
+
end
|
208
|
+
|
209
|
+
assert_equal 1, GirlFriday.queues.size
|
210
|
+
count = GirlFriday.shutdown!
|
211
|
+
assert_equal 0, count
|
212
|
+
cb.call
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
def test_should_create_workers_lazily
|
217
|
+
async_test do |cb|
|
218
|
+
queue = GirlFriday::Queue.new('lazy', :size => 2) do |msg|
|
219
|
+
assert_equal 1, queue.instance_variable_get(:@ready_workers).size
|
220
|
+
queue.shutdown do
|
221
|
+
cb.call
|
222
|
+
end
|
223
|
+
end
|
224
|
+
refute queue.instance_variable_defined?(:@ready_workers)
|
225
|
+
# don't instantiate the worker threads until we actually put
|
226
|
+
# work onto the queue.
|
227
|
+
queue << 'empty msg'
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
def test_stubbing_girl_friday_with_flexmock
|
232
|
+
expected = Thread.current.to_s
|
233
|
+
actual = nil
|
234
|
+
processor = Proc.new do |msg|
|
235
|
+
actual = Thread.current.to_s
|
236
|
+
end
|
237
|
+
async_test do |cb|
|
238
|
+
queue = GirlFriday::Queue.new('flexmock', :size => 2, &processor)
|
239
|
+
flexmock(queue).should_receive(:push).zero_or_more_times.and_return do |msg|
|
240
|
+
processor.call(msg)
|
241
|
+
end
|
242
|
+
queue.push 'hello world!'
|
243
|
+
assert_equal expected, actual
|
244
|
+
queue.shutdown do
|
245
|
+
cb.call
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
end
|
metadata
CHANGED
@@ -1,38 +1,39 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: girl_friday
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
|
4
|
+
prerelease:
|
5
|
+
version: 0.9.5
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Mike Perham
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-
|
12
|
+
date: 2011-09-22 00:00:00.000000000 -07:00
|
13
|
+
default_executable:
|
13
14
|
dependencies:
|
14
15
|
- !ruby/object:Gem::Dependency
|
15
16
|
name: sinatra
|
16
|
-
|
17
|
-
none: false
|
17
|
+
version_requirements: &2086 !ruby/object:Gem::Requirement
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '1.0'
|
22
|
-
|
22
|
+
none: false
|
23
|
+
requirement: *2086
|
23
24
|
prerelease: false
|
24
|
-
|
25
|
+
type: :development
|
25
26
|
- !ruby/object:Gem::Dependency
|
26
27
|
name: rake
|
27
|
-
|
28
|
-
none: false
|
28
|
+
version_requirements: &2104 !ruby/object:Gem::Requirement
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '0'
|
33
|
-
|
33
|
+
none: false
|
34
|
+
requirement: *2104
|
34
35
|
prerelease: false
|
35
|
-
|
36
|
+
type: :development
|
36
37
|
description: Background processing, simplified
|
37
38
|
email:
|
38
39
|
- mperham@gmail.com
|
@@ -49,6 +50,8 @@ files:
|
|
49
50
|
- Rakefile
|
50
51
|
- TODO.md
|
51
52
|
- config.ru
|
53
|
+
- examples/backends/mongo_persistence.rb
|
54
|
+
- examples/backends/using_mongo.rb
|
52
55
|
- examples/batch.rb
|
53
56
|
- examples/pipeline.rb
|
54
57
|
- girl_friday.gemspec
|
@@ -59,7 +62,6 @@ files:
|
|
59
62
|
- lib/girl_friday/monkey_patches.rb
|
60
63
|
- lib/girl_friday/persistence.rb
|
61
64
|
- lib/girl_friday/server.rb
|
62
|
-
- lib/girl_friday/timed_queue.rb
|
63
65
|
- lib/girl_friday/version.rb
|
64
66
|
- lib/girl_friday/work_queue.rb
|
65
67
|
- server/public/css/style.css
|
@@ -70,29 +72,36 @@ files:
|
|
70
72
|
- test/test_batch.rb
|
71
73
|
- test/test_girl_friday.rb
|
72
74
|
- test/test_girl_friday_immediately.rb
|
73
|
-
- test/
|
75
|
+
- test/test_girl_friday_queue.rb
|
76
|
+
has_rdoc: true
|
74
77
|
homepage: http://github.com/mperham/girl_friday
|
75
78
|
licenses: []
|
76
|
-
post_install_message:
|
79
|
+
post_install_message:
|
77
80
|
rdoc_options: []
|
78
81
|
require_paths:
|
79
82
|
- lib
|
80
83
|
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
84
|
requirements:
|
83
85
|
- - ! '>='
|
84
86
|
- !ruby/object:Gem::Version
|
85
87
|
version: '0'
|
86
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
88
|
none: false
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
88
90
|
requirements:
|
89
91
|
- - ! '>='
|
90
92
|
- !ruby/object:Gem::Version
|
91
93
|
version: '0'
|
94
|
+
none: false
|
92
95
|
requirements: []
|
93
96
|
rubyforge_project: girl_friday
|
94
|
-
rubygems_version: 1.
|
95
|
-
signing_key:
|
97
|
+
rubygems_version: 1.5.1
|
98
|
+
signing_key:
|
96
99
|
specification_version: 3
|
97
100
|
summary: Background processing, simplified
|
98
|
-
test_files:
|
101
|
+
test_files:
|
102
|
+
- test/helper.rb
|
103
|
+
- test/test_batch.rb
|
104
|
+
- test/test_girl_friday.rb
|
105
|
+
- test/test_girl_friday_immediately.rb
|
106
|
+
- test/test_girl_friday_queue.rb
|
107
|
+
...
|
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'thread'
|
2
|
-
require 'timeout'
|
3
|
-
|
4
|
-
class TimedQueue
|
5
|
-
def initialize
|
6
|
-
@que = []
|
7
|
-
@waiting = []
|
8
|
-
@mutex = Mutex.new
|
9
|
-
@resource = ConditionVariable.new
|
10
|
-
end
|
11
|
-
|
12
|
-
def push(obj)
|
13
|
-
@mutex.synchronize do
|
14
|
-
@que.push obj
|
15
|
-
@resource.signal
|
16
|
-
end
|
17
|
-
end
|
18
|
-
alias << push
|
19
|
-
|
20
|
-
def timed_pop(timeout=0.5)
|
21
|
-
while true
|
22
|
-
@mutex.synchronize do
|
23
|
-
@waiting.delete(Thread.current)
|
24
|
-
if @que.empty?
|
25
|
-
@waiting.push Thread.current
|
26
|
-
@resource.wait(@mutex, timeout)
|
27
|
-
raise TimeoutError if @que.empty?
|
28
|
-
else
|
29
|
-
retval = @que.shift
|
30
|
-
@resource.signal
|
31
|
-
return retval
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def empty?
|
38
|
-
@que.empty?
|
39
|
-
end
|
40
|
-
|
41
|
-
def clear
|
42
|
-
@que.clear
|
43
|
-
end
|
44
|
-
|
45
|
-
def length
|
46
|
-
@que.length
|
47
|
-
end
|
48
|
-
end
|
data/test/timed_queue.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'thread'
|
2
|
-
require 'timeout'
|
3
|
-
|
4
|
-
class TimedQueue
|
5
|
-
def initialize
|
6
|
-
@que = []
|
7
|
-
@waiting = []
|
8
|
-
@mutex = Mutex.new
|
9
|
-
@resource = ConditionVariable.new
|
10
|
-
end
|
11
|
-
|
12
|
-
def push(obj)
|
13
|
-
@mutex.synchronize do
|
14
|
-
@que.push obj
|
15
|
-
@resource.signal
|
16
|
-
end
|
17
|
-
end
|
18
|
-
alias << push
|
19
|
-
|
20
|
-
def timed_pop(timeout=0.5)
|
21
|
-
while true
|
22
|
-
@mutex.synchronize do
|
23
|
-
@waiting.delete(Thread.current)
|
24
|
-
if @que.empty?
|
25
|
-
@waiting.push Thread.current
|
26
|
-
@resource.wait(@mutex, timeout)
|
27
|
-
raise TimeoutError if @que.empty?
|
28
|
-
else
|
29
|
-
retval = @que.shift
|
30
|
-
@resource.signal
|
31
|
-
return retval
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def empty?
|
38
|
-
@que.empty?
|
39
|
-
end
|
40
|
-
|
41
|
-
def clear
|
42
|
-
@que.clear
|
43
|
-
end
|
44
|
-
|
45
|
-
def length
|
46
|
-
@que.length
|
47
|
-
end
|
48
|
-
end
|