perfectqueue 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ module PerfectQueue
2
+
3
+ VERSION = '0.7.0'
4
+
5
+ end
@@ -0,0 +1,211 @@
1
+
2
+ module PerfectQueue
3
+
4
+
5
+ class MonitorThread
6
+ def initialize(engine, conf)
7
+ @engine = engine
8
+ @log = @engine.log
9
+ @backend = engine.backend
10
+
11
+ @timeout = conf[:timeout] || 600
12
+ @heartbeat_interval = conf[:heartbeat_interval] || @timeout*3/4
13
+ @kill_timeout = conf[:kill_timeout] || @timeout*10
14
+ @kill_interval = conf[:kill_interval] || 60
15
+ @retry_wait = conf[:retry_wait] || nil
16
+
17
+ @token = nil
18
+ @heartbeat_time = nil
19
+ @kill_time = nil
20
+ @kill_proc = nil
21
+ @canceled = false
22
+ @mutex = Mutex.new
23
+ @cond = ConditionVariable.new
24
+ end
25
+
26
+ def start
27
+ @log.debug "running worker."
28
+ @thread = Thread.new(&method(:run))
29
+ end
30
+
31
+ def run
32
+ until @engine.finished?
33
+ @mutex.synchronize {
34
+ while true
35
+ return if @engine.finished?
36
+ break if @token
37
+ @cond.wait(@mutex)
38
+ end
39
+ }
40
+ while true
41
+ sleep 1
42
+ @mutex.synchronize {
43
+ return if @engine.finished?
44
+ break unless @token
45
+ now = Time.now.to_i
46
+ try_extend(now)
47
+ try_kill(now)
48
+ }
49
+ end
50
+ end
51
+ rescue
52
+ @engine.stop($!)
53
+ end
54
+
55
+ def try_extend(now)
56
+ if now >= @heartbeat_time && !@canceled
57
+ @log.debug "extending timeout=#{now+@timeout} id=#{@task.id}"
58
+ begin
59
+ @backend.update(@token, now+@timeout)
60
+ rescue CanceledError
61
+ @log.info "task id=#{@task.id} is canceled."
62
+ @canceled = true
63
+ @kill_time = now
64
+ end
65
+ @heartbeat_time = now + @heartbeat_interval
66
+ end
67
+ end
68
+
69
+ def try_kill(now)
70
+ if now >= @kill_time
71
+ kill!
72
+ @kill_time = now + @kill_interval
73
+ end
74
+ end
75
+
76
+ def kill!
77
+ if @kill_proc
78
+ @log.info "killing #{@task.id}..."
79
+ @kill_proc.call rescue nil
80
+ end
81
+ end
82
+
83
+ def stop
84
+ @mutex.synchronize {
85
+ @cond.broadcast
86
+ }
87
+ end
88
+
89
+ def shutdown
90
+ @thread.join
91
+ end
92
+
93
+ def set(token)
94
+ @mutex.synchronize {
95
+ now = Time.now.to_i
96
+ @token = token
97
+ @heartbeat_time = now + @heartbeat_interval
98
+ @kill_time = now + @kill_timeout
99
+ @kill_proc = nil
100
+ @canceled = false
101
+ @cond.broadcast
102
+ }
103
+ end
104
+
105
+ def set_kill_proc(kill_proc)
106
+ @kill_proc = kill_proc
107
+ end
108
+
109
+ def reset(success)
110
+ @mutex.synchronize {
111
+ if success
112
+ @backend.finish(@token)
113
+ elsif @retry_wait && !@canceled
114
+ begin
115
+ @backend.update(@token, Time.now.to_i+@retry_wait)
116
+ rescue
117
+ # ignore CanceledError
118
+ end
119
+ end
120
+ @token = nil
121
+ }
122
+ end
123
+ end
124
+
125
+
126
+ class Worker
127
+ def initialize(engine, conf)
128
+ @engine = engine
129
+ @log = @engine.log
130
+
131
+ @run_class = conf[:run_class]
132
+ @monitor = MonitorThread.new(engine, conf)
133
+
134
+ @token = nil
135
+ @task = nil
136
+ @mutex = Mutex.new
137
+ @cond = ConditionVariable.new
138
+ end
139
+
140
+ def start
141
+ @thread = Thread.new(&method(:run))
142
+ @monitor.start
143
+ end
144
+
145
+ def run
146
+ while true
147
+ @mutex.synchronize {
148
+ while true
149
+ return if @engine.finished?
150
+ break if @token
151
+ @cond.wait(@mutex)
152
+ end
153
+ }
154
+ begin
155
+ process(@token, @task)
156
+ ensure
157
+ @token = nil
158
+ @engine.release_worker(self)
159
+ end
160
+ end
161
+ rescue
162
+ @engine.stop($!)
163
+ end
164
+
165
+ def process(token, task)
166
+ @log.info "processing task id=#{task.id}"
167
+
168
+ @monitor.set(token)
169
+ success = false
170
+ begin
171
+ run = @run_class.new(task)
172
+
173
+ if run.respond_to?(:kill)
174
+ @monitor.set_kill_proc run.method(:kill)
175
+ end
176
+
177
+ run.run
178
+
179
+ @log.info "finished id=#{task.id}"
180
+ success = true
181
+
182
+ rescue
183
+ @log.info "failed id=#{task.id}: #{$!}"
184
+
185
+ ensure
186
+ @monitor.reset(success)
187
+ end
188
+ end
189
+
190
+ def stop
191
+ submit(nil, nil)
192
+ @monitor.stop
193
+ end
194
+
195
+ def shutdown
196
+ @monitor.shutdown
197
+ @thread.join
198
+ end
199
+
200
+ def submit(token, task)
201
+ @mutex.synchronize {
202
+ @token = token
203
+ @task = task
204
+ @cond.broadcast
205
+ }
206
+ end
207
+ end
208
+
209
+
210
+ end
211
+
@@ -0,0 +1,217 @@
1
+ require File.dirname(__FILE__)+'/test_helper'
2
+
3
+ class BackendTest < Test::Unit::TestCase
4
+ TIMEOUT = 10
5
+ DB_PATH = File.dirname(__FILE__)+'/test.db'
6
+ DB_URI = "sqlite://#{DB_PATH}"
7
+
8
+ def clean_backend
9
+ @key_prefix = "test-#{"%08x"%rand(2**32)}-"
10
+ db = open_backend
11
+ db.list {|id,created_at,data,timeout|
12
+ db.cancel(id)
13
+ }
14
+ FileUtils.rm_f DB_PATH
15
+ end
16
+
17
+ def open_backend
18
+ #PerfectQueue::SimpleDBBackend.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], 'perfectqueue-test-1').use_consistent_read
19
+ PerfectQueue::RDBBackend.new(DB_URI, "perfectdb_test")
20
+ end
21
+
22
+ it 'acquire' do
23
+ clean_backend
24
+
25
+ db1 = open_backend
26
+ db2 = open_backend
27
+ db3 = open_backend
28
+
29
+ time = Time.now.to_i
30
+
31
+ ok = db1.submit(@key_prefix+'test1', 'data1', time)
32
+ assert_equal true, ok
33
+
34
+ token, task = db2.acquire(time+TIMEOUT)
35
+ assert_not_equal nil, task
36
+ assert_equal @key_prefix+'test1', task.id
37
+ assert_equal time, task.created_at
38
+ assert_equal 'data1', task.data
39
+
40
+ token_, task_ = db3.acquire(time+TIMEOUT)
41
+ assert_equal nil, token_
42
+ end
43
+
44
+ it 'finish' do
45
+ clean_backend
46
+
47
+ db1 = open_backend
48
+ db2 = open_backend
49
+
50
+ time = Time.now.to_i
51
+
52
+ ok = db1.submit(@key_prefix+'test1', 'data1', time)
53
+ assert_equal true, ok
54
+
55
+ token, task = db1.acquire(time+TIMEOUT)
56
+ assert_not_equal nil, task
57
+
58
+ ok = db1.finish(token)
59
+ assert_equal true, ok
60
+
61
+ token_, task_ = db2.acquire(time+TIMEOUT)
62
+ assert_equal nil, token_
63
+
64
+ ok = db1.finish(token)
65
+ assert_equal false, ok
66
+ end
67
+
68
+ it 'canceled' do
69
+ clean_backend
70
+
71
+ db1 = open_backend
72
+ db2 = open_backend
73
+
74
+ time = Time.now.to_i
75
+
76
+ ok = db1.submit(@key_prefix+'test1', 'data1', time)
77
+ assert_equal true, ok
78
+
79
+ token, task = db1.acquire(time+TIMEOUT)
80
+ assert_not_equal nil, task
81
+
82
+ ok = db1.cancel(task.id)
83
+ assert_equal true, ok
84
+
85
+ token_, task_ = db2.acquire(time+TIMEOUT)
86
+ assert_equal nil, token_
87
+
88
+ assert_raise(PerfectQueue::CanceledError) do
89
+ db1.update(token, time+TIMEOUT)
90
+ end
91
+
92
+ ok = db1.cancel(task.id)
93
+ assert_equal false, ok
94
+ end
95
+
96
+ it 'order' do
97
+ clean_backend
98
+
99
+ db1 = open_backend
100
+ db2 = open_backend
101
+ db3 = open_backend
102
+
103
+ time = Time.now.to_i
104
+
105
+ ok = db1.submit(@key_prefix+'test1', 'data1', time)
106
+ assert_equal true, ok
107
+
108
+ ok = db1.submit(@key_prefix+'test2', 'data2', time-1)
109
+ assert_equal true, ok
110
+
111
+ ok = db1.submit(@key_prefix+'test3', 'data3', time+1)
112
+ assert_equal true, ok
113
+
114
+ token, task = db2.acquire(time+TIMEOUT, time+1)
115
+ assert_not_equal nil, task
116
+ assert_equal @key_prefix+'test2', task.id
117
+ assert_equal time-1, task.created_at
118
+ assert_equal 'data2', task.data
119
+
120
+ token, task = db2.acquire(time+TIMEOUT, time+1)
121
+ assert_not_equal nil, task
122
+ assert_equal @key_prefix+'test1', task.id
123
+ assert_equal time, task.created_at
124
+ assert_equal 'data1', task.data
125
+
126
+ token, task = db2.acquire(time+TIMEOUT, time+1)
127
+ assert_not_equal nil, task
128
+ assert_equal @key_prefix+'test3', task.id
129
+ assert_equal time+1, task.created_at
130
+ assert_equal 'data3', task.data
131
+ end
132
+
133
+ it 'timeout' do
134
+ clean_backend
135
+
136
+ db1 = open_backend
137
+ db2 = open_backend
138
+
139
+ time = Time.now.to_i
140
+
141
+ ok = db1.submit(@key_prefix+'test1', 'data1', time)
142
+ assert_equal true, ok
143
+
144
+ token, task = db1.acquire(time+TIMEOUT)
145
+ assert_not_equal nil, task
146
+ assert_equal @key_prefix+'test1', task.id
147
+ assert_equal time, task.created_at
148
+ assert_equal 'data1', task.data
149
+
150
+ token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT)
151
+ assert_not_equal nil, task
152
+ assert_equal @key_prefix+'test1', task.id
153
+ assert_equal time, task.created_at
154
+ assert_equal 'data1', task.data
155
+ end
156
+
157
+ it 'extend' do
158
+ clean_backend
159
+
160
+ db1 = open_backend
161
+ db2 = open_backend
162
+
163
+ time = Time.now.to_i
164
+
165
+ ok = db1.submit(@key_prefix+'test1', 'data1', time)
166
+ assert_equal true, ok
167
+
168
+ token, task = db1.acquire(time+TIMEOUT)
169
+ assert_not_equal nil, task
170
+
171
+ assert_nothing_raised do
172
+ db1.update(token, time+TIMEOUT)
173
+ end
174
+
175
+ token_, task_ = db2.acquire(time+TIMEOUT, time)
176
+ assert_equal nil, token_
177
+
178
+ token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT)
179
+ assert_not_equal nil, task
180
+ assert_equal @key_prefix+'test1', task.id
181
+ assert_equal time, task.created_at
182
+ assert_equal 'data1', task.data
183
+ end
184
+
185
+ it 'release' do
186
+ clean_backend
187
+
188
+ db1 = open_backend
189
+ db2 = open_backend
190
+
191
+ time = Time.now.to_i
192
+
193
+ ok = db1.submit(@key_prefix+'test1', 'data1', time)
194
+ assert_equal true, ok
195
+
196
+ token, task = db1.acquire(time+TIMEOUT)
197
+ assert_not_equal nil, task
198
+
199
+ assert_nothing_raised do
200
+ db1.update(token, time+TIMEOUT)
201
+ end
202
+
203
+ token_, task_ = db2.acquire(time+TIMEOUT, time)
204
+ assert_equal nil, token_
205
+
206
+ assert_nothing_raised do
207
+ db1.update(token, time)
208
+ end
209
+
210
+ token, task = db2.acquire(time+TIMEOUT, time)
211
+ assert_not_equal nil, task
212
+ assert_equal @key_prefix+'test1', task.id
213
+ assert_equal time, task.created_at
214
+ assert_equal 'data1', task.data
215
+ end
216
+ end
217
+
@@ -0,0 +1,2 @@
1
+ #!/bin/sh
2
+ cat > "$1"
@@ -0,0 +1,4 @@
1
+ #!/bin/sh
2
+ out="$1"
3
+ shift
4
+ echo "$@" > "$out"
@@ -0,0 +1,61 @@
1
+ require File.dirname(__FILE__)+'/test_helper'
2
+
3
+ class ExecTest < Test::Unit::TestCase
4
+ it 'success' do
5
+ success_sh = File.expand_path File.dirname(__FILE__)+"/success.sh"
6
+
7
+ task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
8
+ e = PerfectQueue::ExecRunner.new(success_sh, task)
9
+
10
+ assert_nothing_raised do
11
+ e.run
12
+ end
13
+ end
14
+
15
+ it 'fail' do
16
+ fail_sh = File.expand_path File.dirname(__FILE__)+"/fail.sh"
17
+
18
+ task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
19
+ e = PerfectQueue::ExecRunner.new(fail_sh, task)
20
+
21
+ assert_raise(RuntimeError) do
22
+ e.run
23
+ end
24
+ end
25
+
26
+ it 'stdin' do
27
+ cat_sh = File.expand_path File.dirname(__FILE__)+"/cat.sh"
28
+ out_tmp = File.expand_path File.dirname(__FILE__)+"/cat.sh.tmp"
29
+
30
+ task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
31
+ e = PerfectQueue::ExecRunner.new("#{cat_sh} #{out_tmp}", task)
32
+
33
+ e.run
34
+
35
+ assert_equal 'data1', File.read(out_tmp)
36
+ end
37
+
38
+ it 'echo' do
39
+ echo_sh = File.expand_path File.dirname(__FILE__)+"/echo.sh"
40
+ out_tmp = File.expand_path File.dirname(__FILE__)+"/echo.sh.tmp"
41
+
42
+ task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
43
+ e = PerfectQueue::ExecRunner.new("#{echo_sh} #{out_tmp}", task)
44
+
45
+ e.run
46
+
47
+ assert_equal "test1\n", File.read(out_tmp)
48
+ end
49
+
50
+ it 'huge' do
51
+ huge_sh = File.expand_path File.dirname(__FILE__)+"/huge.sh"
52
+
53
+ task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
54
+ e = PerfectQueue::ExecRunner.new(huge_sh, task)
55
+
56
+ e.run
57
+
58
+ # should finish
59
+ end
60
+ end
61
+