simpleworker 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +1 -1
- data/.travis.yml +6 -0
- data/LICENSE +1 -1
- data/README.md +52 -15
- data/examples/basic.rb +26 -0
- data/examples/cucumber_worker.rb +20 -0
- data/examples/local_worker.rb +8 -0
- data/examples/ssh_worker.rb +13 -0
- data/examples/worker.rb +12 -0
- data/lib/simpleworker/abstract_listener.rb +41 -0
- data/lib/simpleworker/event_monitor.rb +50 -0
- data/lib/simpleworker/event_server.rb +48 -0
- data/lib/simpleworker/local_worker.rb +17 -10
- data/lib/simpleworker/logging_listener.rb +65 -0
- data/lib/simpleworker/redis_support.rb +33 -0
- data/lib/simpleworker/retry_listener.rb +32 -0
- data/lib/simpleworker/runner.rb +85 -26
- data/lib/simpleworker/scripts/expired_tasks.lua +13 -0
- data/lib/simpleworker/scripts/lpopall.lua +9 -0
- data/lib/simpleworker/scripts/reliable_queue.lua +12 -0
- data/lib/simpleworker/ssh_worker.rb +40 -19
- data/lib/simpleworker/task_queue.rb +78 -0
- data/lib/simpleworker/version.rb +1 -1
- data/lib/simpleworker.rb +12 -1
- data/simpleworker.gemspec +1 -1
- data/spec/simpleworker/event_monitor_spec.rb +92 -0
- data/spec/simpleworker/event_server_spec.rb +24 -0
- data/spec/simpleworker/local_worker_spec.rb +11 -18
- data/spec/simpleworker/logging_listener_spec.rb +44 -0
- data/spec/simpleworker/retry_listener_spec.rb +25 -0
- data/spec/simpleworker/runner_spec.rb +74 -12
- data/spec/simpleworker/ssh_worker_spec.rb +20 -31
- data/spec/simpleworker/task_queue_spec.rb +76 -0
- metadata +42 -5
- data/lib/simpleworker/abstract_worker.rb +0 -42
- data/lib/simpleworker/bash/simple-localworker +0 -8
- data/lib/simpleworker/bash/ssh-remoteworker +0 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 07c10b7615c0159ec82775ab8da412e1b0c2d2c1
|
4
|
+
data.tar.gz: 11684695817dc713a5a790ed82347563f46b577b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 929248bbeec93a43a6e6ea2242fcfda7bab2b655f153b8899e9d87a399ee0e1bd83736aff4555e20ff83641ec6251f3c4f694091a40d2a090c43389921338636
|
7
|
+
data.tar.gz: 42839d032356da7b14f841858b0b5b231d9532d006d3aa4a7165dcd9c520ae4d108a4e35b2e8e401301c3b95718afc1be75d334674998bf2afaa2ae7583157b9
|
data/.rspec
CHANGED
@@ -1 +1 @@
|
|
1
|
-
--color
|
1
|
+
--color
|
data/.travis.yml
ADDED
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -1,33 +1,71 @@
|
|
1
1
|
SimpleWorker
|
2
2
|
============
|
3
3
|
|
4
|
-
Distribute automation
|
4
|
+
Distribute automation tasks on multiple machines.
|
5
|
+
|
6
|
+
[![Gem Version](https://badge.fury.io/rb/simpleworker.svg)](http://badge.fury.io/rb/simpleworker)
|
7
|
+
[![Build Status](https://secure.travis-ci.org/jesg/simpleworker.png)](http://travis-ci.org/jesg/simpleworker)
|
5
8
|
|
6
9
|
Usage
|
7
10
|
=====
|
8
11
|
|
9
|
-
|
10
|
-
|
12
|
+
Ruby must be setup on the remote host such that it is available in the user's login shell.
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
require 'simpleworker'
|
16
|
+
|
17
|
+
tasks = ['first', 'second', 'third']
|
18
|
+
redis = Redis.new
|
19
|
+
|
20
|
+
# create a remote worker
|
21
|
+
ssh_worker = SimpleWorker::SshWorker.new(
|
22
|
+
:user => 'jesg'
|
23
|
+
:host => 'localhost',
|
24
|
+
:cmd => 'ruby -s worker.rb',
|
25
|
+
# rsync will wipe out existing files in ~/my_unused_dir
|
26
|
+
:dirname => 'my_unused_dir')
|
27
|
+
|
28
|
+
# create a local worker
|
29
|
+
local_worker = SimpleWorker::LocalWorker.new("ruby", "worker.rb")
|
30
|
+
|
31
|
+
runner = SimpleWorker::Runner.new(redis, tasks,
|
32
|
+
:namespace => 'foobar', # redis key prefix
|
33
|
+
:notify => [local_worker, ssh_worker], # listeners that implement AbstractListener
|
34
|
+
:max_retries => 1, # max times expired tasks will be retried
|
35
|
+
:timeout => 60, # timout in seconds if inactive
|
36
|
+
:task_timeout => 14, # max time for a task before it expires
|
37
|
+
:interval => 2) # interval at which to pull redis event log for the job in seconds
|
11
38
|
|
12
|
-
|
13
|
-
---
|
14
|
-
workers:
|
15
|
-
- type: ssh # type of worker
|
16
|
-
directory: /tmp/foobar # directory on remote host
|
17
|
-
user: bill # user on remote host
|
18
|
-
host: my.remote.host.com # remote host name
|
39
|
+
runner.run
|
19
40
|
```
|
20
41
|
|
42
|
+
Next create a script to work on the automation tasks.
|
43
|
+
|
21
44
|
```ruby
|
22
45
|
require 'simpleworker'
|
23
46
|
|
24
|
-
|
25
|
-
SimpleWorker::
|
47
|
+
redis = Redis.new
|
48
|
+
task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', ENV['JOBID'],
|
49
|
+
:namespace => 'foobar')
|
50
|
+
|
51
|
+
task_queue.fire_start
|
26
52
|
|
27
|
-
|
28
|
-
|
53
|
+
task_queue.each_task do |task|
|
54
|
+
task_queue.fire_log_message(task)
|
55
|
+
end
|
56
|
+
|
57
|
+
task_queue.fire_stop
|
29
58
|
```
|
30
59
|
|
60
|
+
Examples
|
61
|
+
========
|
62
|
+
|
63
|
+
Look in the examples directoy for some basic examples.
|
64
|
+
|
65
|
+
Requirements
|
66
|
+
===========
|
67
|
+
* redis
|
68
|
+
|
31
69
|
Note on Patches/Pull Requests
|
32
70
|
=============================
|
33
71
|
|
@@ -38,4 +76,3 @@ Note on Patches/Pull Requests
|
|
38
76
|
* Commit, do not mess with rakefile, version, or history.
|
39
77
|
(if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
40
78
|
* Send me a pull request. Bonus points for topic branches.
|
41
|
-
|
data/examples/basic.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'simpleworker'
|
2
|
+
|
3
|
+
tasks = ['first', 'second', 'third']
|
4
|
+
redis = Redis.new
|
5
|
+
runner = SimpleWorker::Runner.new(redis, tasks,
|
6
|
+
:max_retries => 2)
|
7
|
+
|
8
|
+
worker_thread = Thread.new do
|
9
|
+
task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', runner.jobid)
|
10
|
+
|
11
|
+
task_queue.fire_start
|
12
|
+
|
13
|
+
task_queue.each_task do |task|
|
14
|
+
if task == 'first'
|
15
|
+
sleep 15
|
16
|
+
elsif task == 'second'
|
17
|
+
task_queue.expire_current_task
|
18
|
+
else
|
19
|
+
task_queue.fire_log_message "Task: #{task}"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
task_queue.fire_stop
|
24
|
+
end
|
25
|
+
|
26
|
+
runner.run
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'simpleworker'
|
2
|
+
require 'cucumber/cli/main'
|
3
|
+
|
4
|
+
redis = Redis.new
|
5
|
+
task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', ENV['JOBID'])
|
6
|
+
|
7
|
+
task_queue.fire_start
|
8
|
+
|
9
|
+
task_queue.each_task do |task|
|
10
|
+
status = nil
|
11
|
+
begin
|
12
|
+
Cucumber::Cli::Main.execute task
|
13
|
+
rescue SystemExit => e
|
14
|
+
status = e.success?
|
15
|
+
end
|
16
|
+
|
17
|
+
fire_log_message "Cucumber task: #{task} status: #{status}"
|
18
|
+
end
|
19
|
+
|
20
|
+
task_queue.fire_stop
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'simpleworker'
|
2
|
+
|
3
|
+
tasks = ['first', 'second', 'third']
|
4
|
+
redis = Redis.new
|
5
|
+
ssh_worker = SimpleWorker::SshWorker.new(
|
6
|
+
:host => 'localhost',
|
7
|
+
:cmd => 'ruby -s worker.rb',
|
8
|
+
# rsync will wipe out existing files in ~/my_unused_dir
|
9
|
+
:dirname => 'my_unused_dir')
|
10
|
+
|
11
|
+
runner = SimpleWorker::Runner.new(redis, tasks,
|
12
|
+
:notify => [ssh_worker])
|
13
|
+
runner.run
|
data/examples/worker.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'simpleworker'
|
2
|
+
|
3
|
+
redis = Redis.new
|
4
|
+
task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', ENV['JOBID'])
|
5
|
+
|
6
|
+
task_queue.fire_start
|
7
|
+
|
8
|
+
task_queue.each_task do |task|
|
9
|
+
task_queue.fire_log_message "Do Task: #{task}"
|
10
|
+
end
|
11
|
+
|
12
|
+
task_queue.fire_stop
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module SimpleWorker
|
2
|
+
class AbstractListener
|
3
|
+
|
4
|
+
def on_start(jobid)
|
5
|
+
end
|
6
|
+
|
7
|
+
def on_stop
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_node_start(hostname)
|
11
|
+
end
|
12
|
+
|
13
|
+
def on_node_stop(hostname)
|
14
|
+
end
|
15
|
+
|
16
|
+
def on_task_start(hostname, task)
|
17
|
+
end
|
18
|
+
|
19
|
+
def on_task_active(hostname, task)
|
20
|
+
end
|
21
|
+
|
22
|
+
def on_task_stop(hostname, task)
|
23
|
+
end
|
24
|
+
|
25
|
+
def on_task_expire(hostname, task)
|
26
|
+
end
|
27
|
+
|
28
|
+
def on_log(hostname, msg)
|
29
|
+
end
|
30
|
+
|
31
|
+
def on_interrupted
|
32
|
+
end
|
33
|
+
|
34
|
+
def on_timeout
|
35
|
+
end
|
36
|
+
|
37
|
+
def update(meth, *args)
|
38
|
+
__send__(meth, *args)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module SimpleWorker
|
2
|
+
class EventMonitor < AbstractListener
|
3
|
+
|
4
|
+
attr_reader :start_time, :latest_time, :expired_tasks
|
5
|
+
|
6
|
+
def initialize(start_time = Time.now)
|
7
|
+
@start_time = start_time
|
8
|
+
@latest_time = start_time
|
9
|
+
@expired_tasks = []
|
10
|
+
@event_tracker = {}
|
11
|
+
end
|
12
|
+
|
13
|
+
def on_start(jobid)
|
14
|
+
@jobid = jobid
|
15
|
+
end
|
16
|
+
|
17
|
+
def on_node_start(hostname)
|
18
|
+
@event_tracker[hostname] = latest_time
|
19
|
+
end
|
20
|
+
|
21
|
+
def on_node_stop(hostname)
|
22
|
+
@event_tracker.delete(hostname)
|
23
|
+
end
|
24
|
+
|
25
|
+
def on_task_start(hostname, task)
|
26
|
+
@event_tracker[hostname] = latest_time
|
27
|
+
@event_tracker[task] = latest_time
|
28
|
+
end
|
29
|
+
|
30
|
+
def on_task_stop(hostname, task)
|
31
|
+
@event_tracker[hostname] = latest_time
|
32
|
+
@event_tracker.delete(task)
|
33
|
+
end
|
34
|
+
|
35
|
+
def on_task_expire(hostname, task)
|
36
|
+
@expired_tasks << {:task => task, :hostname => hostname, :time => latest_time}
|
37
|
+
@event_tracker.delete(task)
|
38
|
+
end
|
39
|
+
|
40
|
+
def done?(remaining)
|
41
|
+
(remaining == 0) && @event_tracker.empty?
|
42
|
+
end
|
43
|
+
|
44
|
+
def update(meth, *args)
|
45
|
+
@latest_time = Time.now
|
46
|
+
super(meth, *args)
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module SimpleWorker
|
2
|
+
class EventServer
|
3
|
+
include Observable
|
4
|
+
include RedisSupport
|
5
|
+
|
6
|
+
def initialize(redis, namespace, jobid)
|
7
|
+
@redis = redis
|
8
|
+
@namespace = namespace
|
9
|
+
@jobid = jobid
|
10
|
+
load_lua_scripts
|
11
|
+
end
|
12
|
+
|
13
|
+
def pull_events
|
14
|
+
log, processing, remaining = @redis.multi do
|
15
|
+
@redis.evalsha @lpopall_sha, [log_key]
|
16
|
+
@redis.evalsha @expired_tasks_sha, [active_tasks_key]
|
17
|
+
@redis.llen tasks_key
|
18
|
+
end
|
19
|
+
|
20
|
+
log.map { |str| JSON.parse(str) }.each do |event|
|
21
|
+
fire(*event)
|
22
|
+
end
|
23
|
+
|
24
|
+
processing[0].each do |key|
|
25
|
+
hostname, task = parse_active_task_key(key)
|
26
|
+
fire('on_task_expire', hostname, task)
|
27
|
+
end
|
28
|
+
|
29
|
+
processing[1].each do |key|
|
30
|
+
hostname, task = parse_active_task_key(key)
|
31
|
+
fire('on_task_active', hostname, task)
|
32
|
+
end
|
33
|
+
|
34
|
+
remaining + processing[0].size
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def parse_active_task_key(str)
|
40
|
+
str.split(':').slice(3, 4)
|
41
|
+
end
|
42
|
+
|
43
|
+
def fire(*args)
|
44
|
+
changed
|
45
|
+
notify_observers *args
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -1,18 +1,25 @@
|
|
1
1
|
|
2
2
|
module SimpleWorker
|
3
|
-
class LocalWorker
|
4
|
-
include AbstractWorker
|
3
|
+
class LocalWorker < AbstractListener
|
5
4
|
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
DEFAULT_CMD = ['/bin/bash', '-l', 'bundle', 'exec', 'rake']
|
6
|
+
|
7
|
+
def initialize(*args)
|
8
|
+
cmd = args
|
9
|
+
cmd = DEFAULT_CMD if cmd.empty?
|
10
|
+
@process = ChildProcess.build(*cmd)
|
11
|
+
@process.io.inherit!
|
12
|
+
end
|
13
|
+
|
14
|
+
# Start a subprocess with the environment variable 'JOBID' set to the current jobid.
|
15
|
+
#
|
16
|
+
def on_start(jobid)
|
17
|
+
@process.environment['JOBID'] = jobid
|
18
|
+
@process.start
|
9
19
|
end
|
10
20
|
|
11
|
-
def
|
12
|
-
|
13
|
-
worker.script = config['script'] if config.has_key? 'script'
|
14
|
-
worker.cmd = config['cmd'] if config.has_key? 'cmd'
|
15
|
-
worker
|
21
|
+
def on_stop
|
22
|
+
@process.stop
|
16
23
|
end
|
17
24
|
end
|
18
25
|
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
|
2
|
+
module SimpleWorker
|
3
|
+
class LoggingListener < AbstractListener
|
4
|
+
|
5
|
+
TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
|
6
|
+
|
7
|
+
def initialize(io = STDOUT)
|
8
|
+
@io = io
|
9
|
+
end
|
10
|
+
|
11
|
+
def on_start(jobid)
|
12
|
+
log.info "start: #{jobid}"
|
13
|
+
end
|
14
|
+
|
15
|
+
def on_stop
|
16
|
+
log.info "stop"
|
17
|
+
end
|
18
|
+
|
19
|
+
def on_node_start(hostname)
|
20
|
+
log.info "start node: #{hostname}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def on_node_stop(hostname)
|
24
|
+
log.info "stop node: #{hostname}"
|
25
|
+
end
|
26
|
+
|
27
|
+
def on_task_start(hostname, task)
|
28
|
+
log.info "start host: #{hostname} task: #{task}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def on_task_active(hostname, task)
|
32
|
+
log.info "active host: #{hostname} task: #{task}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def on_task_stop(hostname, task)
|
36
|
+
log.info "stop host: #{hostname} task: #{task}"
|
37
|
+
end
|
38
|
+
|
39
|
+
def on_task_expire(hostname, task)
|
40
|
+
log.info "expire host: #{hostname} task: #{task}"
|
41
|
+
end
|
42
|
+
|
43
|
+
def on_log(hostname, msg)
|
44
|
+
log.info "host: #{hostname} #{msg}"
|
45
|
+
end
|
46
|
+
|
47
|
+
def on_interrupted
|
48
|
+
log.info "interrupted"
|
49
|
+
end
|
50
|
+
|
51
|
+
def on_timeout
|
52
|
+
log.info "timeout"
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def log
|
58
|
+
@log ||= (
|
59
|
+
log = ::Logger.new @io
|
60
|
+
log.datetime_format = TIME_FORMAT
|
61
|
+
log
|
62
|
+
)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
|
2
|
+
module SimpleWorker
|
3
|
+
module RedisSupport
|
4
|
+
|
5
|
+
attr_reader :namespace, :jobid
|
6
|
+
|
7
|
+
private
|
8
|
+
|
9
|
+
def tasks_key
|
10
|
+
@tasks_key ||= "#{namespace}:tasks:#{jobid}"
|
11
|
+
end
|
12
|
+
|
13
|
+
def log_key
|
14
|
+
@log_key ||= "#{namespace}:log:#{jobid}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def active_tasks_key
|
18
|
+
@active_tasks_key ||= "#{namespace}:active:#{jobid}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def config_key
|
22
|
+
@config_key ||= "#{namespace}:config:#{jobid}"
|
23
|
+
end
|
24
|
+
|
25
|
+
def load_lua_scripts
|
26
|
+
path_to_lua_scripts = File.expand_path("scripts/", File.dirname(__FILE__))
|
27
|
+
['lpopall', 'expired_tasks', 'reliable_queue'].each do |name|
|
28
|
+
sha = @redis.script(:load, IO.read("#{path_to_lua_scripts}/#{name}.lua"))
|
29
|
+
instance_variable_set("@#{name}_sha", sha)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
|
2
|
+
module SimpleWorker
|
3
|
+
class RetryListener < AbstractListener
|
4
|
+
include RedisSupport
|
5
|
+
|
6
|
+
attr_reader :max_retries
|
7
|
+
|
8
|
+
def initialize(redis, max_retries, namespace, jobid)
|
9
|
+
@redis = redis
|
10
|
+
@max_retries = max_retries
|
11
|
+
@namespace = namespace
|
12
|
+
@jobid = jobid
|
13
|
+
@tracker = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def on_task_expire(hostname, task)
|
17
|
+
# warning nil converted to 0
|
18
|
+
count = @tracker[task].to_i
|
19
|
+
|
20
|
+
if count < max_retries
|
21
|
+
fire_retry task
|
22
|
+
@tracker[task] = (count + 1)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def fire_retry(task)
|
29
|
+
@redis.rpush(tasks_key, task)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/lib/simpleworker/runner.rb
CHANGED
@@ -1,43 +1,76 @@
|
|
1
1
|
|
2
2
|
module SimpleWorker
|
3
|
+
|
4
|
+
#
|
5
|
+
# Runner.run(redis, tasks, opts)
|
6
|
+
#
|
7
|
+
# where tasks is an Array of strings
|
8
|
+
# and 'opts' is a Hash of options:
|
9
|
+
#
|
10
|
+
# :namespace => String prefix to keys in redis used by SimpleWorker (default: simpleworker)
|
11
|
+
# :timeout => Fixnum max allowed time between events (default: 30 seconds)
|
12
|
+
# :task_timeout => Fixnum max time allowed for a task to take (default: 10 seconds)
|
13
|
+
# :interval => Fixnum interval at which SimpleWorker checks the status of all tasks (default: 5 seconds)
|
14
|
+
# :notify => Array[AbstractListener] objects implementing the AbstractListener API
|
15
|
+
# :max_retries => Fixnum number of times expired tasks will be retried (default: 0)
|
3
16
|
class Runner
|
17
|
+
include RedisSupport
|
18
|
+
include Observable
|
4
19
|
|
5
|
-
|
6
|
-
|
7
|
-
|
20
|
+
DEFAULT_OPTIONS = {
|
21
|
+
:timeout => 30,
|
22
|
+
:task_timeout => 10,
|
23
|
+
:interval => 5,
|
24
|
+
:namespace => 'simpleworker',
|
25
|
+
:log => true,
|
26
|
+
:max_retries => 0}
|
8
27
|
|
9
|
-
def
|
10
|
-
|
11
|
-
end
|
28
|
+
def initialize(redis, tasks, opts = {})
|
29
|
+
opts = DEFAULT_OPTIONS.dup.merge(opts)
|
12
30
|
|
13
|
-
|
14
|
-
|
15
|
-
|
31
|
+
@redis = redis
|
32
|
+
@jobid = SecureRandom.hex(6)
|
33
|
+
@namespace = opts[:namespace]
|
34
|
+
@timeout = opts[:timeout]
|
35
|
+
@interval = opts[:interval]
|
36
|
+
max_retries = opts[:max_retries]
|
37
|
+
listeners = Array(opts[:notify])
|
16
38
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
@workers << LocalWorker.create(config)
|
25
|
-
end
|
39
|
+
STDERR.puts 'WARNING: to prevent a race condition :timeout should be > :task_timeout' if @timeout < opts[:task_timeout]
|
40
|
+
load_lua_scripts
|
41
|
+
@redis.set(config_key, {'task_timeout' => opts[:task_timeout]}.to_json)
|
42
|
+
@redis.rpush(tasks_key, tasks)
|
43
|
+
|
44
|
+
if opts[:log]
|
45
|
+
listeners << LoggingListener.new
|
26
46
|
end
|
27
47
|
|
28
|
-
|
29
|
-
|
48
|
+
@event_server = EventServer.new(redis, namespace, jobid)
|
49
|
+
@event_monitor = EventMonitor.new
|
50
|
+
listeners << @event_monitor
|
51
|
+
|
52
|
+
@retry_listener = RetryListener.new(redis, max_retries, namespace, jobid)
|
53
|
+
listeners << @retry_listener
|
30
54
|
|
31
|
-
|
32
|
-
|
33
|
-
|
55
|
+
listeners.each do |listener|
|
56
|
+
add_observer listener
|
57
|
+
@event_server.add_observer listener
|
34
58
|
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.run(redis, tasks, opts = {})
|
62
|
+
new(redis, tasks, opts).run
|
63
|
+
end
|
64
|
+
|
65
|
+
def run
|
35
66
|
start
|
36
67
|
process
|
37
68
|
stop
|
38
69
|
rescue Interrupt
|
70
|
+
fire 'on_interrupted'
|
39
71
|
stop
|
40
72
|
rescue StandardError => e
|
73
|
+
fire 'on_interrupted'
|
41
74
|
stop
|
42
75
|
raise e
|
43
76
|
end
|
@@ -45,15 +78,41 @@ module SimpleWorker
|
|
45
78
|
private
|
46
79
|
|
47
80
|
def start
|
48
|
-
@
|
81
|
+
fire('on_start', @jobid)
|
49
82
|
end
|
50
83
|
|
51
84
|
def process
|
52
|
-
@
|
85
|
+
remaining_tasks = @event_server.pull_events
|
86
|
+
|
87
|
+
until @event_monitor.done? remaining_tasks
|
88
|
+
sleep @interval
|
89
|
+
|
90
|
+
remaining_tasks = @event_server.pull_events
|
91
|
+
|
92
|
+
current_time = Time.now
|
93
|
+
if (current_time - @event_monitor.latest_time) > @timeout
|
94
|
+
fire 'on_timeout'
|
95
|
+
break
|
96
|
+
end
|
97
|
+
end
|
53
98
|
end
|
54
99
|
|
55
100
|
def stop
|
56
|
-
|
101
|
+
fire 'on_stop'
|
102
|
+
|
103
|
+
@redis.multi do
|
104
|
+
@redis.del tasks_key
|
105
|
+
@redis.del active_tasks_key
|
106
|
+
@redis.del log_key
|
107
|
+
@redis.del config_key
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
private
|
112
|
+
|
113
|
+
def fire(*args)
|
114
|
+
changed
|
115
|
+
notify_observers *args
|
57
116
|
end
|
58
117
|
end
|
59
118
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
local result = {{},{}}
|
2
|
+
local active = redis.pcall('SMEMBERS', KEYS[1])
|
3
|
+
|
4
|
+
for _, v in pairs(active) do
|
5
|
+
local exists = tonumber(redis.pcall('EXISTS', v))
|
6
|
+
if exists == 0 then
|
7
|
+
table.insert(result[1], v)
|
8
|
+
redis.pcall('SREM', KEYS[1], v)
|
9
|
+
else
|
10
|
+
table.insert(result[2], v)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
return result
|