aeden-refinery 0.9.12 → 0.9.13
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/refinery/daemon.rb +34 -34
- data/lib/refinery/event_publisher.rb +7 -7
- data/lib/refinery/heartbeat.rb +2 -5
- data/lib/refinery/monitor.rb +7 -13
- data/lib/refinery/publisher.rb +10 -7
- data/lib/refinery/queueable.rb +12 -0
- data/lib/refinery/server.rb +2 -10
- data/refinery.gemspec +2 -2
- data/test/unit/daemon_test.rb +18 -4
- metadata +2 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.9.
|
1
|
+
0.9.13
|
data/lib/refinery/daemon.rb
CHANGED
@@ -4,6 +4,7 @@ module Refinery #:nodoc:
|
|
4
4
|
include Refinery::Loggable
|
5
5
|
include Refinery::Configurable
|
6
6
|
include Refinery::Utilities
|
7
|
+
include Refinery::Queueable
|
7
8
|
|
8
9
|
RUNNING = 'running'
|
9
10
|
STOPPED = 'stopped'
|
@@ -12,12 +13,10 @@ module Refinery #:nodoc:
|
|
12
13
|
attr_reader :thread
|
13
14
|
# The name of the daemon
|
14
15
|
attr_reader :name
|
15
|
-
# The
|
16
|
-
attr_reader :
|
17
|
-
# The queue
|
18
|
-
attr_reader :
|
19
|
-
# The queue for error messages
|
20
|
-
attr_reader :error_queue
|
16
|
+
# The settings for the daemon
|
17
|
+
attr_reader :settings
|
18
|
+
# The base queue name
|
19
|
+
attr_reader :queue_name
|
21
20
|
|
22
21
|
# Stop the daemon
|
23
22
|
def stop
|
@@ -52,56 +51,57 @@ module Refinery #:nodoc:
|
|
52
51
|
# The settings hash may contain the following options:
|
53
52
|
# * <tt>visibility</tt>: The time in seconds that the message is hidden
|
54
53
|
# in the queue.
|
55
|
-
def initialize(server, name,
|
54
|
+
def initialize(server, name, queue_prefix='', settings={})
|
56
55
|
Refinery::Server.logger.debug "Starting daemon"
|
57
56
|
|
58
57
|
@server = server
|
59
58
|
@name = name
|
60
|
-
@
|
61
|
-
|
62
|
-
|
59
|
+
@settings = settings
|
60
|
+
|
61
|
+
queue_name = settings['queue'] || name
|
62
|
+
queue_name = "#{queue_prefix}#{queue_name}"
|
63
|
+
logger.debug "Using queue #{queue_name}"
|
64
|
+
@queue_name = queue_name
|
63
65
|
|
64
66
|
@thread = Thread.new(self) do |daemon|
|
65
67
|
logger.debug "Running daemon thread: #{name} (settings: #{settings.inspect})"
|
66
68
|
while(running?)
|
67
|
-
|
69
|
+
with_queue("#{queue_name}_waiting") do |waiting_queue|
|
68
70
|
while (message = waiting_queue.receive(settings['visibility']))
|
69
71
|
worker = load_worker_class(name).new(self)
|
70
72
|
begin
|
71
73
|
result, run_time = worker.run(decode_message(message.body))
|
72
74
|
if result
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
75
|
+
with_queue("#{queue_name}_done") do |done_queue|
|
76
|
+
done_message = {
|
77
|
+
'host_info' => host_info,
|
78
|
+
'original' => message.body,
|
79
|
+
'run_time' => run_time
|
80
|
+
}
|
81
|
+
logger.debug "Sending 'done' message to #{done_queue.name}"
|
82
|
+
done_queue.send_message(encode_message(done_message))
|
83
|
+
end
|
80
84
|
|
81
85
|
logger.debug "Deleting message from queue"
|
82
86
|
message.delete()
|
83
87
|
end
|
84
88
|
rescue Exception => e
|
85
|
-
|
86
|
-
|
87
|
-
'
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
89
|
+
with_queue("#{queue_name}_error") do |error_queue|
|
90
|
+
error_message = {
|
91
|
+
'error' => {
|
92
|
+
'message' => e.message,
|
93
|
+
'class' => e.class.name
|
94
|
+
},
|
95
|
+
'host_info' => host_info,
|
96
|
+
'original' => message.body
|
97
|
+
}
|
98
|
+
logger.error "Sending 'error' message to #{error_queue.name}: #{e.message}"
|
99
|
+
error_queue.send_message(encode_message(error_message))
|
100
|
+
end
|
95
101
|
message.delete()
|
96
102
|
end
|
97
103
|
end
|
98
104
|
sleep(settings['sleep'] || 5)
|
99
|
-
rescue Exception => e
|
100
|
-
logger.error "An error occurred while receiving from the waiting queue: #{e.message}"
|
101
|
-
# delay to try to get past the issue with the queue
|
102
|
-
sleep(30)
|
103
|
-
# assign a new queue instance
|
104
|
-
@waiting_queue = queue(waiting_queue.name)
|
105
105
|
end
|
106
106
|
end
|
107
107
|
logger.debug "Exiting daemon thread"
|
@@ -51,9 +51,9 @@ module Refinery #:nodoc:
|
|
51
51
|
raise RuntimeError, "No processor configuration found for #{key}" unless settings
|
52
52
|
queue_name = settings['queue'] || key
|
53
53
|
queue_name = "#{prefix}#{queue_name}"
|
54
|
-
|
55
|
-
|
56
|
-
load_publisher_class(key).new(
|
54
|
+
waiting_queue_name = "#{queue_name}_waiting"
|
55
|
+
logger.debug "Using queue #{waiting_queue_name}"
|
56
|
+
load_publisher_class(key).new(waiting_queue_name).execute
|
57
57
|
end
|
58
58
|
|
59
59
|
# Run the event publisher
|
@@ -85,13 +85,13 @@ module Refinery #:nodoc:
|
|
85
85
|
logger.info "Creating publisher for #{key}"
|
86
86
|
queue_name = settings['queue'] || key
|
87
87
|
queue_name = "#{prefix}#{queue_name}"
|
88
|
-
|
89
|
-
|
88
|
+
waiting_queue_name = "#{queue_name}_waiting"
|
89
|
+
logger.debug "Using queue #{waiting_queue_name}"
|
90
90
|
|
91
|
-
threads << Thread.new(
|
91
|
+
threads << Thread.new(waiting_queue_name, settings) do |waiting_queue_name, settings|
|
92
92
|
while(running?)
|
93
93
|
begin
|
94
|
-
load_publisher_class(key).new(
|
94
|
+
load_publisher_class(key).new(waiting_queue_name).execute
|
95
95
|
rescue Exception => e
|
96
96
|
logger.error e
|
97
97
|
raise e
|
data/lib/refinery/heartbeat.rb
CHANGED
@@ -9,9 +9,9 @@ module Refinery #:nodoc:
|
|
9
9
|
# Initialize the heartbeat for the given server.
|
10
10
|
def initialize(server)
|
11
11
|
@server = server
|
12
|
-
@thread = Thread.new
|
12
|
+
@thread = Thread.new do
|
13
13
|
loop do
|
14
|
-
|
14
|
+
with_queue('heartbeat') do |heartbeat_queue|
|
15
15
|
logger.debug "Send heartbeat"
|
16
16
|
message = {
|
17
17
|
'host_info' => host_info,
|
@@ -20,9 +20,6 @@ module Refinery #:nodoc:
|
|
20
20
|
}
|
21
21
|
heartbeat_queue.send_message(Base64.encode64(message.to_json))
|
22
22
|
sleep(60)
|
23
|
-
rescue Exception => e
|
24
|
-
logger.error "Error sending heartbeat: #{e.message}"
|
25
|
-
sleep(30)
|
26
23
|
end
|
27
24
|
end
|
28
25
|
end
|
data/lib/refinery/monitor.rb
CHANGED
@@ -49,15 +49,13 @@ module Refinery #:nodoc:
|
|
49
49
|
|
50
50
|
def run_heartbeat_monitor
|
51
51
|
logger.info "Starting heartbeat monitor"
|
52
|
-
Thread.new(
|
52
|
+
Thread.new('heartbeat') do |heartbeat_queue_name|
|
53
53
|
loop do
|
54
|
-
|
54
|
+
with_queue(heartbeat_queue_name) do |heartbeat_queue|
|
55
55
|
while (message = heartbeat_queue.receive)
|
56
56
|
logger.debug decode_message(message.body).inspect
|
57
57
|
message.delete()
|
58
58
|
end
|
59
|
-
rescue Exception => e
|
60
|
-
logger.error e
|
61
59
|
end
|
62
60
|
sleep(5)
|
63
61
|
end
|
@@ -69,9 +67,9 @@ module Refinery #:nodoc:
|
|
69
67
|
queue_name = settings['queue'] || key
|
70
68
|
done_queue_name = "#{queue_name}_done"
|
71
69
|
logger.debug "Starting monitor for queue #{done_queue_name}"
|
72
|
-
Thread.new(
|
70
|
+
Thread.new(done_queue_name) do |done_queue_name|
|
73
71
|
loop do
|
74
|
-
|
72
|
+
with_queue(done_queue_name) do |done_queue|
|
75
73
|
while (message = done_queue.receive)
|
76
74
|
done_message = decode_message(message.body)
|
77
75
|
processed = decode_message(done_message['original'])
|
@@ -79,10 +77,8 @@ module Refinery #:nodoc:
|
|
79
77
|
message.delete()
|
80
78
|
statistics.record_done(done_message)
|
81
79
|
end
|
82
|
-
|
83
|
-
logger.error e
|
80
|
+
sleep(5)
|
84
81
|
end
|
85
|
-
sleep(5)
|
86
82
|
end
|
87
83
|
end
|
88
84
|
end
|
@@ -93,9 +89,9 @@ module Refinery #:nodoc:
|
|
93
89
|
queue_name = settings['queue'] || key
|
94
90
|
error_queue_name = "#{queue_name}_error"
|
95
91
|
logger.info "Starting error monitor for queue #{error_queue_name}"
|
96
|
-
Thread.new(
|
92
|
+
Thread.new(error_queue_name) do |error_queue_name|
|
97
93
|
loop do
|
98
|
-
|
94
|
+
with(error_queue_name) do |error_queue|
|
99
95
|
while (message = error_queue.receive)
|
100
96
|
error_message = decode_message(message.body)
|
101
97
|
processed = decode_message(error_message['original'])
|
@@ -103,8 +99,6 @@ module Refinery #:nodoc:
|
|
103
99
|
message.delete()
|
104
100
|
statistics.record_error(error_message)
|
105
101
|
end
|
106
|
-
rescue Exception => e
|
107
|
-
logger.error e
|
108
102
|
end
|
109
103
|
sleep(5)
|
110
104
|
end
|
data/lib/refinery/publisher.rb
CHANGED
@@ -2,23 +2,26 @@ module Refinery #:nodoc:
|
|
2
2
|
# Base class for publishers to be implemented by subclasses.
|
3
3
|
class Publisher
|
4
4
|
include Refinery::Loggable
|
5
|
+
include Refinery::Queueable
|
5
6
|
|
6
7
|
# Initialize the publisher with the queue to publish messages to.
|
7
|
-
def initialize(
|
8
|
-
@
|
8
|
+
def initialize(waiting_queue_name)
|
9
|
+
@waiting_queue_name = waiting_queue_name
|
9
10
|
end
|
10
11
|
|
11
12
|
protected
|
12
|
-
# Get the publish queue
|
13
|
-
def
|
14
|
-
@
|
13
|
+
# Get the publish queue name
|
14
|
+
def waiting_queue_name
|
15
|
+
@waiting_queue_name
|
15
16
|
end
|
16
17
|
|
17
18
|
# Publish the message. The message will be converted to JSON and pushed
|
18
19
|
# into the queue associated with the publisher.
|
19
20
|
def publish(message)
|
20
|
-
|
21
|
-
|
21
|
+
with_queue(waiting_queue_name) do |waiting_queue|
|
22
|
+
logger.debug "Publisher #{self.class.name} sending message: #{message.to_json}"
|
23
|
+
waiting_queue.send_message(Base64.encode64(message.to_json))
|
24
|
+
end
|
22
25
|
end
|
23
26
|
end
|
24
27
|
end
|
data/lib/refinery/queueable.rb
CHANGED
@@ -1,11 +1,23 @@
|
|
1
1
|
module Refinery #:nodoc:
|
2
2
|
# Mix this module in to classes that want to access a queue.
|
3
3
|
module Queueable
|
4
|
+
include Loggable
|
5
|
+
include Configurable
|
4
6
|
# Get a named queue
|
5
7
|
def queue(name)
|
6
8
|
queue_provider.queue(name)
|
7
9
|
end
|
8
10
|
|
11
|
+
def with_queue(name, &block)
|
12
|
+
begin
|
13
|
+
yield queue(name)
|
14
|
+
rescue Exception => e
|
15
|
+
logger.error "Queue error: #{e.message}"
|
16
|
+
sleep(5)
|
17
|
+
retry
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
9
21
|
protected
|
10
22
|
# Get the queue provider. Defaults to RightAws::SqsGen2 running
|
11
23
|
# in multi-thread mode.
|
data/lib/refinery/server.rb
CHANGED
@@ -60,19 +60,11 @@ module Refinery #:nodoc:
|
|
60
60
|
|
61
61
|
private
|
62
62
|
def execute_daemons
|
63
|
-
|
63
|
+
queue_prefix = config['prefix'] || ''
|
64
64
|
config['processors'].each do |key, settings|
|
65
65
|
logger.debug "Creating daemons for #{key}"
|
66
|
-
|
67
|
-
queue_name = settings['queue'] || key
|
68
|
-
queue_name = "#{prefix}#{queue_name}"
|
69
|
-
logger.debug "Using queue #{queue_name}"
|
70
|
-
waiting_queue = queue("#{queue_name}_waiting")
|
71
|
-
error_queue = queue("#{queue_name}_error")
|
72
|
-
done_queue = queue("#{queue_name}_done")
|
73
|
-
|
74
66
|
1.upto(settings['workers']['initial']) do
|
75
|
-
daemons << Refinery::Daemon.new(self, key,
|
67
|
+
daemons << Refinery::Daemon.new(self, key, queue_prefix, settings)
|
76
68
|
end
|
77
69
|
|
78
70
|
logger.debug "Running #{daemons.length} daemons"
|
data/refinery.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{refinery}
|
5
|
-
s.version = "0.9.
|
5
|
+
s.version = "0.9.13"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Anthony Eden"]
|
9
|
-
s.date = %q{2009-07-
|
9
|
+
s.date = %q{2009-07-09}
|
10
10
|
s.description = %q{Process data in a distributed fashion.}
|
11
11
|
s.email = %q{anthonyeden@gmail.com}
|
12
12
|
s.executables = ["epub", "monitor", "pubnow", "refinery"]
|
data/test/unit/daemon_test.rb
CHANGED
@@ -6,26 +6,40 @@ class DaemonTest < Test::Unit::TestCase
|
|
6
6
|
@waiting_queue = stub('Queue(waiting)')
|
7
7
|
@error_queue = stub('Queue(error)')
|
8
8
|
@done_queue = stub('Queue(done)')
|
9
|
+
|
10
|
+
Refinery::Daemon.any_instance.stubs(:queue).with(
|
11
|
+
'sample_waiting').returns(@waiting_queue)
|
12
|
+
Refinery::Daemon.any_instance.stubs(:queue).with(
|
13
|
+
'sample_error').returns(@error_queue)
|
14
|
+
Refinery::Daemon.any_instance.stubs(:queue).with(
|
15
|
+
'sample_done').returns(@done_queue)
|
9
16
|
end
|
10
17
|
should "be startable" do
|
11
18
|
@waiting_queue.stubs(:receive)
|
12
19
|
assert_nothing_raised do
|
13
|
-
daemon = Refinery::Daemon.new(@server, 'sample'
|
20
|
+
daemon = Refinery::Daemon.new(@server, 'sample')
|
14
21
|
end
|
15
22
|
end
|
16
23
|
should "have logging" do
|
17
24
|
@waiting_queue.stubs(:receive)
|
18
|
-
daemon = Refinery::Daemon.new(@server, 'sample'
|
25
|
+
daemon = Refinery::Daemon.new(@server, 'sample')
|
19
26
|
assert_not_nil daemon.logger
|
20
27
|
end
|
21
28
|
should "allow visibility setting" do
|
22
29
|
@waiting_queue.expects(:receive).with(600)
|
23
|
-
daemon = Refinery::Daemon.new(@server, 'sample',
|
30
|
+
daemon = Refinery::Daemon.new(@server, 'sample', '', {'visibility' => 600})
|
31
|
+
end
|
32
|
+
should "have a queue name" do
|
33
|
+
@waiting_queue.stubs(:receive)
|
34
|
+
Refinery::Daemon.any_instance.stubs(:queue).with(
|
35
|
+
'prefix_sample_waiting').returns(@waiting_queue)
|
36
|
+
daemon = Refinery::Daemon.new(@server, 'sample', 'prefix_')
|
37
|
+
assert_equal 'prefix_sample', daemon.queue_name
|
24
38
|
end
|
25
39
|
context "that is started" do
|
26
40
|
setup do
|
27
41
|
@waiting_queue.stubs(:receive)
|
28
|
-
@daemon = Refinery::Daemon.new(@server, 'sample'
|
42
|
+
@daemon = Refinery::Daemon.new(@server, 'sample')
|
29
43
|
end
|
30
44
|
should "have a state of running" do
|
31
45
|
assert @daemon.running?
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aeden-refinery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Anthony Eden
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-07-
|
12
|
+
date: 2009-07-09 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|