aeden-refinery 0.9.15 → 0.10.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/refinery/daemon.rb +50 -42
- data/lib/refinery/processor.rb +49 -0
- data/lib/refinery/publisher.rb +17 -2
- data/lib/refinery/queueable.rb +2 -17
- data/lib/refinery/server.rb +6 -11
- data/lib/refinery.rb +3 -0
- data/publishers/error.rb +1 -3
- data/publishers/sample.rb +1 -3
- data/publishers/sleep.rb +1 -3
- data/refinery.gemspec +5 -2
- data/test/unit/daemon_test.rb +16 -14
- data/test/unit/processor_test.rb +30 -0
- data/test/unit/server_test.rb +1 -7
- metadata +5 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.10.0
|
data/lib/refinery/daemon.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Refinery #:nodoc:
|
2
2
|
# A daemon provides a thread to run workers in.
|
3
|
-
class Daemon
|
3
|
+
class Daemon < Thread
|
4
4
|
include Refinery::Loggable
|
5
5
|
include Refinery::Configurable
|
6
6
|
include Refinery::Utilities
|
@@ -9,8 +9,6 @@ module Refinery #:nodoc:
|
|
9
9
|
RUNNING = 'running'
|
10
10
|
STOPPED = 'stopped'
|
11
11
|
|
12
|
-
# The daemon's thread
|
13
|
-
attr_reader :thread
|
14
12
|
# The name of the daemon
|
15
13
|
attr_reader :name
|
16
14
|
# The settings for the daemon
|
@@ -41,7 +39,7 @@ module Refinery #:nodoc:
|
|
41
39
|
|
42
40
|
# Initialize the daemon.
|
43
41
|
#
|
44
|
-
# * <tt>
|
42
|
+
# * <tt>processor</tt>: The processor instance
|
45
43
|
# * <tt>name</tt>: The processor name
|
46
44
|
# * <tt>waiting_queue</tt>: The waiting queue that provides messages to be processed
|
47
45
|
# * <tt>error_queue</tt>: The queue where errors are posted.
|
@@ -51,10 +49,10 @@ module Refinery #:nodoc:
|
|
51
49
|
# The settings hash may contain the following options:
|
52
50
|
# * <tt>visibility</tt>: The time in seconds that the message is hidden
|
53
51
|
# in the queue.
|
54
|
-
def initialize(
|
55
|
-
|
52
|
+
def initialize(processor, name, queue_prefix='', settings={})
|
53
|
+
logger.debug "Starting daemon"
|
56
54
|
|
57
|
-
@
|
55
|
+
@processor = processor
|
58
56
|
@name = name
|
59
57
|
@settings = settings
|
60
58
|
|
@@ -63,49 +61,59 @@ module Refinery #:nodoc:
|
|
63
61
|
logger.debug "Using queue #{queue_name}"
|
64
62
|
@queue_name = queue_name
|
65
63
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
rescue Exception => e
|
89
|
-
with_queue("#{queue_name}_error") do |error_queue|
|
90
|
-
error_message = {
|
91
|
-
'error' => {
|
92
|
-
'message' => e.message,
|
93
|
-
'class' => e.class.name
|
94
|
-
},
|
64
|
+
super do
|
65
|
+
begin
|
66
|
+
execute
|
67
|
+
rescue Exception => e
|
68
|
+
logger.error e
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
def execute
|
75
|
+
logger.debug "Running daemon thread: #{name} (settings: #{settings.inspect})"
|
76
|
+
while(running?)
|
77
|
+
#logger.debug "Checking #{queue_name}_waiting"
|
78
|
+
with_queue("#{queue_name}_waiting") do |waiting_queue|
|
79
|
+
while (message = waiting_queue.receive(settings['visibility']))
|
80
|
+
worker = load_worker_class(name).new(self)
|
81
|
+
begin
|
82
|
+
result, run_time = worker.run(decode_message(message.body))
|
83
|
+
if result
|
84
|
+
with_queue("#{queue_name}_done") do |done_queue|
|
85
|
+
done_message = {
|
95
86
|
'host_info' => host_info,
|
96
|
-
'original' => message.body
|
87
|
+
'original' => message.body,
|
88
|
+
'run_time' => run_time
|
97
89
|
}
|
98
|
-
logger.
|
99
|
-
|
90
|
+
logger.debug "Sending 'done' message to #{done_queue.name}"
|
91
|
+
done_queue.send_message(encode_message(done_message))
|
100
92
|
end
|
93
|
+
|
94
|
+
logger.debug "Deleting message from queue"
|
101
95
|
message.delete()
|
102
96
|
end
|
97
|
+
rescue Exception => e
|
98
|
+
with_queue("#{queue_name}_error") do |error_queue|
|
99
|
+
error_message = {
|
100
|
+
'error' => {
|
101
|
+
'message' => e.message,
|
102
|
+
'class' => e.class.name
|
103
|
+
},
|
104
|
+
'host_info' => host_info,
|
105
|
+
'original' => message.body
|
106
|
+
}
|
107
|
+
logger.error "Sending 'error' message to #{error_queue.name}: #{e.message}"
|
108
|
+
error_queue.send_message(encode_message(error_message))
|
109
|
+
end
|
110
|
+
message.delete()
|
103
111
|
end
|
104
|
-
sleep(settings['sleep'] || 5)
|
105
112
|
end
|
113
|
+
sleep(settings['sleep'] || 5)
|
106
114
|
end
|
107
|
-
logger.debug "Exiting daemon thread"
|
108
115
|
end
|
116
|
+
logger.debug "Exiting daemon thread"
|
109
117
|
end
|
110
118
|
|
111
119
|
# A hash of worker classes
|
@@ -116,7 +124,7 @@ module Refinery #:nodoc:
|
|
116
124
|
private
|
117
125
|
# Load the appropriate worker class
|
118
126
|
def load_worker_class(name)
|
119
|
-
source_file = "#{@server.workers_directory}/#{name}.rb"
|
127
|
+
source_file = "#{@processor.server.workers_directory}/#{name}.rb"
|
120
128
|
if File.exist?(source_file)
|
121
129
|
modified_at = File.mtime(source_file)
|
122
130
|
if workers[name] != modified_at
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Refinery #:nodoc:
|
2
|
+
# This class is used to monitor all of the threads for a single
|
3
|
+
# processor.
|
4
|
+
class Processor < Thread
|
5
|
+
include Refinery::Configurable
|
6
|
+
include Refinery::Loggable
|
7
|
+
|
8
|
+
attr_reader :server
|
9
|
+
attr_reader :key
|
10
|
+
attr_reader :settings
|
11
|
+
attr_reader :daemons
|
12
|
+
|
13
|
+
# Initialize the processor.
|
14
|
+
def initialize(server, key, settings={})
|
15
|
+
@server = server
|
16
|
+
@key = key
|
17
|
+
@settings = settings
|
18
|
+
@daemons = []
|
19
|
+
super do
|
20
|
+
execute
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
# Execute the processor
|
26
|
+
def execute
|
27
|
+
queue_prefix = config['prefix'] || ''
|
28
|
+
|
29
|
+
logger.debug "Creating daemons for #{key}"
|
30
|
+
1.upto(settings['workers']['initial']) do
|
31
|
+
daemons << Daemon.new(self, key, queue_prefix, settings)
|
32
|
+
end
|
33
|
+
|
34
|
+
logger.debug "Running #{daemons.length} daemons"
|
35
|
+
|
36
|
+
wait = ThreadsWait.new(*daemons)
|
37
|
+
wait.all_waits do |daemon|
|
38
|
+
puts "a #{daemon.name} just died"
|
39
|
+
daemons.remove(daemon)
|
40
|
+
puts "starting a new #{key} daemon"
|
41
|
+
daemon = Daemon.new(self, key, queue_prefix, settings)
|
42
|
+
daemons << daemon
|
43
|
+
wait.join(daemon)
|
44
|
+
end
|
45
|
+
|
46
|
+
logger.debug "Processor #{key} is exiting"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/refinery/publisher.rb
CHANGED
@@ -19,9 +19,24 @@ module Refinery #:nodoc:
|
|
19
19
|
# into the queue associated with the publisher.
|
20
20
|
def publish(message)
|
21
21
|
with_queue(waiting_queue_name) do |waiting_queue|
|
22
|
-
|
23
|
-
waiting_queue.send_message(Base64.encode64(message.to_json))
|
22
|
+
publish_to_queue(waiting_queue, message)
|
24
23
|
end
|
25
24
|
end
|
25
|
+
|
26
|
+
# Publish the given message if the waiting queue is empty. The message will
|
27
|
+
# be converted to JSON and pushed into the queue associated with the
|
28
|
+
# publisher.
|
29
|
+
def publish_if_empty(message)
|
30
|
+
with_queue(waiting_queue_name) do |waiting_queue|
|
31
|
+
publish_to_queue(waiting_queue, message) if waiting_queue.size == 0
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# Publish the given message to the queue. The message will be converted
|
36
|
+
# to JSON and pushed into the queue associated with the publisher.
|
37
|
+
def publish_to_queue(queue, message)
|
38
|
+
logger.debug "Publisher #{self.class.name} sending message: #{message.to_json}"
|
39
|
+
queue.send_message(Base64.encode64(message.to_json))
|
40
|
+
end
|
26
41
|
end
|
27
42
|
end
|
data/lib/refinery/queueable.rb
CHANGED
@@ -9,24 +9,9 @@ module Refinery #:nodoc:
|
|
9
9
|
end
|
10
10
|
|
11
11
|
# Given the queue name and a block, yield the named queue into
|
12
|
-
# the block.
|
13
|
-
# in the block and will recreate the provider automatically.
|
14
|
-
#
|
15
|
-
# Note that errors will not be propagated beyond this block. You
|
16
|
-
# have been warned.
|
12
|
+
# the block.
|
17
13
|
def with_queue(name, &block)
|
18
|
-
|
19
|
-
yield queue(name)
|
20
|
-
rescue Exception => e
|
21
|
-
logger.error "Queue error: #{e.message}"
|
22
|
-
# this removes the sqs connection from the current thread.
|
23
|
-
# note that this is brittle and will break if the RightAWS
|
24
|
-
# library changes the name or the way the sqs connection is
|
25
|
-
# stored in the thread local hash
|
26
|
-
Thread.current[:sqs_connection] = nil
|
27
|
-
sleep(5)
|
28
|
-
retry
|
29
|
-
end
|
14
|
+
yield queue(name)
|
30
15
|
end
|
31
16
|
|
32
17
|
protected
|
data/lib/refinery/server.rb
CHANGED
@@ -54,26 +54,21 @@ module Refinery #:nodoc:
|
|
54
54
|
# Run the server
|
55
55
|
def run
|
56
56
|
logger.info "Starting Refinery server"
|
57
|
-
|
57
|
+
execute_processors
|
58
58
|
logger.info "Server is exiting"
|
59
59
|
end
|
60
60
|
|
61
61
|
private
|
62
|
-
def
|
63
|
-
|
64
|
-
config['processors'].
|
65
|
-
|
66
|
-
1.upto(settings['workers']['initial']) do
|
67
|
-
daemons << Refinery::Daemon.new(self, key, queue_prefix, settings)
|
68
|
-
end
|
69
|
-
|
70
|
-
logger.debug "Running #{daemons.length} daemons"
|
62
|
+
def execute_processors
|
63
|
+
|
64
|
+
@processors = config['processors'].map do |key, settings|
|
65
|
+
Processor.new(self, key, settings)
|
71
66
|
end
|
72
67
|
|
73
68
|
Heartbeat.new(self)
|
74
69
|
|
75
70
|
begin
|
76
|
-
|
71
|
+
@processors.each { |p| p.join }
|
77
72
|
rescue Interrupt => e
|
78
73
|
end
|
79
74
|
end
|
data/lib/refinery.rb
CHANGED
@@ -45,6 +45,8 @@ module Refinery
|
|
45
45
|
|
46
46
|
# Require internal code files
|
47
47
|
def self.require_internals
|
48
|
+
require 'thwait'
|
49
|
+
|
48
50
|
require 'refinery/loggable'
|
49
51
|
require 'refinery/configurable'
|
50
52
|
require 'refinery/queueable'
|
@@ -56,6 +58,7 @@ module Refinery
|
|
56
58
|
require 'refinery/config'
|
57
59
|
require 'refinery/heartbeat'
|
58
60
|
require 'refinery/server'
|
61
|
+
require 'refinery/processor'
|
59
62
|
require 'refinery/daemon'
|
60
63
|
require 'refinery/worker'
|
61
64
|
require 'refinery/event_publisher'
|
data/publishers/error.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
1
|
# An example publisher that posts a message to the queue that should raise an error.
|
2
2
|
class ErrorPublisher < Refinery::Publisher
|
3
3
|
def execute
|
4
|
-
|
5
|
-
publish({'text' => 'fire an error, please'})
|
6
|
-
end
|
4
|
+
publish_if_empty({'text' => 'fire an error, please'})
|
7
5
|
end
|
8
6
|
end
|
data/publishers/sample.rb
CHANGED
data/publishers/sleep.rb
CHANGED
data/refinery.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{refinery}
|
5
|
-
s.version = "0.
|
5
|
+
s.version = "0.10.0"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Anthony Eden"]
|
9
|
-
s.date = %q{2009-07-
|
9
|
+
s.date = %q{2009-07-13}
|
10
10
|
s.description = %q{Process data in a distributed fashion.}
|
11
11
|
s.email = %q{anthonyeden@gmail.com}
|
12
12
|
s.executables = ["epub", "monitor", "pubnow", "refinery"]
|
@@ -37,6 +37,7 @@ Gem::Specification.new do |s|
|
|
37
37
|
"lib/refinery/heartbeat.rb",
|
38
38
|
"lib/refinery/loggable.rb",
|
39
39
|
"lib/refinery/monitor.rb",
|
40
|
+
"lib/refinery/processor.rb",
|
40
41
|
"lib/refinery/publisher.rb",
|
41
42
|
"lib/refinery/queueable.rb",
|
42
43
|
"lib/refinery/server.rb",
|
@@ -58,6 +59,7 @@ Gem::Specification.new do |s|
|
|
58
59
|
"test/unit/event_publisher_test.rb",
|
59
60
|
"test/unit/heartbeat_test.rb",
|
60
61
|
"test/unit/loggable_test.rb",
|
62
|
+
"test/unit/processor_test.rb",
|
61
63
|
"test/unit/publisher_test.rb",
|
62
64
|
"test/unit/queueable_test.rb",
|
63
65
|
"test/unit/server_test.rb",
|
@@ -84,6 +86,7 @@ Gem::Specification.new do |s|
|
|
84
86
|
"test/unit/event_publisher_test.rb",
|
85
87
|
"test/unit/heartbeat_test.rb",
|
86
88
|
"test/unit/loggable_test.rb",
|
89
|
+
"test/unit/processor_test.rb",
|
87
90
|
"test/unit/publisher_test.rb",
|
88
91
|
"test/unit/queueable_test.rb",
|
89
92
|
"test/unit/server_test.rb",
|
data/test/unit/daemon_test.rb
CHANGED
@@ -3,6 +3,8 @@ class DaemonTest < Test::Unit::TestCase
|
|
3
3
|
context "a daemon" do
|
4
4
|
setup do
|
5
5
|
@server = stub('Server')
|
6
|
+
@processor = stub('Processor', :server => @server)
|
7
|
+
|
6
8
|
@waiting_queue = stub('Queue(waiting)')
|
7
9
|
@error_queue = stub('Queue(error)')
|
8
10
|
@done_queue = stub('Queue(done)')
|
@@ -17,41 +19,41 @@ class DaemonTest < Test::Unit::TestCase
|
|
17
19
|
should "be startable" do
|
18
20
|
@waiting_queue.stubs(:receive)
|
19
21
|
assert_nothing_raised do
|
20
|
-
daemon = Refinery::Daemon.new(@
|
22
|
+
daemon = Refinery::Daemon.new(@processor, 'sample')
|
21
23
|
end
|
22
24
|
end
|
23
25
|
should "have logging" do
|
24
26
|
@waiting_queue.stubs(:receive)
|
25
|
-
daemon = Refinery::Daemon.new(@
|
27
|
+
daemon = Refinery::Daemon.new(@processor, 'sample')
|
26
28
|
assert_not_nil daemon.logger
|
27
29
|
end
|
28
30
|
should "allow visibility setting" do
|
29
31
|
@waiting_queue.expects(:receive).with(600)
|
30
|
-
daemon = Refinery::Daemon.new(@
|
32
|
+
daemon = Refinery::Daemon.new(@processor, 'sample', '', {'visibility' => 600})
|
31
33
|
end
|
32
34
|
should "have a queue name" do
|
33
35
|
@waiting_queue.stubs(:receive)
|
34
|
-
Refinery::Daemon.any_instance.
|
36
|
+
Refinery::Daemon.any_instance.expects(:queue).with(
|
35
37
|
'prefix_sample_waiting').returns(@waiting_queue)
|
36
|
-
daemon = Refinery::Daemon.new(@
|
38
|
+
daemon = Refinery::Daemon.new(@processor, 'sample', 'prefix_')
|
37
39
|
assert_equal 'prefix_sample', daemon.queue_name
|
38
40
|
end
|
39
41
|
context "that is started" do
|
40
42
|
setup do
|
41
43
|
@waiting_queue.stubs(:receive)
|
42
|
-
@daemon = Refinery::Daemon.new(@
|
44
|
+
@daemon = Refinery::Daemon.new(@processor, 'sample')
|
43
45
|
end
|
44
46
|
should "have a state of running" do
|
45
47
|
assert @daemon.running?
|
46
48
|
end
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
49
|
+
# context "after calling stop" do
|
50
|
+
# setup do
|
51
|
+
# @daemon.stop
|
52
|
+
# end
|
53
|
+
# should "not be running" do
|
54
|
+
# assert !@daemon.running?
|
55
|
+
# end
|
56
|
+
# end
|
55
57
|
end
|
56
58
|
end
|
57
59
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../test_helper'
|
2
|
+
class ProcessorTest < Test::Unit::TestCase
|
3
|
+
context "a processor" do
|
4
|
+
setup do
|
5
|
+
@server = stub('Server')
|
6
|
+
@settings = {
|
7
|
+
'workers' => {
|
8
|
+
'initial' => 1
|
9
|
+
}
|
10
|
+
}
|
11
|
+
|
12
|
+
@waiting_queue = stub('Queue(waiting)')
|
13
|
+
@error_queue = stub('Queue(error)')
|
14
|
+
@done_queue = stub('Queue(done)')
|
15
|
+
|
16
|
+
Refinery::Daemon.any_instance.stubs(:queue).with(
|
17
|
+
'sample_waiting').returns(@waiting_queue)
|
18
|
+
Refinery::Daemon.any_instance.stubs(:queue).with(
|
19
|
+
'sample_error').returns(@error_queue)
|
20
|
+
Refinery::Daemon.any_instance.stubs(:queue).with(
|
21
|
+
'sample_done').returns(@done_queue)
|
22
|
+
end
|
23
|
+
should "initialize" do
|
24
|
+
assert_nothing_raised do
|
25
|
+
@waiting_queue.stubs(:receive)
|
26
|
+
Refinery::Processor.new(@server, 'sample', @settings)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/test/unit/server_test.rb
CHANGED
@@ -20,13 +20,7 @@ class ServerTest < Test::Unit::TestCase
|
|
20
20
|
should "be runnable" do
|
21
21
|
setup_default_config
|
22
22
|
|
23
|
-
|
24
|
-
heartbeat_queue.stubs(:send_message)
|
25
|
-
queue_provider = stub('queue provider')
|
26
|
-
queue_provider.expects(:queue).with('heartbeat').returns(heartbeat_queue)
|
27
|
-
RightAws::SqsGen2.expects(:new).with(
|
28
|
-
'aki', 'sak', {:multi_thread => true}
|
29
|
-
).returns(queue_provider)
|
23
|
+
Refinery::Heartbeat.expects(:new)
|
30
24
|
|
31
25
|
assert_nothing_raised do
|
32
26
|
thread = Thread.new do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aeden-refinery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Anthony Eden
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-07-
|
12
|
+
date: 2009-07-13 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -48,6 +48,7 @@ files:
|
|
48
48
|
- lib/refinery/heartbeat.rb
|
49
49
|
- lib/refinery/loggable.rb
|
50
50
|
- lib/refinery/monitor.rb
|
51
|
+
- lib/refinery/processor.rb
|
51
52
|
- lib/refinery/publisher.rb
|
52
53
|
- lib/refinery/queueable.rb
|
53
54
|
- lib/refinery/server.rb
|
@@ -69,6 +70,7 @@ files:
|
|
69
70
|
- test/unit/event_publisher_test.rb
|
70
71
|
- test/unit/heartbeat_test.rb
|
71
72
|
- test/unit/loggable_test.rb
|
73
|
+
- test/unit/processor_test.rb
|
72
74
|
- test/unit/publisher_test.rb
|
73
75
|
- test/unit/queueable_test.rb
|
74
76
|
- test/unit/server_test.rb
|
@@ -113,6 +115,7 @@ test_files:
|
|
113
115
|
- test/unit/event_publisher_test.rb
|
114
116
|
- test/unit/heartbeat_test.rb
|
115
117
|
- test/unit/loggable_test.rb
|
118
|
+
- test/unit/processor_test.rb
|
116
119
|
- test/unit/publisher_test.rb
|
117
120
|
- test/unit/queueable_test.rb
|
118
121
|
- test/unit/server_test.rb
|