refinery 0.10.5 → 0.10.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/lib/refinery/heartbeat.rb +2 -1
- data/lib/refinery/monitor.rb +11 -8
- data/lib/refinery/server.rb +1 -1
- data/lib/refinery/stats_server.rb +1 -1
- data/refinery.gemspec +2 -2
- metadata +2 -2
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.10.
|
|
1
|
+
0.10.6
|
data/lib/refinery/heartbeat.rb
CHANGED
|
@@ -8,10 +8,11 @@ module Refinery #:nodoc:
|
|
|
8
8
|
|
|
9
9
|
# Initialize the heartbeat for the given server.
|
|
10
10
|
def initialize(server)
|
|
11
|
+
queue_prefix = config['prefix'] || ''
|
|
11
12
|
@server = server
|
|
12
13
|
@thread = Thread.new do
|
|
13
14
|
loop do
|
|
14
|
-
with_queue(
|
|
15
|
+
with_queue("#{queue_prefix}heartbeat") do |heartbeat_queue|
|
|
15
16
|
logger.debug "Send heartbeat"
|
|
16
17
|
message = {
|
|
17
18
|
'host_info' => host_info,
|
data/lib/refinery/monitor.rb
CHANGED
|
@@ -17,6 +17,7 @@ module Refinery #:nodoc:
|
|
|
17
17
|
logger.level = Logger::INFO if options[:verbose]
|
|
18
18
|
logger.level = Logger::DEBUG if options[:debug]
|
|
19
19
|
config.load_file(options[:config]) if options[:config]
|
|
20
|
+
@queue_prefix = config['prefix'] || ''
|
|
20
21
|
end
|
|
21
22
|
|
|
22
23
|
# Execute the monitor. The monitor will start one heartbeat
|
|
@@ -48,12 +49,12 @@ module Refinery #:nodoc:
|
|
|
48
49
|
end
|
|
49
50
|
|
|
50
51
|
def run_heartbeat_monitor
|
|
51
|
-
logger.info "Starting heartbeat monitor"
|
|
52
|
-
Thread.new(
|
|
52
|
+
logger.info "Starting #{@queue_prefix}heartbeat monitor"
|
|
53
|
+
Thread.new("#{@queue_prefix}heartbeat") do |heartbeat_queue_name|
|
|
53
54
|
loop do
|
|
54
55
|
with_queue(heartbeat_queue_name) do |heartbeat_queue|
|
|
55
56
|
while (message = heartbeat_queue.receive)
|
|
56
|
-
logger.debug decode_message(message.body).inspect
|
|
57
|
+
logger.debug "#{heartbeat_queue.name}: #{decode_message(message.body).inspect}"
|
|
57
58
|
message.delete()
|
|
58
59
|
end
|
|
59
60
|
end
|
|
@@ -65,15 +66,16 @@ module Refinery #:nodoc:
|
|
|
65
66
|
def run_done_monitors
|
|
66
67
|
config['processors'].collect do |key, settings|
|
|
67
68
|
queue_name = settings['queue'] || key
|
|
68
|
-
done_queue_name = "#{queue_name}_done"
|
|
69
|
+
done_queue_name = "#{@queue_prefix}#{queue_name}_done"
|
|
69
70
|
logger.debug "Starting monitor for queue #{done_queue_name}"
|
|
70
71
|
Thread.new(done_queue_name) do |done_queue_name|
|
|
71
72
|
loop do
|
|
72
73
|
with_queue(done_queue_name) do |done_queue|
|
|
73
74
|
while (message = done_queue.receive)
|
|
74
75
|
done_message = decode_message(message.body)
|
|
76
|
+
logger.debug "#{done_queue.name}: #{done_message.pretty_inspect}"
|
|
75
77
|
processed = decode_message(done_message['original'])
|
|
76
|
-
logger.info "Done: #{processed.inspect}"
|
|
78
|
+
logger.info "Done: #{queue_name} #{processed.inspect}"
|
|
77
79
|
message.delete()
|
|
78
80
|
statistics.record_done(done_message)
|
|
79
81
|
end
|
|
@@ -87,15 +89,16 @@ module Refinery #:nodoc:
|
|
|
87
89
|
def run_error_monitors
|
|
88
90
|
config['processors'].collect do |key, settings|
|
|
89
91
|
queue_name = settings['queue'] || key
|
|
90
|
-
error_queue_name = "#{queue_name}_error"
|
|
92
|
+
error_queue_name = "#{@queue_prefix}#{queue_name}_error"
|
|
91
93
|
logger.info "Starting error monitor for queue #{error_queue_name}"
|
|
92
94
|
Thread.new(error_queue_name) do |error_queue_name|
|
|
93
95
|
loop do
|
|
94
|
-
|
|
96
|
+
with_queue(error_queue_name) do |error_queue|
|
|
95
97
|
while (message = error_queue.receive)
|
|
96
98
|
error_message = decode_message(message.body)
|
|
99
|
+
logger.debug "#{error_queue.name}: #{error_message.pretty_inspect}"
|
|
97
100
|
processed = decode_message(error_message['original'])
|
|
98
|
-
logger.info "Error: #{processed.inspect}"
|
|
101
|
+
logger.info "Error: #{queue_name} #{processed.inspect}"
|
|
99
102
|
message.delete()
|
|
100
103
|
statistics.record_error(error_message)
|
|
101
104
|
end
|
data/lib/refinery/server.rb
CHANGED
|
@@ -79,7 +79,7 @@ module Refinery #:nodoc:
|
|
|
79
79
|
@format ||= "%s, [%s#%d][%s] %5s -- %s: %s\n"
|
|
80
80
|
end
|
|
81
81
|
def call(severity, time, progname, msg)
|
|
82
|
-
format % [severity[0..0], format_datetime(time), $$, Thread.current.object_id.to_s, severity, progname, msg.to_s]
|
|
82
|
+
format % [severity[0..0], format_datetime(time.utc), $$, Thread.current.object_id.to_s, severity, progname, msg.to_s]
|
|
83
83
|
end
|
|
84
84
|
def format_datetime(time)
|
|
85
85
|
time.strftime("%Y-%m-%dT%H:%M:%S.") << "%06d " % time.usec
|
|
@@ -10,7 +10,7 @@ module Refinery #:nodoc:
|
|
|
10
10
|
Ramaze::Log.loggers.clear # supress all Ramaze logging
|
|
11
11
|
Ramaze.start # start the Ramaze server on port 7000
|
|
12
12
|
rescue NameError
|
|
13
|
-
self.logger.warn "Install
|
|
13
|
+
self.logger.warn "Install Ramaze to enable the stats server"
|
|
14
14
|
end
|
|
15
15
|
end
|
|
16
16
|
|
data/refinery.gemspec
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
Gem::Specification.new do |s|
|
|
4
4
|
s.name = %q{refinery}
|
|
5
|
-
s.version = "0.10.
|
|
5
|
+
s.version = "0.10.6"
|
|
6
6
|
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
8
8
|
s.authors = ["Anthony Eden"]
|
|
9
|
-
s.date = %q{2009-07-
|
|
9
|
+
s.date = %q{2009-07-24}
|
|
10
10
|
s.description = %q{Process data in a distributed fashion.}
|
|
11
11
|
s.email = %q{anthonyeden@gmail.com}
|
|
12
12
|
s.executables = ["epub", "monitor", "pubnow", "refinery"]
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: refinery
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.10.
|
|
4
|
+
version: 0.10.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Anthony Eden
|
|
@@ -9,7 +9,7 @@ autorequire:
|
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
11
|
|
|
12
|
-
date: 2009-07-
|
|
12
|
+
date: 2009-07-24 00:00:00 -10:00
|
|
13
13
|
default_executable:
|
|
14
14
|
dependencies: []
|
|
15
15
|
|