aeden-refinery 0.10.5 → 0.10.6
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/refinery/heartbeat.rb +2 -1
- data/lib/refinery/monitor.rb +11 -8
- data/lib/refinery/server.rb +1 -1
- data/lib/refinery/stats_server.rb +1 -1
- data/refinery.gemspec +4 -5
- metadata +4 -4
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.10.
|
1
|
+
0.10.6
|
data/lib/refinery/heartbeat.rb
CHANGED
@@ -8,10 +8,11 @@ module Refinery #:nodoc:
|
|
8
8
|
|
9
9
|
# Initialize the heartbeat for the given server.
|
10
10
|
def initialize(server)
|
11
|
+
queue_prefix = config['prefix'] || ''
|
11
12
|
@server = server
|
12
13
|
@thread = Thread.new do
|
13
14
|
loop do
|
14
|
-
with_queue(
|
15
|
+
with_queue("#{queue_prefix}heartbeat") do |heartbeat_queue|
|
15
16
|
logger.debug "Send heartbeat"
|
16
17
|
message = {
|
17
18
|
'host_info' => host_info,
|
data/lib/refinery/monitor.rb
CHANGED
@@ -17,6 +17,7 @@ module Refinery #:nodoc:
|
|
17
17
|
logger.level = Logger::INFO if options[:verbose]
|
18
18
|
logger.level = Logger::DEBUG if options[:debug]
|
19
19
|
config.load_file(options[:config]) if options[:config]
|
20
|
+
@queue_prefix = config['prefix'] || ''
|
20
21
|
end
|
21
22
|
|
22
23
|
# Execute the monitor. The monitor will start one heartbeat
|
@@ -48,12 +49,12 @@ module Refinery #:nodoc:
|
|
48
49
|
end
|
49
50
|
|
50
51
|
def run_heartbeat_monitor
|
51
|
-
logger.info "Starting heartbeat monitor"
|
52
|
-
Thread.new(
|
52
|
+
logger.info "Starting #{@queue_prefix}heartbeat monitor"
|
53
|
+
Thread.new("#{@queue_prefix}heartbeat") do |heartbeat_queue_name|
|
53
54
|
loop do
|
54
55
|
with_queue(heartbeat_queue_name) do |heartbeat_queue|
|
55
56
|
while (message = heartbeat_queue.receive)
|
56
|
-
logger.debug decode_message(message.body).inspect
|
57
|
+
logger.debug "#{heartbeat_queue.name}: #{decode_message(message.body).inspect}"
|
57
58
|
message.delete()
|
58
59
|
end
|
59
60
|
end
|
@@ -65,15 +66,16 @@ module Refinery #:nodoc:
|
|
65
66
|
def run_done_monitors
|
66
67
|
config['processors'].collect do |key, settings|
|
67
68
|
queue_name = settings['queue'] || key
|
68
|
-
done_queue_name = "#{queue_name}_done"
|
69
|
+
done_queue_name = "#{@queue_prefix}#{queue_name}_done"
|
69
70
|
logger.debug "Starting monitor for queue #{done_queue_name}"
|
70
71
|
Thread.new(done_queue_name) do |done_queue_name|
|
71
72
|
loop do
|
72
73
|
with_queue(done_queue_name) do |done_queue|
|
73
74
|
while (message = done_queue.receive)
|
74
75
|
done_message = decode_message(message.body)
|
76
|
+
logger.debug "#{done_queue.name}: #{done_message.pretty_inspect}"
|
75
77
|
processed = decode_message(done_message['original'])
|
76
|
-
logger.info "Done: #{processed.inspect}"
|
78
|
+
logger.info "Done: #{queue_name} #{processed.inspect}"
|
77
79
|
message.delete()
|
78
80
|
statistics.record_done(done_message)
|
79
81
|
end
|
@@ -87,15 +89,16 @@ module Refinery #:nodoc:
|
|
87
89
|
def run_error_monitors
|
88
90
|
config['processors'].collect do |key, settings|
|
89
91
|
queue_name = settings['queue'] || key
|
90
|
-
error_queue_name = "#{queue_name}_error"
|
92
|
+
error_queue_name = "#{@queue_prefix}#{queue_name}_error"
|
91
93
|
logger.info "Starting error monitor for queue #{error_queue_name}"
|
92
94
|
Thread.new(error_queue_name) do |error_queue_name|
|
93
95
|
loop do
|
94
|
-
|
96
|
+
with_queue(error_queue_name) do |error_queue|
|
95
97
|
while (message = error_queue.receive)
|
96
98
|
error_message = decode_message(message.body)
|
99
|
+
logger.debug "#{error_queue.name}: #{error_message.pretty_inspect}"
|
97
100
|
processed = decode_message(error_message['original'])
|
98
|
-
logger.info "Error: #{processed.inspect}"
|
101
|
+
logger.info "Error: #{queue_name} #{processed.inspect}"
|
99
102
|
message.delete()
|
100
103
|
statistics.record_error(error_message)
|
101
104
|
end
|
data/lib/refinery/server.rb
CHANGED
@@ -79,7 +79,7 @@ module Refinery #:nodoc:
|
|
79
79
|
@format ||= "%s, [%s#%d][%s] %5s -- %s: %s\n"
|
80
80
|
end
|
81
81
|
def call(severity, time, progname, msg)
|
82
|
-
format % [severity[0..0], format_datetime(time), $$, Thread.current.object_id.to_s, severity, progname, msg.to_s]
|
82
|
+
format % [severity[0..0], format_datetime(time.utc), $$, Thread.current.object_id.to_s, severity, progname, msg.to_s]
|
83
83
|
end
|
84
84
|
def format_datetime(time)
|
85
85
|
time.strftime("%Y-%m-%dT%H:%M:%S.") << "%06d " % time.usec
|
@@ -10,7 +10,7 @@ module Refinery #:nodoc:
|
|
10
10
|
Ramaze::Log.loggers.clear # supress all Ramaze logging
|
11
11
|
Ramaze.start # start the Ramaze server on port 7000
|
12
12
|
rescue NameError
|
13
|
-
self.logger.warn "Install
|
13
|
+
self.logger.warn "Install Ramaze to enable the stats server"
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
data/refinery.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{refinery}
|
5
|
-
s.version = "0.10.
|
5
|
+
s.version = "0.10.6"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Anthony Eden"]
|
9
|
-
s.date = %q{2009-07-
|
9
|
+
s.date = %q{2009-07-23}
|
10
10
|
s.description = %q{Process data in a distributed fashion.}
|
11
11
|
s.email = %q{anthonyeden@gmail.com}
|
12
12
|
s.executables = ["epub", "monitor", "pubnow", "refinery"]
|
@@ -71,12 +71,11 @@ Gem::Specification.new do |s|
|
|
71
71
|
"workers/sample.rb",
|
72
72
|
"workers/sleep.rb"
|
73
73
|
]
|
74
|
-
s.has_rdoc = true
|
75
74
|
s.homepage = %q{http://github.com/aeden/refinery}
|
76
75
|
s.rdoc_options = ["--charset=UTF-8"]
|
77
76
|
s.require_paths = ["lib"]
|
78
77
|
s.rubyforge_project = %q{refinery}
|
79
|
-
s.rubygems_version = %q{1.3.
|
78
|
+
s.rubygems_version = %q{1.3.5}
|
80
79
|
s.summary = %q{Refinery processes data in a distributed environment.}
|
81
80
|
s.test_files = [
|
82
81
|
"test/test_helper.rb",
|
@@ -98,7 +97,7 @@ Gem::Specification.new do |s|
|
|
98
97
|
|
99
98
|
if s.respond_to? :specification_version then
|
100
99
|
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
101
|
-
s.specification_version =
|
100
|
+
s.specification_version = 3
|
102
101
|
|
103
102
|
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
104
103
|
else
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aeden-refinery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Anthony Eden
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-07-
|
12
|
+
date: 2009-07-23 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -81,7 +81,7 @@ files:
|
|
81
81
|
- workers/error.rb
|
82
82
|
- workers/sample.rb
|
83
83
|
- workers/sleep.rb
|
84
|
-
has_rdoc:
|
84
|
+
has_rdoc: false
|
85
85
|
homepage: http://github.com/aeden/refinery
|
86
86
|
post_install_message:
|
87
87
|
rdoc_options:
|
@@ -105,7 +105,7 @@ requirements: []
|
|
105
105
|
rubyforge_project: refinery
|
106
106
|
rubygems_version: 1.2.0
|
107
107
|
signing_key:
|
108
|
-
specification_version:
|
108
|
+
specification_version: 3
|
109
109
|
summary: Refinery processes data in a distributed environment.
|
110
110
|
test_files:
|
111
111
|
- test/test_helper.rb
|