smith 0.5.12 → 0.5.13.1
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/smithctl +16 -19
- data/lib/smith.rb +25 -41
- data/lib/smith/agent.rb +96 -66
- data/lib/smith/agent_monitoring.rb +3 -4
- data/lib/smith/agent_process.rb +16 -9
- data/lib/smith/amqp_errors.rb +53 -0
- data/lib/smith/application/agency.rb +23 -20
- data/lib/smith/bootstrap.rb +3 -3
- data/lib/smith/command.rb +2 -2
- data/lib/smith/command_base.rb +4 -0
- data/lib/smith/commands/agency/agents.rb +19 -19
- data/lib/smith/commands/agency/kill.rb +6 -2
- data/lib/smith/commands/agency/list.rb +2 -4
- data/lib/smith/commands/agency/logger.rb +27 -28
- data/lib/smith/commands/agency/metadata.rb +1 -5
- data/lib/smith/commands/agency/object_count.rb +13 -11
- data/lib/smith/commands/agency/restart.rb +18 -9
- data/lib/smith/commands/agency/start.rb +34 -25
- data/lib/smith/commands/agency/stop.rb +58 -41
- data/lib/smith/commands/agency/version.rb +10 -10
- data/lib/smith/commands/common.rb +7 -4
- data/lib/smith/commands/smithctl/acl.rb +46 -37
- data/lib/smith/commands/smithctl/commands.rb +1 -1
- data/lib/smith/commands/smithctl/firehose.rb +30 -0
- data/lib/smith/commands/smithctl/pop.rb +39 -32
- data/lib/smith/commands/smithctl/push.rb +70 -51
- data/lib/smith/commands/smithctl/rm.rb +32 -9
- data/lib/smith/commands/smithctl/subscribe.rb +36 -0
- data/lib/smith/commands/smithctl/top.rb +1 -1
- data/lib/smith/exceptions.rb +2 -0
- data/lib/smith/messaging/acl/agency_command.proto +4 -0
- data/lib/smith/messaging/acl/default.rb +8 -1
- data/lib/smith/messaging/amqp_options.rb +2 -2
- data/lib/smith/messaging/message_counter.rb +21 -0
- data/lib/smith/messaging/payload.rb +47 -49
- data/lib/smith/messaging/queue.rb +50 -0
- data/lib/smith/messaging/queue_definition.rb +18 -0
- data/lib/smith/messaging/queue_factory.rb +20 -29
- data/lib/smith/messaging/receiver.rb +211 -173
- data/lib/smith/messaging/requeue.rb +91 -0
- data/lib/smith/messaging/sender.rb +184 -28
- data/lib/smith/messaging/util.rb +72 -0
- data/lib/smith/queue_definitions.rb +11 -0
- data/lib/smith/version.rb +1 -1
- metadata +18 -10
- data/lib/smith/messaging/endpoint.rb +0 -116
data/bin/smithctl
CHANGED
@@ -10,10 +10,11 @@ $:.unshift(File.dirname(__FILE__) + '/../lib')
|
|
10
10
|
|
11
11
|
require 'smith'
|
12
12
|
|
13
|
+
include Smith::Logger
|
14
|
+
|
13
15
|
module Smith
|
14
16
|
class SmithControl
|
15
17
|
|
16
|
-
include Logger
|
17
18
|
|
18
19
|
def initialize(options={})
|
19
20
|
log_level((options[:log_level_given]) ? options[:log_level].to_sym : :info)
|
@@ -31,32 +32,28 @@ module Smith
|
|
31
32
|
private
|
32
33
|
|
33
34
|
def smithctl_command(command, args, &blk)
|
34
|
-
responder
|
35
|
-
responder.callback do |v|
|
36
|
-
blk.call(v)
|
37
|
-
end
|
38
|
-
|
39
|
-
Command.run(command, args, :responder => responder)
|
35
|
+
Command.run(command, args, :responder => EM::Completion.new.tap { |c| c.completion(blk) })
|
40
36
|
end
|
41
37
|
|
42
38
|
def agency_command(command, args, &blk)
|
43
|
-
Messaging::Sender.new(
|
39
|
+
Messaging::Sender.new(QueueDefinitions::Agency_control) do |sender|
|
44
40
|
|
45
|
-
|
41
|
+
timeout = Messaging::Timeout.new(5) { |message_id| blk.call("Timeout. Is the agency still running?") }
|
46
42
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
sender.publish_and_receive(payload) do |r|
|
51
|
-
blk.call(r.payload)
|
52
|
-
end
|
43
|
+
sender.on_reply(:timeout => timeout) do |reply_payload, r|
|
44
|
+
r.ack
|
45
|
+
blk.call(reply_payload[:response])
|
53
46
|
end
|
54
47
|
|
55
|
-
|
56
|
-
|
57
|
-
end
|
48
|
+
# callback = proc do |sender|
|
49
|
+
sender.publish(Smith::ACL::Factory.create(:agency_command, :command => command, :args => args))
|
50
|
+
# end
|
51
|
+
|
52
|
+
# errback = proc do
|
53
|
+
# blk.call("Agency not running.")
|
54
|
+
# end
|
58
55
|
|
59
|
-
sender.consumers?(callback, errback)
|
56
|
+
# sender.consumers?(callback, errback)
|
60
57
|
end
|
61
58
|
end
|
62
59
|
end
|
data/lib/smith.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
require 'amqp'
|
3
3
|
require 'tmpdir'
|
4
|
+
require "socket"
|
4
5
|
require 'logging'
|
5
6
|
require 'pathname'
|
6
7
|
require 'fileutils'
|
@@ -21,9 +22,9 @@ module Smith
|
|
21
22
|
|
22
23
|
class << self
|
23
24
|
|
24
|
-
def
|
25
|
-
raise RuntimeError, "You must run this in a Smith.start block" if @
|
26
|
-
@
|
25
|
+
def connection
|
26
|
+
raise RuntimeError, "You must run this in a Smith.start block" if @connection.nil?
|
27
|
+
@connection
|
27
28
|
end
|
28
29
|
|
29
30
|
def environment
|
@@ -42,6 +43,11 @@ module Smith
|
|
42
43
|
path_to_pathnames(config.agency.agent_path)
|
43
44
|
end
|
44
45
|
|
46
|
+
# Convenience method to get the hostname
|
47
|
+
def hostname
|
48
|
+
Socket.gethostname
|
49
|
+
end
|
50
|
+
|
45
51
|
def acl_path
|
46
52
|
path_to_pathnames(config.agency.acl_path)
|
47
53
|
end
|
@@ -81,23 +87,12 @@ module Smith
|
|
81
87
|
EM.reactor_running?
|
82
88
|
end
|
83
89
|
|
84
|
-
# Define a channel error handler.
|
85
|
-
def on_error(chain=false, &blk)
|
86
|
-
# This strikes me as egregiously wrong but I don't know how to
|
87
|
-
# overwrite an already existing handler.
|
88
|
-
if chain
|
89
|
-
Smith.channel.callbacks[:error] << blk
|
90
|
-
else
|
91
|
-
Smith.channel.callbacks[:error] = [blk]
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
90
|
def start(opts={}, &block)
|
96
91
|
EM.epoll if EM.epoll?
|
97
92
|
EM.kqueue if EM.kqueue?
|
98
93
|
EM.set_descriptor_table_size(opts[:fdsize] || 1024)
|
99
94
|
|
100
|
-
connection_settings = config.amqp.broker.
|
95
|
+
connection_settings = config.amqp.broker.to_hash.merge(
|
101
96
|
:on_tcp_connection_failure => method(:tcp_connection_failure_handler),
|
102
97
|
:on_possible_authentication_failure => method(:authentication_failure_handler))
|
103
98
|
|
@@ -121,15 +116,16 @@ module Smith
|
|
121
116
|
logger.info { "Connection to AMQP server restored" }
|
122
117
|
end
|
123
118
|
|
124
|
-
connection.on_error do |connection,
|
125
|
-
case
|
119
|
+
connection.on_error do |connection, connection_close|
|
120
|
+
case connection_close.reply_code
|
126
121
|
when 320
|
127
122
|
logger.warn { "AMQP server shutdown. Waiting." }
|
128
123
|
else
|
129
124
|
if @handler
|
130
125
|
@handler.call(connection, reason)
|
131
126
|
else
|
132
|
-
logger.error { "AMQP Server error: #{
|
127
|
+
logger.error { "AMQP Server error: #{connection_close.reply_code}: #{connection_close.reply_text}" }
|
128
|
+
EM.stop_event_loop
|
133
129
|
end
|
134
130
|
end
|
135
131
|
end
|
@@ -137,25 +133,7 @@ module Smith
|
|
137
133
|
# This will be the last thing run by the reactor.
|
138
134
|
shutdown_hook { logger.debug { "Reactor Stopped" } }
|
139
135
|
|
140
|
-
|
141
|
-
@channel = channel
|
142
|
-
# Set up QOS. If you do not do this then the subscribe in receive_message
|
143
|
-
# will get overwhelmed and the whole thing will collapse in on itself.
|
144
|
-
channel.prefetch(1)
|
145
|
-
|
146
|
-
# Set up a default handler.
|
147
|
-
on_error do |ch,channel_close|
|
148
|
-
logger.fatal { "Channel level exception: #{channel_close.reply_code}: #{channel_close.reply_text}" }
|
149
|
-
logger.fatal { "Exiting" }
|
150
|
-
Smith.stop(true)
|
151
|
-
end
|
152
|
-
|
153
|
-
# Set up auto-recovery. This will ensure that the AMQP gem reconnects each
|
154
|
-
# channel and sets up the various exchanges & queues.
|
155
|
-
channel.auto_recovery = true
|
156
|
-
|
157
|
-
block.call
|
158
|
-
end
|
136
|
+
block.call
|
159
137
|
end
|
160
138
|
end
|
161
139
|
|
@@ -168,7 +146,9 @@ module Smith
|
|
168
146
|
|
169
147
|
if running?
|
170
148
|
if immediately
|
171
|
-
|
149
|
+
EM.next_tick do
|
150
|
+
@connection.close { EM.stop_event_loop }
|
151
|
+
end
|
172
152
|
else
|
173
153
|
EM.add_timer(1) do
|
174
154
|
@connection.close { EM.stop_event_loop }
|
@@ -221,22 +201,26 @@ module Smith
|
|
221
201
|
end
|
222
202
|
end
|
223
203
|
|
204
|
+
require_relative 'smith/amqp_errors'
|
224
205
|
require_relative 'smith/object_count'
|
225
206
|
require_relative 'smith/cache'
|
226
|
-
require_relative 'smith/agent'
|
227
207
|
require_relative 'smith/agent_cache'
|
228
208
|
require_relative 'smith/agent_process'
|
229
209
|
require_relative 'smith/agent_monitoring'
|
230
210
|
require_relative 'smith/command'
|
231
211
|
require_relative 'smith/command_base'
|
232
212
|
require_relative 'smith/exceptions'
|
213
|
+
require_relative 'smith/object_count'
|
233
214
|
require_relative 'smith/version'
|
234
215
|
|
216
|
+
require_relative 'smith/messaging/queue_definition'
|
235
217
|
require_relative 'smith/messaging/amqp_options'
|
236
218
|
require_relative 'smith/messaging/queue_factory'
|
237
|
-
require_relative 'smith/messaging/payload'
|
238
219
|
require_relative 'smith/messaging/acl/default'
|
239
|
-
require_relative 'smith/messaging/
|
220
|
+
require_relative 'smith/messaging/payload'
|
221
|
+
require_relative 'smith/messaging/util'
|
240
222
|
require_relative 'smith/messaging/responder'
|
241
223
|
require_relative 'smith/messaging/receiver'
|
242
224
|
require_relative 'smith/messaging/sender'
|
225
|
+
|
226
|
+
require_relative 'smith/queue_definitions'
|
data/lib/smith/agent.rb
CHANGED
@@ -17,22 +17,54 @@ module Smith
|
|
17
17
|
@signal_handlers = Hash.new { |h,k| h[k] = Array.new }
|
18
18
|
|
19
19
|
setup_control_queue
|
20
|
-
setup_stats_queue
|
21
20
|
|
22
21
|
@start_time = Time.now
|
23
22
|
|
24
|
-
|
25
|
-
|
23
|
+
@state = :starting
|
24
|
+
|
25
|
+
@on_stopping = proc {|completion| completion.succeed }
|
26
|
+
@on_starting = proc {|completion| completion.succeed }
|
27
|
+
@on_running = proc {|completion| completion.succeed }
|
28
|
+
|
29
|
+
@on_starting_completion = EM::Completion.new.tap do |c|
|
30
|
+
c.completion do |completion|
|
31
|
+
acknowledge_start do
|
32
|
+
@on_running.call(@on_running_completion)
|
33
|
+
logger.info { "Agent started: #{name}:[#{pid}]." }
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
@on_running_completion = EM::Completion.new.tap do |c|
|
39
|
+
c.completion do |completion|
|
40
|
+
start_keep_alive
|
41
|
+
setup_stats_queue
|
42
|
+
@state = :running
|
43
|
+
end
|
26
44
|
end
|
27
45
|
|
28
|
-
|
29
|
-
|
46
|
+
@on_stopping_completion = EM::Completion.new.tap do |c|
|
47
|
+
c.completion do |completion|
|
48
|
+
acknowledge_stop do
|
49
|
+
@state = :stopping
|
50
|
+
Smith.stop do
|
51
|
+
logger.info { "Agent stopped: #{name}:[#{pid}]." }
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
30
55
|
end
|
31
56
|
|
32
57
|
EM.threadpool_size = 1
|
33
58
|
|
34
|
-
|
35
|
-
|
59
|
+
@on_starting.call(@on_starting_completion)
|
60
|
+
end
|
61
|
+
|
62
|
+
def on_stopping(&blk)
|
63
|
+
@on_stopping = blk
|
64
|
+
end
|
65
|
+
|
66
|
+
def on_running(&blk)
|
67
|
+
@on_running = blk
|
36
68
|
end
|
37
69
|
|
38
70
|
# Override this method to implement your own agent. You can use task but this may
|
@@ -47,14 +79,6 @@ module Smith
|
|
47
79
|
end
|
48
80
|
end
|
49
81
|
|
50
|
-
def on_started(&blk)
|
51
|
-
@on_started = blk
|
52
|
-
end
|
53
|
-
|
54
|
-
def on_stopped(&blk)
|
55
|
-
Smith.shutdown_hook(&blk)
|
56
|
-
end
|
57
|
-
|
58
82
|
def install_signal_handler(signal, position=:end, &blk)
|
59
83
|
raise ArgumentError, "Unknown position: #{position}" if ![:beginning, :end].include?(position)
|
60
84
|
|
@@ -65,20 +89,21 @@ module Smith
|
|
65
89
|
end
|
66
90
|
end
|
67
91
|
|
68
|
-
def
|
69
|
-
@
|
92
|
+
def state
|
93
|
+
@state
|
70
94
|
end
|
71
95
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
96
|
+
# Set convenience state methods.
|
97
|
+
[:starting, :stopping, :running].each do |method|
|
98
|
+
define_method("#{method}?", proc { state == method })
|
99
|
+
end
|
100
|
+
|
101
|
+
def receiver(queue_name, opts={}, &blk)
|
102
|
+
queues.receiver(queue_name, opts, &blk)
|
78
103
|
end
|
79
104
|
|
80
|
-
def sender(queue_name, opts={})
|
81
|
-
queues.sender(queue_name, opts)
|
105
|
+
def sender(queue_name, opts={}, &blk)
|
106
|
+
queues.sender(queue_name, opts, &blk)
|
82
107
|
end
|
83
108
|
|
84
109
|
class << self
|
@@ -104,56 +129,59 @@ module Smith
|
|
104
129
|
|
105
130
|
def setup_control_queue
|
106
131
|
logger.debug { "Setting up control queue: #{control_queue_name}" }
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
132
|
+
|
133
|
+
Messaging::Receiver.new(control_queue_name, :durable => false, :auto_delete => true) do |receiver|
|
134
|
+
receiver.subscribe do |payload|
|
135
|
+
logger.debug { "Command received on agent control queue: #{payload.command} #{payload.options}" }
|
136
|
+
|
137
|
+
case payload.command
|
138
|
+
when 'object_count'
|
139
|
+
object_count(payload.options.first.to_i).each{|o| logger.info{o}}
|
140
|
+
when 'stop'
|
141
|
+
@on_stopping.call(@on_stopping_completion)
|
142
|
+
when 'log_level'
|
143
|
+
begin
|
144
|
+
level = payload.options.first
|
145
|
+
logger.info { "Setting log level to #{level} for: #{name}" }
|
146
|
+
log_level(level)
|
147
|
+
rescue ArgumentError => e
|
148
|
+
logger.error { "Incorrect log level: #{level}" }
|
149
|
+
end
|
150
|
+
else
|
151
|
+
logger.warn { "Unknown command: #{level} -> #{level.inspect}" }
|
122
152
|
end
|
123
|
-
else
|
124
|
-
logger.warn { "Unknown command: #{level} -> #{level.inspect}" }
|
125
153
|
end
|
126
154
|
end
|
127
155
|
end
|
128
156
|
|
129
157
|
def setup_stats_queue
|
158
|
+
puts "setting stats"
|
130
159
|
# instantiate this queue without using the factory so it doesn't show
|
131
160
|
# up in the stats.
|
132
|
-
|
161
|
+
Messaging::Sender.new(QueueDefinitions::Agent_stats) do |stats_queue|
|
133
162
|
EventMachine.add_periodic_timer(2) do
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
163
|
+
stats_queue.number_of_consumers do |consumers|
|
164
|
+
if consumers > 0
|
165
|
+
payload = ACL::Factory.create(:agent_stats) do |p|
|
166
|
+
p.agent_name = self.name
|
167
|
+
p.pid = self.pid
|
168
|
+
p.rss = (File.read("/proc/#{pid}/statm").split[1].to_i * 4) / 1024 # This assumes the page size is 4K & is MB
|
169
|
+
p.up_time = (Time.now - @start_time).to_i
|
170
|
+
factory.each_queue do |q|
|
171
|
+
p.queues << ACL::Factory.create('agent_stats::queue_stats', :name => q.denormalised_queue_name, :type => q.class.to_s, :length => q.counter)
|
172
|
+
end
|
142
173
|
end
|
143
|
-
end
|
144
174
|
|
145
|
-
|
175
|
+
stats_queue.publish(payload)
|
176
|
+
end
|
146
177
|
end
|
147
|
-
|
148
|
-
# The errback argument is set to nil so as to suppress the default message.
|
149
|
-
stats_queue.consumers?(callback, nil)
|
150
178
|
end
|
151
179
|
end
|
152
180
|
end
|
153
181
|
|
154
|
-
def acknowledge_start
|
155
|
-
|
156
|
-
payload = ACL::
|
182
|
+
def acknowledge_start(&blk)
|
183
|
+
Messaging::Sender.new(QueueDefinitions::Agent_lifecycle) do |ack_start_queue|
|
184
|
+
payload = ACL::Factory.create(:agent_lifecycle) do |p|
|
157
185
|
p.state = 'acknowledge_start'
|
158
186
|
p.pid = $$
|
159
187
|
p.name = self.class.to_s
|
@@ -162,24 +190,26 @@ module Smith
|
|
162
190
|
p.singleton = Smith.config.agent.singleton
|
163
191
|
p.started_at = Time.now.to_i
|
164
192
|
end
|
165
|
-
ack_start_queue.publish(payload)
|
193
|
+
ack_start_queue.publish(payload, &blk)
|
166
194
|
end
|
167
195
|
end
|
168
196
|
|
169
|
-
def acknowledge_stop(&
|
170
|
-
|
197
|
+
def acknowledge_stop(&blk)
|
198
|
+
Messaging::Sender.new(QueueDefinitions::Agent_lifecycle) do |ack_stop_queue|
|
171
199
|
message = {:state => 'acknowledge_stop', :pid => $$, :name => self.class.to_s}
|
172
|
-
ack_stop_queue.publish(ACL::
|
200
|
+
ack_stop_queue.publish(ACL::Factory.create(:agent_lifecycle, message), &blk)
|
173
201
|
end
|
174
202
|
end
|
175
203
|
|
176
204
|
def start_keep_alive
|
177
205
|
if Smith.config.agent.monitor
|
178
206
|
EventMachine::add_periodic_timer(1) do
|
179
|
-
|
207
|
+
Messaging::Sender.new(QueueDefinitions::Agent_keepalive) do |queue|
|
180
208
|
message = {:name => self.class.to_s, :pid => $$, :time => Time.now.to_i}
|
181
|
-
keep_alive_queue.consumers
|
182
|
-
|
209
|
+
keep_alive_queue.consumers do |consumers|
|
210
|
+
if consumers > 0
|
211
|
+
keep_alive_queue.publish(ACL::Factory.create(:agent_keepalive, message))
|
212
|
+
end
|
183
213
|
end
|
184
214
|
end
|
185
215
|
end
|
@@ -36,10 +36,9 @@ module Smith
|
|
36
36
|
logger.info { "Agent is shutting down: #{agent_process.name}" }
|
37
37
|
when 'dead'
|
38
38
|
logger.info { "Restarting dead agent: #{agent_process.name}" }
|
39
|
-
Messaging::Sender.new(
|
40
|
-
sender.
|
41
|
-
|
42
|
-
end
|
39
|
+
Messaging::Sender.new(QueueDefinitions::Agency_control) do |sender|
|
40
|
+
sender.on_reply { |p, r| logger.debug { "Agent restart message acknowledged: #{agent_process.name}" } }
|
41
|
+
sender.publish(ACL::Factory.create(:agency_command).content(:command => 'start', :args => [agent_process.name]))
|
43
42
|
end
|
44
43
|
when 'unknown'
|
45
44
|
logger.info { "Agent is in an unknown state: #{agent_process.name}" }
|