smith 0.5.7
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/agency +55 -0
- data/bin/smithctl +102 -0
- data/lib/smith.rb +237 -0
- data/lib/smith/acl_compiler.rb +74 -0
- data/lib/smith/agent.rb +207 -0
- data/lib/smith/agent_cache.rb +40 -0
- data/lib/smith/agent_config.rb +22 -0
- data/lib/smith/agent_monitoring.rb +52 -0
- data/lib/smith/agent_process.rb +181 -0
- data/lib/smith/application/agency.rb +126 -0
- data/lib/smith/bootstrap.rb +153 -0
- data/lib/smith/cache.rb +61 -0
- data/lib/smith/command.rb +128 -0
- data/lib/smith/commands/agency/agents.rb +28 -0
- data/lib/smith/commands/agency/common.rb +18 -0
- data/lib/smith/commands/agency/kill.rb +13 -0
- data/lib/smith/commands/agency/list.rb +65 -0
- data/lib/smith/commands/agency/logger.rb +56 -0
- data/lib/smith/commands/agency/metadata.rb +14 -0
- data/lib/smith/commands/agency/restart.rb +39 -0
- data/lib/smith/commands/agency/start.rb +62 -0
- data/lib/smith/commands/agency/state.rb +14 -0
- data/lib/smith/commands/agency/stop.rb +70 -0
- data/lib/smith/commands/agency/version.rb +23 -0
- data/lib/smith/commands/smithctl/cat.rb +70 -0
- data/lib/smith/commands/smithctl/pop.rb +76 -0
- data/lib/smith/commands/smithctl/rm.rb +36 -0
- data/lib/smith/commands/smithctl/smithctl_version.rb +23 -0
- data/lib/smith/commands/smithctl/top.rb +42 -0
- data/lib/smith/commands/template.rb +9 -0
- data/lib/smith/config.rb +32 -0
- data/lib/smith/logger.rb +91 -0
- data/lib/smith/messaging/acl/agency_command.proto +5 -0
- data/lib/smith/messaging/acl/agent_command.proto +5 -0
- data/lib/smith/messaging/acl/agent_config_request.proto +4 -0
- data/lib/smith/messaging/acl/agent_config_update.proto +5 -0
- data/lib/smith/messaging/acl/agent_keepalive.proto +6 -0
- data/lib/smith/messaging/acl/agent_lifecycle.proto +12 -0
- data/lib/smith/messaging/acl/agent_stats.proto +14 -0
- data/lib/smith/messaging/acl/default.rb +51 -0
- data/lib/smith/messaging/acl/search.proto +9 -0
- data/lib/smith/messaging/amqp_options.rb +55 -0
- data/lib/smith/messaging/endpoint.rb +116 -0
- data/lib/smith/messaging/exceptions.rb +7 -0
- data/lib/smith/messaging/payload.rb +102 -0
- data/lib/smith/messaging/queue_factory.rb +67 -0
- data/lib/smith/messaging/receiver.rb +237 -0
- data/lib/smith/messaging/responder.rb +15 -0
- data/lib/smith/messaging/sender.rb +61 -0
- metadata +239 -0
data/bin/agency
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# -*- encoding: utf-8 -*-
|
3
|
+
|
4
|
+
require 'pp'
|
5
|
+
require 'pathname'
|
6
|
+
|
7
|
+
root_path = Pathname.new(__FILE__).dirname.join('..').expand_path
|
8
|
+
$:.unshift(root_path.join('lib'))
|
9
|
+
|
10
|
+
require 'smith'
|
11
|
+
require 'smith/acl_compiler'
|
12
|
+
require 'smith/application/agency'
|
13
|
+
|
14
|
+
module Smith
|
15
|
+
class AgencyRunner
|
16
|
+
|
17
|
+
include Logger
|
18
|
+
|
19
|
+
def initialize(root)
|
20
|
+
|
21
|
+
Smith.compile_acls
|
22
|
+
Smith.load_acls
|
23
|
+
|
24
|
+
@agency = Agency.new(:paths => Smith.agent_paths)
|
25
|
+
|
26
|
+
# Setup signal handlers to clean up.
|
27
|
+
%w{TERM INT QUIT}.each do |sig|
|
28
|
+
trap sig, proc { puts "Shutting down"; @agency.stop }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def run
|
33
|
+
Smith.start do
|
34
|
+
|
35
|
+
# This block is here so the that the shutdown hook added in
|
36
|
+
# Smith.start runs last. Yes I know this is leaky but that's how
|
37
|
+
# it is at the moment.
|
38
|
+
|
39
|
+
Smith.shutdown_hook do
|
40
|
+
# TODO put this as a command line option
|
41
|
+
if false
|
42
|
+
@compiler.clear_cache
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
logger.info { "Starting #{File.basename($0)}" }
|
47
|
+
@agency.setup_queues
|
48
|
+
@agency.start_monitoring
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
agency_runner = Smith::AgencyRunner.new(root_path)
|
55
|
+
agency_runner.run
|
data/bin/smithctl
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# -*- encoding: utf-8 -*-
|
3
|
+
|
4
|
+
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
5
|
+
|
6
|
+
require 'smith'
|
7
|
+
|
8
|
+
module Smith
|
9
|
+
class SmithControl
|
10
|
+
|
11
|
+
include Logger
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
log_level(:info)
|
15
|
+
end
|
16
|
+
|
17
|
+
def send_command(command, args, &blk)
|
18
|
+
begin
|
19
|
+
send("#{Command.command_type(command)}_command", command, args, &blk)
|
20
|
+
rescue Smith::Command::UnkownCommandError => e
|
21
|
+
puts e.message
|
22
|
+
Smith.stop(true)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def smithctl_command(command, args, &blk)
|
29
|
+
Smith.channel.on_error do |ch,channel_close|
|
30
|
+
case channel_close.reply_code
|
31
|
+
when 404
|
32
|
+
puts "No such queue: #{extract_queue(channel_close.reply_text)}"
|
33
|
+
when 406
|
34
|
+
puts "Queue in use: #{extract_queue(channel_close.reply_text)}"
|
35
|
+
else
|
36
|
+
puts channel_close.reply_text
|
37
|
+
puts channel_close.reply_code
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
responder = Messaging::Responder.new
|
42
|
+
responder.callback do |v|
|
43
|
+
puts v if v
|
44
|
+
Smith.stop(true)
|
45
|
+
end
|
46
|
+
|
47
|
+
Command.run(command, args, :responder => responder)
|
48
|
+
end
|
49
|
+
|
50
|
+
def extract_queue(message)
|
51
|
+
match = /.*?'(.*?)'.*$/.match(message) #[1]
|
52
|
+
if match && match[1]
|
53
|
+
match[1].sub(/smith\./, '')
|
54
|
+
else
|
55
|
+
message
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def agency_command(command, args, &blk)
|
60
|
+
Messaging::Sender.new('agency.control', :auto_delete => true, :durable => false, :persistent => true, :strict => true).ready do |sender|
|
61
|
+
|
62
|
+
sender.timeout(8) { puts "Timeout. Is the agency still running"; Smith.stop(true) }
|
63
|
+
|
64
|
+
payload = ACL::Payload.new(:agency_command).content(:command => command, :args => args)
|
65
|
+
|
66
|
+
callback = proc do |sender|
|
67
|
+
sender.publish_and_receive(payload) do |r|
|
68
|
+
blk.call(r.payload)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
errback = proc do puts "Agency not running"
|
73
|
+
Smith.stop(true)
|
74
|
+
end
|
75
|
+
|
76
|
+
sender.consumers?(callback, errback)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
command = ARGV.shift
|
83
|
+
args = ARGV
|
84
|
+
|
85
|
+
command || (puts "usage #{File.basename($0)} <command> opts]"; exit 2)
|
86
|
+
|
87
|
+
trap 'INT', proc { (Smith.running?) ? Smith.stop(true) : exit}
|
88
|
+
|
89
|
+
Smith.load_acls
|
90
|
+
|
91
|
+
control = Smith::SmithControl.new
|
92
|
+
|
93
|
+
Smith.on_error do
|
94
|
+
Smith.stop(true)
|
95
|
+
end
|
96
|
+
|
97
|
+
Smith.start(:quiet => true) do
|
98
|
+
control.send_command(command, args) do |result|
|
99
|
+
puts result if result && !result.empty?
|
100
|
+
Smith.stop(true)
|
101
|
+
end
|
102
|
+
end
|
data/lib/smith.rb
ADDED
@@ -0,0 +1,237 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'amqp'
|
3
|
+
require 'tmpdir'
|
4
|
+
require 'logging'
|
5
|
+
require 'pathname'
|
6
|
+
require 'fileutils'
|
7
|
+
require 'optimism'
|
8
|
+
require 'dm-core'
|
9
|
+
require 'securerandom'
|
10
|
+
require 'dm-yaml-adapter'
|
11
|
+
require 'extlib/string'
|
12
|
+
require 'extlib/inflection'
|
13
|
+
require 'daemons/pidfile'
|
14
|
+
|
15
|
+
require_relative 'smith/config'
|
16
|
+
require_relative 'smith/logger'
|
17
|
+
require_relative 'smith/acl_compiler'
|
18
|
+
|
19
|
+
module Smith
|
20
|
+
include Logger
|
21
|
+
|
22
|
+
class << self
|
23
|
+
|
24
|
+
def channel
|
25
|
+
raise RuntimeError, "You must run this in a Smith.start block" if @channel.nil?
|
26
|
+
@channel
|
27
|
+
end
|
28
|
+
|
29
|
+
def on_error=(handler)
|
30
|
+
@handler = handler
|
31
|
+
end
|
32
|
+
|
33
|
+
def config
|
34
|
+
Config.get
|
35
|
+
end
|
36
|
+
|
37
|
+
def root_path
|
38
|
+
Pathname.new(File.dirname(__FILE__) + '/..').expand_path
|
39
|
+
end
|
40
|
+
|
41
|
+
def agent_paths
|
42
|
+
path_to_pathnames(config.agency.agent_path)
|
43
|
+
end
|
44
|
+
|
45
|
+
def acl_path
|
46
|
+
path_to_pathnames(config.agency.acl_path)
|
47
|
+
end
|
48
|
+
|
49
|
+
# Return the acl cache path. If it's not specified in the config
|
50
|
+
# generate a temporary path.
|
51
|
+
def acl_cache_path
|
52
|
+
@acl_cache_path ||= if Smith.config.agency._has_key?(:acl_cache_path)
|
53
|
+
Pathname.new(Smith.config.agency.acl_cache_path).tap { |path| check_path(path) }
|
54
|
+
else
|
55
|
+
cache_dir = Pathname.new(ENV['HOME']).join('.smith').join('acl')
|
56
|
+
if cache_dir.exist?
|
57
|
+
cache_dir
|
58
|
+
else
|
59
|
+
FileUtils.mkdir_p(cache_dir)
|
60
|
+
cache_dir
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def compile_acls
|
66
|
+
@compiler = ACLCompiler.new
|
67
|
+
@compiler.compile
|
68
|
+
end
|
69
|
+
|
70
|
+
# Load all acls. This fucking horrible but for the time
|
71
|
+
# being it's how it's going to be. This will really start
|
72
|
+
# to be a problem when there are a lot of acls.
|
73
|
+
def load_acls
|
74
|
+
acl_cache_path.each_child do |acl_file|
|
75
|
+
logger.verbose { "Loading acl file: #{acl_file}" }
|
76
|
+
require acl_file
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def running?
|
81
|
+
EM.reactor_running?
|
82
|
+
end
|
83
|
+
|
84
|
+
def on_error(&blk)
|
85
|
+
@on_error = blk
|
86
|
+
end
|
87
|
+
|
88
|
+
def start(opts={}, &block)
|
89
|
+
EM.epoll if EM.epoll?
|
90
|
+
EM.kqueue if EM.kqueue?
|
91
|
+
EM.set_descriptor_table_size(opts[:fdsize] || 1024)
|
92
|
+
|
93
|
+
connection_settings = config.amqp.broker._child.merge({
|
94
|
+
:on_tcp_connection_failure => method(:tcp_connection_failure_handler),
|
95
|
+
:on_possible_authentication_failure => method(:authentication_failure_handler)
|
96
|
+
})
|
97
|
+
|
98
|
+
AMQP.start(connection_settings) do |connection|
|
99
|
+
@connection = connection
|
100
|
+
|
101
|
+
connection.on_connection do
|
102
|
+
broker = connection.broker.properties
|
103
|
+
endpoint = connection.broker_endpoint
|
104
|
+
logger.debug { "Connected to: AMQP Broker: #{endpoint}, (#{broker['product']}/v#{broker['version']})" } unless opts[:quiet]
|
105
|
+
end
|
106
|
+
|
107
|
+
connection.on_tcp_connection_loss do |connection, settings|
|
108
|
+
EM.next_tick do
|
109
|
+
logger.error { "TCP connection error. Attempting restart" }
|
110
|
+
connection.reconnect
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
connection.after_recovery do
|
115
|
+
logger.info { "Connection to AMQP server restored" }
|
116
|
+
end
|
117
|
+
|
118
|
+
connection.on_error do |connection, reason|
|
119
|
+
case reason.reply_code
|
120
|
+
when 320
|
121
|
+
logger.warn { "AMQP server shutdown. Waiting." }
|
122
|
+
else
|
123
|
+
if @handler
|
124
|
+
@handler.call(connection, reason)
|
125
|
+
else
|
126
|
+
logger.error { "AMQP Server error: #{reason.reply_code}: #{reason.reply_text}" }
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
# This will be the last thing run by the reactor.
|
132
|
+
shutdown_hook { logger.debug { "Reactor Stopped" } }
|
133
|
+
|
134
|
+
AMQP::Channel.new(connection) do |channel,ok|
|
135
|
+
@channel = channel
|
136
|
+
# Set up QOS. If you do not do this then the subscribe in receive_message
|
137
|
+
# will get overwelmd and the whole thing will collapse in on itself.
|
138
|
+
channel.prefetch(1)
|
139
|
+
|
140
|
+
if @on_error
|
141
|
+
channel.on_error(&@on_error)
|
142
|
+
else
|
143
|
+
# Log the error and stop the agency when there are channel errors.
|
144
|
+
# TODO Add recovery instead of stopping the agency.
|
145
|
+
channel.on_error do |ch,channel_close|
|
146
|
+
logger.fatal { "Channel level exception: #{channel_close.reply_text}. Class id: #{channel_close.class_id}, Method id: #{channel_close.method_id}, Status code : #{channel_close.reply_code}" }
|
147
|
+
logger.fatal { "Agency is exiting" }
|
148
|
+
Smith.stop(true)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
# Set up auto-recovery. This will ensure that the AMQP gem reconnects each
|
153
|
+
# channel and sets up the various exchanges & queues.
|
154
|
+
channel.auto_recovery = true
|
155
|
+
|
156
|
+
block.call
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def shutdown_hook(&block)
|
162
|
+
EM.add_shutdown_hook(&block)
|
163
|
+
end
|
164
|
+
|
165
|
+
def stop(immediately=false, &blk)
|
166
|
+
shutdown_hook(&blk) if blk
|
167
|
+
|
168
|
+
if running?
|
169
|
+
if immediately
|
170
|
+
@connection.close { EM.stop_event_loop }
|
171
|
+
else
|
172
|
+
EM.add_timer(1) do
|
173
|
+
@connection.close { EM.stop_event_loop }
|
174
|
+
end
|
175
|
+
end
|
176
|
+
else
|
177
|
+
logger.fatal { "Eventmachine is not running, exiting with prejudice" }
|
178
|
+
exit!
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
private
|
183
|
+
|
184
|
+
def tcp_connection_failure_handler(settings)
|
185
|
+
# Only display the following settings.
|
186
|
+
s = settings.select { |k,v| ([:user, :pass, :vhost, :host, :port, :ssl].include?(k)) }
|
187
|
+
|
188
|
+
logger.fatal { "Cannot connect to the AMQP server." }
|
189
|
+
logger.fatal { "Is the server running and are the connection details correct?" }
|
190
|
+
logger.info { "Details:" }
|
191
|
+
s.each do |k,v|
|
192
|
+
logger.info { " Setting: %-7s%s" % [k, v] }
|
193
|
+
end
|
194
|
+
EM.stop
|
195
|
+
end
|
196
|
+
|
197
|
+
def authentication_failure_handler(settings)
|
198
|
+
# Only display the following settings.
|
199
|
+
s = settings.select { |k,v| [:user, :pass, :vhost, :host].include?(k) }
|
200
|
+
|
201
|
+
logger.fatal { "Authenticaton failure." }
|
202
|
+
logger.info { "Details:" }
|
203
|
+
s.each do |k,v|
|
204
|
+
logger.info { " Setting: %-7s%s" % [k, v] }
|
205
|
+
end
|
206
|
+
EM.stop
|
207
|
+
end
|
208
|
+
|
209
|
+
def path_to_pathnames(path)
|
210
|
+
path ||= []
|
211
|
+
path.split(':').map do |path|
|
212
|
+
p = Pathname.new(path)
|
213
|
+
((p.absolute?) ? p : root_path.join(p)).tap { |path| check_path(path) }
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def check_path(path)
|
218
|
+
logger.error("Path does not exist: #{path}") unless path.exist?
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
require_relative 'smith/cache'
|
224
|
+
require_relative 'smith/agent'
|
225
|
+
require_relative 'smith/agent_cache'
|
226
|
+
require_relative 'smith/agent_process'
|
227
|
+
require_relative 'smith/agent_monitoring'
|
228
|
+
require_relative 'smith/command'
|
229
|
+
require_relative 'smith/messaging/amqp_options'
|
230
|
+
require_relative 'smith/messaging/queue_factory'
|
231
|
+
require_relative 'smith/messaging/payload'
|
232
|
+
require_relative 'smith/messaging/acl/default'
|
233
|
+
require_relative 'smith/messaging/endpoint'
|
234
|
+
require_relative 'smith/messaging/exceptions'
|
235
|
+
require_relative 'smith/messaging/responder'
|
236
|
+
require_relative 'smith/messaging/receiver'
|
237
|
+
require_relative 'smith/messaging/sender'
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'pp'
|
3
|
+
require 'protobuf/compiler/compiler'
|
4
|
+
|
5
|
+
module Smith
|
6
|
+
class ACLCompiler
|
7
|
+
|
8
|
+
include Logger
|
9
|
+
|
10
|
+
def initialize(force=false)
|
11
|
+
# TODO Add the force code.
|
12
|
+
@cache_path = Smith.acl_cache_path
|
13
|
+
end
|
14
|
+
|
15
|
+
# Compile any protocol buffer files. This checks the timestamp
|
16
|
+
# to see if the file needs compiling.
|
17
|
+
def compile
|
18
|
+
logger.debug { "Protocol buffer cache path: #{@cache_path}" }
|
19
|
+
Smith.acl_path.each do |path|
|
20
|
+
results = {}
|
21
|
+
path_glob(path) do |p|
|
22
|
+
if should_compile?(p)
|
23
|
+
logger.info { "Compiling: #{p}" }
|
24
|
+
# TODO put some error handling here.
|
25
|
+
Protobuf::Compiler.compile(p.basename, p.dirname, @cache_path)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
@cache_path
|
30
|
+
end
|
31
|
+
|
32
|
+
def cache_path
|
33
|
+
@cache_path.to_s
|
34
|
+
end
|
35
|
+
|
36
|
+
# Clears the Protocol Buffer cache. If acl_cache_path is
|
37
|
+
# specified in the config then the directory itself won't be removed
|
38
|
+
# but if it's not specified and a temporary directory was created then
|
39
|
+
# the directory is removed as well.
|
40
|
+
def clear_cache
|
41
|
+
logger.info { "Clearing the Protocol Buffer cache: #{Smith.acl_cache_path}" }
|
42
|
+
|
43
|
+
Pathname.glob(@cache_path.join("*")).each do |path|
|
44
|
+
path.unlink
|
45
|
+
end
|
46
|
+
|
47
|
+
unless Smith.config.agency._has_key?(:acl_cache_path)
|
48
|
+
@cache_path.rmdir
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
# Returns true if the .proto file is newer that the .pb.rb file
|
55
|
+
def should_compile?(file)
|
56
|
+
cached_file = @cache_path.join(file.basename).sub_ext(".pb.rb")
|
57
|
+
if cached_file.exist?
|
58
|
+
if file.mtime > cached_file.mtime
|
59
|
+
true
|
60
|
+
else
|
61
|
+
false
|
62
|
+
end
|
63
|
+
else
|
64
|
+
true
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def path_glob(path)
|
69
|
+
Pathname.glob("#{path.join("*.proto")}").map do |acl|
|
70
|
+
yield acl.realpath
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|