right_agent 0.5.1 → 0.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/right_agent.rb +3 -13
- data/lib/right_agent/actors/agent_manager.rb +78 -4
- data/lib/right_agent/agent.rb +81 -4
- data/lib/right_agent/agent_config.rb +17 -1
- data/lib/right_agent/agent_tags_manager.rb +2 -2
- data/lib/right_agent/broker_client.rb +32 -34
- data/lib/right_agent/command/agent_manager_commands.rb +16 -0
- data/lib/right_agent/command/command_constants.rb +0 -9
- data/lib/right_agent/dispatcher.rb +6 -3
- data/lib/right_agent/ha_broker_client.rb +63 -14
- data/lib/right_agent/log.rb +1 -1
- data/lib/right_agent/minimal.rb +43 -0
- data/lib/right_agent/monkey_patches/amqp_patch.rb +91 -182
- data/lib/right_agent/packets.rb +10 -5
- data/lib/right_agent/platform.rb +8 -0
- data/lib/right_agent/platform/darwin.rb +14 -0
- data/lib/right_agent/platform/linux.rb +23 -0
- data/lib/right_agent/platform/windows.rb +31 -0
- data/lib/right_agent/scripts/agent_controller.rb +16 -8
- data/lib/right_agent/scripts/agent_deployer.rb +6 -0
- data/lib/right_agent/scripts/log_level_manager.rb +4 -5
- data/lib/right_agent/scripts/stats_manager.rb +9 -1
- data/lib/right_agent/sender.rb +623 -371
- data/lib/right_agent/stats_helper.rb +15 -1
- data/lib/right_agent/tracer.rb +1 -1
- data/right_agent.gemspec +14 -15
- data/spec/agent_config_spec.rb +9 -0
- data/spec/agent_spec.rb +154 -18
- data/spec/broker_client_spec.rb +171 -170
- data/spec/dispatcher_spec.rb +24 -8
- data/spec/ha_broker_client_spec.rb +55 -33
- data/spec/monkey_patches/amqp_patch_spec.rb +12 -0
- data/spec/packets_spec.rb +2 -0
- data/spec/sender_spec.rb +140 -69
- data/spec/stats_helper_spec.rb +5 -0
- metadata +54 -53
@@ -34,6 +34,7 @@ module RightScale
|
|
34
34
|
:get_log_level => 'Get log level',
|
35
35
|
:ping => 'Ping agent',
|
36
36
|
:stats => 'Get statistics about agent operation',
|
37
|
+
:profile => 'Manage memory profiling',
|
37
38
|
:terminate => 'Terminate agent'
|
38
39
|
}
|
39
40
|
|
@@ -117,6 +118,21 @@ module RightScale
|
|
117
118
|
CommandIO.instance.reply(opts[:conn], @serializer.dump(@agent_manager.stats({:reset => opts[:reset]})))
|
118
119
|
end
|
119
120
|
|
121
|
+
# Profile command
|
122
|
+
#
|
123
|
+
# === Parameters
|
124
|
+
# opts[:conn](EM::Connection):: Connection used to send reply
|
125
|
+
# opts[:start](Boolean):: Whether to start profiling
|
126
|
+
# opts[:stats](Boolean):: Whether to display profile statistics to stdout
|
127
|
+
# opts[:reset](Boolean):: Whether to reset profile statistics when after displaying them
|
128
|
+
# opts[:stop](Boolean):: Whether to stop profiling
|
129
|
+
#
|
130
|
+
# === Return
|
131
|
+
# true
|
132
|
+
def profile_command(opts)
|
133
|
+
CommandIO.instance.reply(opts[:conn], @serializer.dump(@agent_manager.profile(opts)))
|
134
|
+
end
|
135
|
+
|
120
136
|
# Terminate command
|
121
137
|
#
|
122
138
|
# === Parameters
|
@@ -29,14 +29,5 @@ module RightScale
|
|
29
29
|
BASE_INSTANCE_AGENT_SOCKET_PORT = 60000
|
30
30
|
BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 61000
|
31
31
|
|
32
|
-
BASE_CORE_AGENT_SOCKET_PORT = 70000
|
33
|
-
BASE_LABORER_AGENT_SOCKET_PORT = 71000
|
34
|
-
BASE_REPLICANT_AGENT_SOCKET_PORT = 72000
|
35
|
-
BASE_PROXY_AGENT_SOCKET_PORT = 73000
|
36
|
-
BASE_LIBRARY_AGENT_SOCKET_PORT = 74000
|
37
|
-
BASE_WASABI_AGENT_SOCKET_PORT = 75000
|
38
|
-
|
39
|
-
BASE_MAPPER_SOCKET_PORT = 79000
|
40
|
-
|
41
32
|
end
|
42
33
|
end
|
@@ -27,6 +27,9 @@ module RightScale
|
|
27
27
|
|
28
28
|
include StatsHelper
|
29
29
|
|
30
|
+
# Response queue name
|
31
|
+
RESPONSE_QUEUE = "response"
|
32
|
+
|
30
33
|
# Cache for requests that have been dispatched recently
|
31
34
|
# This cache is intended for use in checking for duplicate requests
|
32
35
|
class Dispatched
|
@@ -50,7 +53,7 @@ module RightScale
|
|
50
53
|
# === Return
|
51
54
|
# true:: Always return true
|
52
55
|
def store(token)
|
53
|
-
now
|
56
|
+
now = Time.now.to_i
|
54
57
|
if @cache.has_key?(token)
|
55
58
|
@cache[token] = now
|
56
59
|
@lru.push(@lru.delete(token))
|
@@ -180,7 +183,7 @@ module RightScale
|
|
180
183
|
OperationResult.non_delivery(OperationResult::TTL_EXPIRATION)
|
181
184
|
end
|
182
185
|
result = Result.new(token, request.reply_to, non_delivery, @identity, request.from, request.tries, request.persistent)
|
183
|
-
exchange = {:type => :queue, :name =>
|
186
|
+
exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
|
184
187
|
@broker.publish(exchange, result, :persistent => true, :mandatory => true)
|
185
188
|
end
|
186
189
|
return nil
|
@@ -222,7 +225,7 @@ module RightScale
|
|
222
225
|
if request.kind_of?(Request)
|
223
226
|
duration = @requests.finish(received_at, token)
|
224
227
|
r = Result.new(token, request.reply_to, r, @identity, request.from, request.tries, request.persistent, duration)
|
225
|
-
exchange = {:type => :queue, :name =>
|
228
|
+
exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
|
226
229
|
@broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
|
227
230
|
end
|
228
231
|
rescue HABrokerClient::NoConnectedBrokers => e
|
@@ -109,6 +109,8 @@ module RightScale
|
|
109
109
|
# :vhost(String):: Virtual host path name
|
110
110
|
# :insist(Boolean):: Whether to suppress redirection of connection
|
111
111
|
# :reconnect_interval(Integer):: Number of seconds between reconnect attempts, defaults to RECONNECT_INTERVAL
|
112
|
+
# :heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
|
113
|
+
# connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
|
112
114
|
# :host{String):: Comma-separated list of AMQP broker host names; if only one, it is reapplied
|
113
115
|
# to successive ports; if none, defaults to localhost; each host may be followed by ':'
|
114
116
|
# and a short string to be used as a broker index; the index defaults to the list index,
|
@@ -425,6 +427,18 @@ module RightScale
|
|
425
427
|
@brokers.inject([]) { |c, b| b.failed? ? c << b.identity : c }
|
426
428
|
end
|
427
429
|
|
430
|
+
# Change connection heartbeat frequency to be used for any new connections
|
431
|
+
#
|
432
|
+
# === Parameters
|
433
|
+
# heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
|
434
|
+
# connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
|
435
|
+
#
|
436
|
+
# === Return
|
437
|
+
# (Integer|nil):: New heartbeat setting
|
438
|
+
def heartbeat=(heartbeat)
|
439
|
+
@options[:heartbeat] = heartbeat
|
440
|
+
end
|
441
|
+
|
428
442
|
# Make new connection to broker at specified address unless already connected
|
429
443
|
# or currently connecting
|
430
444
|
#
|
@@ -454,17 +468,19 @@ module RightScale
|
|
454
468
|
Log.info("Ignored request to reconnect #{identity} because already #{existing.status.to_s}")
|
455
469
|
false
|
456
470
|
else
|
471
|
+
old_identity = identity
|
457
472
|
@brokers.each do |b|
|
458
473
|
if index == b.index && (island.nil? || in_island?(b, island.id))
|
459
|
-
|
460
|
-
|
474
|
+
# Changing host and/or port of existing broker client
|
475
|
+
old_identity = b.identity
|
476
|
+
break
|
461
477
|
end
|
462
478
|
end unless existing
|
463
479
|
|
464
480
|
address = {:host => host, :port => port, :index => index}
|
465
481
|
broker = BrokerClient.new(identity, address, @serializer, @exceptions, @options, island, existing)
|
466
482
|
island_id = island && island.id
|
467
|
-
p, i = priority(
|
483
|
+
p, i = priority(old_identity, island_id)
|
468
484
|
if priority && priority < p
|
469
485
|
@brokers.insert(i + priority, broker)
|
470
486
|
elsif priority && priority > p
|
@@ -505,15 +521,24 @@ module RightScale
|
|
505
521
|
if @brokers_hash[identity]
|
506
522
|
old.delete(identity)
|
507
523
|
else
|
508
|
-
|
524
|
+
begin
|
525
|
+
new << identity if connect(a[:host], a[:port], a[:index], priority, i)
|
526
|
+
rescue Exception => e
|
527
|
+
Log.error("Failed to connect to broker #{identity}", e, :trace)
|
528
|
+
@exceptions.track("connect update", e)
|
529
|
+
end
|
509
530
|
end
|
510
531
|
priority += 1
|
511
532
|
end
|
512
533
|
end
|
513
534
|
|
514
535
|
old.each do |identity|
|
515
|
-
b = @brokers_hash[identity]
|
516
|
-
|
536
|
+
if b = @brokers_hash[identity]
|
537
|
+
remove(b.host, b.port)
|
538
|
+
else
|
539
|
+
Log.error("Could not remove broker #{identity} during connection update because not found, " +
|
540
|
+
"current broker configuration: #{status.inspect}")
|
541
|
+
end
|
517
542
|
end
|
518
543
|
{ :add => new, :remove => old, :home => home }
|
519
544
|
end
|
@@ -558,9 +583,10 @@ module RightScale
|
|
558
583
|
# identities(Array):: Identity of brokers where successfully subscribed
|
559
584
|
def subscribe(queue, exchange = nil, options = {}, &blk)
|
560
585
|
identities = []
|
561
|
-
|
586
|
+
brokers = options.delete(:brokers)
|
587
|
+
each_usable(brokers) { |b| identities << b.identity if b.subscribe(queue, exchange, options, &blk) }
|
562
588
|
Log.info("Could not subscribe to queue #{queue.inspect} on exchange #{exchange.inspect} " +
|
563
|
-
"on brokers #{each_usable(
|
589
|
+
"on brokers #{each_usable(brokers).inspect} when selected #{brokers.inspect} " +
|
564
590
|
"from usable #{usable.inspect}") if identities.empty?
|
565
591
|
identities
|
566
592
|
end
|
@@ -603,9 +629,10 @@ module RightScale
|
|
603
629
|
# identities(Array):: Identity of brokers where successfully declared
|
604
630
|
def declare(type, name, options = {})
|
605
631
|
identities = []
|
606
|
-
|
607
|
-
|
608
|
-
|
632
|
+
brokers = options.delete(:brokers)
|
633
|
+
each_usable(brokers) { |b| identities << b.identity if b.declare(type, name, options) }
|
634
|
+
Log.info("Could not declare #{type.to_s} #{name.inspect} on brokers #{each_usable(brokers).inspect} " +
|
635
|
+
"when selected #{brokers.inspect} from usable #{usable.inspect}") if identities.empty?
|
609
636
|
identities
|
610
637
|
end
|
611
638
|
|
@@ -727,7 +754,24 @@ module RightScale
|
|
727
754
|
def delete(name, options = {})
|
728
755
|
identities = []
|
729
756
|
u = usable
|
730
|
-
|
757
|
+
brokers = options.delete(:brokers)
|
758
|
+
((brokers || u) & u).each { |i| identities << i if (b = @brokers_hash[i]) && b.delete(name, options) }
|
759
|
+
identities
|
760
|
+
end
|
761
|
+
|
762
|
+
# Delete queue resources from AMQP in all usable brokers
|
763
|
+
#
|
764
|
+
# === Parameters
|
765
|
+
# name(String):: Queue name
|
766
|
+
# options(Hash):: Queue declare options plus
|
767
|
+
# :brokers(Array):: Identity of brokers in which queue is to be deleted
|
768
|
+
#
|
769
|
+
# === Return
|
770
|
+
# identities(Array):: Identity of brokers where queue was deleted
|
771
|
+
def delete_amqp_resources(name, options = {})
|
772
|
+
identities = []
|
773
|
+
u = usable
|
774
|
+
((options[:brokers] || u) & u).each { |i| identities << i if (b = @brokers_hash[i]) && b.delete_amqp_resources(:queue, name) }
|
731
775
|
identities
|
732
776
|
end
|
733
777
|
|
@@ -891,12 +935,14 @@ module RightScale
|
|
891
935
|
# "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
|
892
936
|
# "total"(Integer):: Total exceptions for this category
|
893
937
|
# "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
|
938
|
+
# "heartbeat"(Integer|nil):: Number of seconds between AMQP heartbeats, or nil if heartbeat disabled
|
894
939
|
# "returns"(Hash|nil):: Message return activity stats with keys "total", "percent", "last", and "rate"
|
895
940
|
# with percentage breakdown per return reason, or nil if none
|
896
941
|
def stats(reset = false)
|
897
942
|
stats = {
|
898
943
|
"brokers" => @brokers.map { |b| b.stats },
|
899
944
|
"exceptions" => @exceptions.stats,
|
945
|
+
"heartbeat" => @options[:heartbeat],
|
900
946
|
"returns" => @returns.all
|
901
947
|
}
|
902
948
|
reset_stats if reset
|
@@ -1140,8 +1186,8 @@ module RightScale
|
|
1140
1186
|
persistent = options[:persistent]
|
1141
1187
|
mandatory = true
|
1142
1188
|
remaining = (context.brokers - context.failed) & all_connected
|
1143
|
-
Log.info("RETURN reason #{reason} token
|
1144
|
-
"
|
1189
|
+
Log.info("RETURN reason #{reason} token <#{token}> to #{to} from #{context.from} brokers #{context.brokers.inspect} " +
|
1190
|
+
"failed #{context.failed.inspect} remaining #{remaining.inspect} connected #{all_connected.inspect}")
|
1145
1191
|
if remaining.empty?
|
1146
1192
|
if (persistent || one_way) &&
|
1147
1193
|
["ACCESS_REFUSED", "NO_CONSUMERS"].include?(reason) &&
|
@@ -1170,6 +1216,9 @@ module RightScale
|
|
1170
1216
|
"because no message context available for re-routing it to #{to}")
|
1171
1217
|
end
|
1172
1218
|
true
|
1219
|
+
rescue Exception => e
|
1220
|
+
Log.error("Failed to handle #{reason} return from #{identity} for message being routed to #{to}", e, :trace)
|
1221
|
+
@exceptions.track("return", e)
|
1173
1222
|
end
|
1174
1223
|
|
1175
1224
|
# Helper for deferring block execution until specified number of actions have completed
|
data/lib/right_agent/log.rb
CHANGED
@@ -402,7 +402,7 @@ module RightScale
|
|
402
402
|
logger.formatter.datetime_format = "%b %d %H:%M:%S"
|
403
403
|
else
|
404
404
|
$stderr.puts "Logging to syslog" if opts[:print]
|
405
|
-
logger = RightSupport::SystemLogger.new(@program_name || identity || 'RightAgent')
|
405
|
+
logger = RightSupport::Log::SystemLogger.new(@program_name || identity || 'RightAgent')
|
406
406
|
end
|
407
407
|
|
408
408
|
@logger = Multiplexer.new(logger)
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'rubygems'
|
24
|
+
require 'eventmachine'
|
25
|
+
require 'fileutils'
|
26
|
+
|
27
|
+
# load definition for File.normalize_path, etc.
|
28
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'platform'))
|
29
|
+
|
30
|
+
unless defined?(RIGHT_AGENT_BASE_DIR)
|
31
|
+
RIGHT_AGENT_BASE_DIR = File.normalize_path(File.dirname(__FILE__))
|
32
|
+
end
|
33
|
+
|
34
|
+
# require minimal gems needed to create a CommandClient and send a command.
|
35
|
+
#
|
36
|
+
# FIX: agent_controller is currently the only minimal-load use case so these
|
37
|
+
# requires are oriented toward that. any additional use cases may require a
|
38
|
+
# rethink of minimal loading.
|
39
|
+
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_config'))
|
40
|
+
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'command'))
|
41
|
+
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'log'))
|
42
|
+
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'pid_file'))
|
43
|
+
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'serialize', 'serializable'))
|
@@ -20,170 +20,44 @@
|
|
20
20
|
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
21
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
#
|
34
|
-
def recover(requeue = false)
|
35
|
-
@mq.callback{
|
36
|
-
@mq.send Protocol::Basic::Recover.new({ :requeue => requeue })
|
37
|
-
}
|
38
|
-
self
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
# May raise a MQ::Error exception when the frame payload contains a
|
43
|
-
# Protocol::Channel::Close object.
|
44
|
-
#
|
45
|
-
# This usually occurs when a client attempts to perform an illegal
|
46
|
-
# operation. A short, and incomplete, list of potential illegal operations
|
47
|
-
# follows:
|
48
|
-
# * publish a message to a deleted exchange (NOT_FOUND)
|
49
|
-
# * declare an exchange using the reserved 'amq.' naming structure (ACCESS_REFUSED)
|
50
|
-
#
|
51
|
-
def process_frame frame
|
52
|
-
log :received, frame
|
53
|
-
|
54
|
-
case frame
|
55
|
-
when Frame::Header
|
56
|
-
@header = frame.payload
|
57
|
-
@body = ''
|
58
|
-
|
59
|
-
when Frame::Body
|
60
|
-
@body << frame.payload
|
61
|
-
if @body.length >= @header.size
|
62
|
-
if @method.is_a? Protocol::Basic::Return
|
63
|
-
@on_return_message.call @method, @body if @on_return_message
|
64
|
-
else
|
65
|
-
@header.properties.update(@method.arguments)
|
66
|
-
@consumer.receive @header, @body if @consumer
|
67
|
-
end
|
68
|
-
@body = @header = @consumer = @method = nil
|
69
|
-
end
|
70
|
-
|
71
|
-
when Frame::Method
|
72
|
-
case method = frame.payload
|
73
|
-
when Protocol::Channel::OpenOk
|
74
|
-
send Protocol::Access::Request.new(:realm => '/data',
|
75
|
-
:read => true,
|
76
|
-
:write => true,
|
77
|
-
:active => true,
|
78
|
-
:passive => true)
|
79
|
-
|
80
|
-
when Protocol::Access::RequestOk
|
81
|
-
@ticket = method.ticket
|
82
|
-
callback{
|
83
|
-
send Protocol::Channel::Close.new(:reply_code => 200,
|
84
|
-
:reply_text => 'bye',
|
85
|
-
:method_id => 0,
|
86
|
-
:class_id => 0)
|
87
|
-
} if @closing
|
88
|
-
succeed
|
89
|
-
|
90
|
-
when Protocol::Basic::CancelOk
|
91
|
-
if @consumer = consumers[ method.consumer_tag ]
|
92
|
-
@consumer.cancelled
|
93
|
-
else
|
94
|
-
MQ.error "Basic.CancelOk for invalid consumer tag: #{method.consumer_tag}"
|
95
|
-
end
|
96
|
-
|
97
|
-
when Protocol::Queue::DeclareOk
|
98
|
-
queues[ method.queue ].receive_status method
|
99
|
-
|
100
|
-
when Protocol::Basic::Deliver, Protocol::Basic::GetOk
|
101
|
-
@method = method
|
102
|
-
@header = nil
|
103
|
-
@body = ''
|
104
|
-
|
105
|
-
if method.is_a? Protocol::Basic::GetOk
|
106
|
-
@consumer = get_queue{|q| q.shift }
|
107
|
-
MQ.error "No pending Basic.GetOk requests" unless @consumer
|
108
|
-
else
|
109
|
-
@consumer = consumers[ method.consumer_tag ]
|
110
|
-
MQ.error "Basic.Deliver for invalid consumer tag: #{method.consumer_tag}" unless @consumer
|
111
|
-
end
|
112
|
-
|
113
|
-
when Protocol::Basic::GetEmpty
|
114
|
-
if @consumer = get_queue{|q| q.shift }
|
115
|
-
@consumer.receive nil, nil
|
116
|
-
else
|
117
|
-
MQ.error "Basic.GetEmpty for invalid consumer"
|
118
|
-
end
|
119
|
-
|
120
|
-
when Protocol::Basic::Return
|
121
|
-
@method = method
|
122
|
-
@header = nil
|
123
|
-
@body = ''
|
124
|
-
|
125
|
-
when Protocol::Channel::Close
|
126
|
-
raise Error, "#{method.reply_text} in #{Protocol.classes[method.class_id].methods[method.method_id]} on #{@channel}"
|
127
|
-
|
128
|
-
when Protocol::Channel::CloseOk
|
129
|
-
@closing = false
|
130
|
-
conn.callback{ |c|
|
131
|
-
c.channels.delete @channel
|
132
|
-
c.close if c.channels.empty?
|
23
|
+
begin
|
24
|
+
# Clean up AMQP connection when an error is raised after a broker request failure,
|
25
|
+
# otherwise AMQP becomes unusable
|
26
|
+
AMQP.module_eval do
|
27
|
+
def self.start *args, &blk
|
28
|
+
begin
|
29
|
+
EM.run{
|
30
|
+
@conn ||= connect *args
|
31
|
+
@conn.callback(&blk) if blk
|
32
|
+
@conn
|
133
33
|
}
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
@consumer.confirm_subscribe
|
138
|
-
else
|
139
|
-
MQ.error "Basic.ConsumeOk for invalid consumer tag: #{method.consumer_tag}"
|
140
|
-
end
|
34
|
+
rescue Exception => e
|
35
|
+
@conn = nil
|
36
|
+
raise e
|
141
37
|
end
|
142
38
|
end
|
143
39
|
end
|
144
40
|
|
145
|
-
# Provide callback to be activated when a message is returned
|
146
|
-
def return_message(&blk)
|
147
|
-
@on_return_message = blk
|
148
|
-
end
|
149
|
-
|
150
|
-
end
|
151
|
-
|
152
|
-
# monkey patch to the amqp gem that adds :no_declare => true option for new Queue objects.
|
153
|
-
# This allows an instance that has no configuration privileges to enroll without blowing
|
154
|
-
# up the AMQP gem when it tries to subscribe to its queue before it has been created.
|
155
|
-
# Exchange :no_declare support is already in the eventmachine-0.12.10 gem.
|
156
|
-
# temporary until we get this into amqp proper
|
157
|
-
MQ::Queue.class_eval do
|
158
|
-
def initialize mq, name, opts = {}
|
159
|
-
@mq = mq
|
160
|
-
@opts = opts
|
161
|
-
@bindings ||= {}
|
162
|
-
@mq.queues[@name = name] ||= self
|
163
|
-
unless opts[:no_declare]
|
164
|
-
@mq.callback{
|
165
|
-
@mq.send AMQP::Protocol::Queue::Declare.new({ :queue => name,
|
166
|
-
:nowait => true }.merge(opts))
|
167
|
-
}
|
168
|
-
end
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
|
-
begin
|
173
|
-
# Monkey patch AMQP reconnect backoff
|
174
41
|
AMQP::Client.module_eval do
|
42
|
+
# Add callback for connection failure
|
175
43
|
def initialize opts = {}
|
176
44
|
@settings = opts
|
177
45
|
extend AMQP.client
|
178
46
|
|
47
|
+
@_channel_mutex = Mutex.new
|
48
|
+
|
179
49
|
@on_disconnect ||= proc{ @connection_status.call(:failed) if @connection_status }
|
180
50
|
|
181
51
|
timeout @settings[:timeout] if @settings[:timeout]
|
182
52
|
errback{ @on_disconnect.call } unless @reconnecting
|
183
53
|
|
184
|
-
|
54
|
+
# TCP connection "openness"
|
55
|
+
@tcp_connection_established = false
|
56
|
+
# AMQP connection "openness"
|
57
|
+
@connected = false
|
185
58
|
end
|
186
59
|
|
60
|
+
# Add backoff controls to the reconnect algorithm
|
187
61
|
def reconnect(force = false)
|
188
62
|
if @reconnecting and not force
|
189
63
|
# Wait after first reconnect attempt and in between each subsequent attempt
|
@@ -216,53 +90,88 @@ begin
|
|
216
90
|
"#{RightScale::AgentIdentity.new('rs', 'broker', @settings[:port].to_i, @settings[:host].gsub('-', '~')).to_s}")
|
217
91
|
log 'reconnecting'
|
218
92
|
EM.reconnect(@settings[:host], @settings[:port], self)
|
93
|
+
rescue Exception => e
|
94
|
+
RightScale::Log.error("Exception caught during AMQP reconnect", e, :trace)
|
95
|
+
reconnect if @reconnecting
|
219
96
|
end
|
220
|
-
end
|
221
97
|
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
def self.start *args, &blk
|
226
|
-
begin
|
227
|
-
EM.run{
|
228
|
-
@conn ||= connect *args
|
229
|
-
@conn.callback(&blk) if blk
|
230
|
-
@conn
|
231
|
-
}
|
232
|
-
rescue Exception => e
|
233
|
-
@conn = nil
|
234
|
-
raise e
|
235
|
-
end
|
236
|
-
end
|
237
|
-
end
|
238
|
-
|
239
|
-
# This monkey patch catches exceptions that would otherwise cause EM to stop or be in a bad
|
240
|
-
# state if a top level EM error handler was setup. Instead close the connection and leave EM
|
241
|
-
# alone.
|
242
|
-
# Don't log an error if the environment variable IGNORE_AMQP_FAILURES is set (used in the
|
243
|
-
# enroll script)
|
244
|
-
AMQP::Client.module_eval do
|
98
|
+
# Catch exceptions that would otherwise cause EM to stop or be in a bad state if a top
|
99
|
+
# level EM error handler was setup. Instead close the connection and leave EM alone.
|
100
|
+
# Don't log an error if the environment variable IGNORE_AMQP_FAILURES is set
|
245
101
|
alias :orig_receive_data :receive_data
|
246
102
|
def receive_data(*args)
|
247
103
|
begin
|
248
104
|
orig_receive_data(*args)
|
249
105
|
rescue Exception => e
|
250
|
-
|
251
|
-
|
106
|
+
unless ENV['IGNORE_AMQP_FAILURES']
|
107
|
+
RightScale::Log.error("Exception caught while processing AMQP frame, closing connection", e, :trace)
|
108
|
+
end
|
252
109
|
close_connection
|
253
110
|
end
|
254
111
|
end
|
112
|
+
|
113
|
+
# Make it log to RightScale when logging enabled
|
114
|
+
def log(*args)
|
115
|
+
return unless @settings[:logging] or AMQP.logging
|
116
|
+
require 'pp'
|
117
|
+
RightScale::Log.info("AMQP #{args.pretty_inspect.chomp}")
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
AMQP::Channel.class_eval do
|
122
|
+
# Detect message return and make callback
|
123
|
+
def check_content_completion
|
124
|
+
if @body.length >= @header.size
|
125
|
+
if @method.is_a? AMQP::Protocol::Basic::Return
|
126
|
+
@on_return_message.call @method, @body if @on_return_message
|
127
|
+
else
|
128
|
+
@header.properties.update(@method.arguments)
|
129
|
+
@consumer.receive @header, @body if @consumer
|
130
|
+
end
|
131
|
+
@body = @header = @consumer = @method = nil
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# Provide callback to be activated when a message is returned
|
136
|
+
def return_message(&blk)
|
137
|
+
@on_return_message = blk
|
138
|
+
end
|
139
|
+
|
140
|
+
# Apply :no_declare option
|
141
|
+
def validate_parameters_match!(entity, parameters)
|
142
|
+
unless entity.opts == parameters || parameters[:passive] || parameters[:no_declare] || entity.opts[:no_declare]
|
143
|
+
raise AMQP::IncompatibleOptionsError.new(entity.name, entity.opts, parameters)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
# Make it log to RightScale when logging enabled
|
148
|
+
def log(*args)
|
149
|
+
return unless AMQP.logging
|
150
|
+
require 'pp'
|
151
|
+
RightScale::Log.info("AMQP #{args.pretty_inspect.chomp}")
|
152
|
+
end
|
255
153
|
end
|
256
154
|
|
257
|
-
# Add
|
258
|
-
#
|
259
|
-
#
|
260
|
-
#
|
261
|
-
AMQP::
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
@
|
155
|
+
# Add :no_declare => true option for new Queue objects to allow an instance that has
|
156
|
+
# no configuration privileges to enroll without blowing up the AMQP gem when it tries
|
157
|
+
# to subscribe to its queue before it has been created (already supported in gem for
|
158
|
+
# Exchange)
|
159
|
+
AMQP::Queue.class_eval do
|
160
|
+
def initialize(mq, name, opts = {}, &block)
|
161
|
+
@mq = mq
|
162
|
+
@opts = self.class.add_default_options(name, opts, block)
|
163
|
+
@bindings ||= {}
|
164
|
+
@name = name unless name.empty?
|
165
|
+
@status = @opts[:nowait] ? :unknown : :unfinished
|
166
|
+
unless opts[:no_declare]
|
167
|
+
@mq.callback{
|
168
|
+
@mq.send AMQP::Protocol::Queue::Declare.new(@opts)
|
169
|
+
}
|
170
|
+
end
|
171
|
+
|
172
|
+
self.callback = block
|
173
|
+
|
174
|
+
block.call(self) if @opts[:nowait] && block
|
266
175
|
end
|
267
176
|
end
|
268
177
|
|