right_agent 0.5.1 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. data/lib/right_agent.rb +3 -13
  2. data/lib/right_agent/actors/agent_manager.rb +78 -4
  3. data/lib/right_agent/agent.rb +81 -4
  4. data/lib/right_agent/agent_config.rb +17 -1
  5. data/lib/right_agent/agent_tags_manager.rb +2 -2
  6. data/lib/right_agent/broker_client.rb +32 -34
  7. data/lib/right_agent/command/agent_manager_commands.rb +16 -0
  8. data/lib/right_agent/command/command_constants.rb +0 -9
  9. data/lib/right_agent/dispatcher.rb +6 -3
  10. data/lib/right_agent/ha_broker_client.rb +63 -14
  11. data/lib/right_agent/log.rb +1 -1
  12. data/lib/right_agent/minimal.rb +43 -0
  13. data/lib/right_agent/monkey_patches/amqp_patch.rb +91 -182
  14. data/lib/right_agent/packets.rb +10 -5
  15. data/lib/right_agent/platform.rb +8 -0
  16. data/lib/right_agent/platform/darwin.rb +14 -0
  17. data/lib/right_agent/platform/linux.rb +23 -0
  18. data/lib/right_agent/platform/windows.rb +31 -0
  19. data/lib/right_agent/scripts/agent_controller.rb +16 -8
  20. data/lib/right_agent/scripts/agent_deployer.rb +6 -0
  21. data/lib/right_agent/scripts/log_level_manager.rb +4 -5
  22. data/lib/right_agent/scripts/stats_manager.rb +9 -1
  23. data/lib/right_agent/sender.rb +623 -371
  24. data/lib/right_agent/stats_helper.rb +15 -1
  25. data/lib/right_agent/tracer.rb +1 -1
  26. data/right_agent.gemspec +14 -15
  27. data/spec/agent_config_spec.rb +9 -0
  28. data/spec/agent_spec.rb +154 -18
  29. data/spec/broker_client_spec.rb +171 -170
  30. data/spec/dispatcher_spec.rb +24 -8
  31. data/spec/ha_broker_client_spec.rb +55 -33
  32. data/spec/monkey_patches/amqp_patch_spec.rb +12 -0
  33. data/spec/packets_spec.rb +2 -0
  34. data/spec/sender_spec.rb +140 -69
  35. data/spec/stats_helper_spec.rb +5 -0
  36. metadata +54 -53
@@ -34,6 +34,7 @@ module RightScale
34
34
  :get_log_level => 'Get log level',
35
35
  :ping => 'Ping agent',
36
36
  :stats => 'Get statistics about agent operation',
37
+ :profile => 'Manage memory profiling',
37
38
  :terminate => 'Terminate agent'
38
39
  }
39
40
 
@@ -117,6 +118,21 @@ module RightScale
117
118
  CommandIO.instance.reply(opts[:conn], @serializer.dump(@agent_manager.stats({:reset => opts[:reset]})))
118
119
  end
119
120
 
121
+ # Profile command
122
+ #
123
+ # === Parameters
124
+ # opts[:conn](EM::Connection):: Connection used to send reply
125
+ # opts[:start](Boolean):: Whether to start profiling
126
+ # opts[:stats](Boolean):: Whether to display profile statistics to stdout
127
+ # opts[:reset](Boolean):: Whether to reset profile statistics when after displaying them
128
+ # opts[:stop](Boolean):: Whether to stop profiling
129
+ #
130
+ # === Return
131
+ # true
132
+ def profile_command(opts)
133
+ CommandIO.instance.reply(opts[:conn], @serializer.dump(@agent_manager.profile(opts)))
134
+ end
135
+
120
136
  # Terminate command
121
137
  #
122
138
  # === Parameters
@@ -29,14 +29,5 @@ module RightScale
29
29
  BASE_INSTANCE_AGENT_SOCKET_PORT = 60000
30
30
  BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 61000
31
31
 
32
- BASE_CORE_AGENT_SOCKET_PORT = 70000
33
- BASE_LABORER_AGENT_SOCKET_PORT = 71000
34
- BASE_REPLICANT_AGENT_SOCKET_PORT = 72000
35
- BASE_PROXY_AGENT_SOCKET_PORT = 73000
36
- BASE_LIBRARY_AGENT_SOCKET_PORT = 74000
37
- BASE_WASABI_AGENT_SOCKET_PORT = 75000
38
-
39
- BASE_MAPPER_SOCKET_PORT = 79000
40
-
41
32
  end
42
33
  end
@@ -27,6 +27,9 @@ module RightScale
27
27
 
28
28
  include StatsHelper
29
29
 
30
+ # Response queue name
31
+ RESPONSE_QUEUE = "response"
32
+
30
33
  # Cache for requests that have been dispatched recently
31
34
  # This cache is intended for use in checking for duplicate requests
32
35
  class Dispatched
@@ -50,7 +53,7 @@ module RightScale
50
53
  # === Return
51
54
  # true:: Always return true
52
55
  def store(token)
53
- now ||= Time.now.to_i
56
+ now = Time.now.to_i
54
57
  if @cache.has_key?(token)
55
58
  @cache[token] = now
56
59
  @lru.push(@lru.delete(token))
@@ -180,7 +183,7 @@ module RightScale
180
183
  OperationResult.non_delivery(OperationResult::TTL_EXPIRATION)
181
184
  end
182
185
  result = Result.new(token, request.reply_to, non_delivery, @identity, request.from, request.tries, request.persistent)
183
- exchange = {:type => :queue, :name => request.reply_to, :options => {:durable => true, :no_declare => @secure}}
186
+ exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
184
187
  @broker.publish(exchange, result, :persistent => true, :mandatory => true)
185
188
  end
186
189
  return nil
@@ -222,7 +225,7 @@ module RightScale
222
225
  if request.kind_of?(Request)
223
226
  duration = @requests.finish(received_at, token)
224
227
  r = Result.new(token, request.reply_to, r, @identity, request.from, request.tries, request.persistent, duration)
225
- exchange = {:type => :queue, :name => request.reply_to, :options => {:durable => true, :no_declare => @secure}}
228
+ exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
226
229
  @broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
227
230
  end
228
231
  rescue HABrokerClient::NoConnectedBrokers => e
@@ -109,6 +109,8 @@ module RightScale
109
109
  # :vhost(String):: Virtual host path name
110
110
  # :insist(Boolean):: Whether to suppress redirection of connection
111
111
  # :reconnect_interval(Integer):: Number of seconds between reconnect attempts, defaults to RECONNECT_INTERVAL
112
+ # :heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
113
+ # connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
112
114
  # :host{String):: Comma-separated list of AMQP broker host names; if only one, it is reapplied
113
115
  # to successive ports; if none, defaults to localhost; each host may be followed by ':'
114
116
  # and a short string to be used as a broker index; the index defaults to the list index,
@@ -425,6 +427,18 @@ module RightScale
425
427
  @brokers.inject([]) { |c, b| b.failed? ? c << b.identity : c }
426
428
  end
427
429
 
430
+ # Change connection heartbeat frequency to be used for any new connections
431
+ #
432
+ # === Parameters
433
+ # heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
434
+ # connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
435
+ #
436
+ # === Return
437
+ # (Integer|nil):: New heartbeat setting
438
+ def heartbeat=(heartbeat)
439
+ @options[:heartbeat] = heartbeat
440
+ end
441
+
428
442
  # Make new connection to broker at specified address unless already connected
429
443
  # or currently connecting
430
444
  #
@@ -454,17 +468,19 @@ module RightScale
454
468
  Log.info("Ignored request to reconnect #{identity} because already #{existing.status.to_s}")
455
469
  false
456
470
  else
471
+ old_identity = identity
457
472
  @brokers.each do |b|
458
473
  if index == b.index && (island.nil? || in_island?(b, island.id))
459
- raise Exception, "Not allowed to change host or port of existing broker #{identity}, " +
460
- "alias #{b.alias}, to #{host} and #{port.inspect}"
474
+ # Changing host and/or port of existing broker client
475
+ old_identity = b.identity
476
+ break
461
477
  end
462
478
  end unless existing
463
479
 
464
480
  address = {:host => host, :port => port, :index => index}
465
481
  broker = BrokerClient.new(identity, address, @serializer, @exceptions, @options, island, existing)
466
482
  island_id = island && island.id
467
- p, i = priority(identity, island_id)
483
+ p, i = priority(old_identity, island_id)
468
484
  if priority && priority < p
469
485
  @brokers.insert(i + priority, broker)
470
486
  elsif priority && priority > p
@@ -505,15 +521,24 @@ module RightScale
505
521
  if @brokers_hash[identity]
506
522
  old.delete(identity)
507
523
  else
508
- new << identity if connect(a[:host], a[:port], a[:index], priority, i)
524
+ begin
525
+ new << identity if connect(a[:host], a[:port], a[:index], priority, i)
526
+ rescue Exception => e
527
+ Log.error("Failed to connect to broker #{identity}", e, :trace)
528
+ @exceptions.track("connect update", e)
529
+ end
509
530
  end
510
531
  priority += 1
511
532
  end
512
533
  end
513
534
 
514
535
  old.each do |identity|
515
- b = @brokers_hash[identity]
516
- remove(b.host, b.port)
536
+ if b = @brokers_hash[identity]
537
+ remove(b.host, b.port)
538
+ else
539
+ Log.error("Could not remove broker #{identity} during connection update because not found, " +
540
+ "current broker configuration: #{status.inspect}")
541
+ end
517
542
  end
518
543
  { :add => new, :remove => old, :home => home }
519
544
  end
@@ -558,9 +583,10 @@ module RightScale
558
583
  # identities(Array):: Identity of brokers where successfully subscribed
559
584
  def subscribe(queue, exchange = nil, options = {}, &blk)
560
585
  identities = []
561
- each_usable(options[:brokers]) { |b| identities << b.identity if b.subscribe(queue, exchange, options, &blk) }
586
+ brokers = options.delete(:brokers)
587
+ each_usable(brokers) { |b| identities << b.identity if b.subscribe(queue, exchange, options, &blk) }
562
588
  Log.info("Could not subscribe to queue #{queue.inspect} on exchange #{exchange.inspect} " +
563
- "on brokers #{each_usable(options[:brokers]).inspect} when selected #{options[:brokers].inspect} " +
589
+ "on brokers #{each_usable(brokers).inspect} when selected #{brokers.inspect} " +
564
590
  "from usable #{usable.inspect}") if identities.empty?
565
591
  identities
566
592
  end
@@ -603,9 +629,10 @@ module RightScale
603
629
  # identities(Array):: Identity of brokers where successfully declared
604
630
  def declare(type, name, options = {})
605
631
  identities = []
606
- each_usable(options[:brokers]) { |b| identities << b.identity if b.declare(type, name, options) }
607
- Log.info("Could not declare #{type.to_s} #{name.inspect} on brokers #{each_usable(options[:brokers]).inspect} " +
608
- "when selected #{options[:brokers].inspect} from usable #{usable.inspect}") if identities.empty?
632
+ brokers = options.delete(:brokers)
633
+ each_usable(brokers) { |b| identities << b.identity if b.declare(type, name, options) }
634
+ Log.info("Could not declare #{type.to_s} #{name.inspect} on brokers #{each_usable(brokers).inspect} " +
635
+ "when selected #{brokers.inspect} from usable #{usable.inspect}") if identities.empty?
609
636
  identities
610
637
  end
611
638
 
@@ -727,7 +754,24 @@ module RightScale
727
754
  def delete(name, options = {})
728
755
  identities = []
729
756
  u = usable
730
- ((options[:brokers] || u) & u).each { |i| identities << i if (b = @brokers_hash[i]) && b.delete(name, options) }
757
+ brokers = options.delete(:brokers)
758
+ ((brokers || u) & u).each { |i| identities << i if (b = @brokers_hash[i]) && b.delete(name, options) }
759
+ identities
760
+ end
761
+
762
+ # Delete queue resources from AMQP in all usable brokers
763
+ #
764
+ # === Parameters
765
+ # name(String):: Queue name
766
+ # options(Hash):: Queue declare options plus
767
+ # :brokers(Array):: Identity of brokers in which queue is to be deleted
768
+ #
769
+ # === Return
770
+ # identities(Array):: Identity of brokers where queue was deleted
771
+ def delete_amqp_resources(name, options = {})
772
+ identities = []
773
+ u = usable
774
+ ((options[:brokers] || u) & u).each { |i| identities << i if (b = @brokers_hash[i]) && b.delete_amqp_resources(:queue, name) }
731
775
  identities
732
776
  end
733
777
 
@@ -891,12 +935,14 @@ module RightScale
891
935
  # "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
892
936
  # "total"(Integer):: Total exceptions for this category
893
937
  # "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
938
+ # "heartbeat"(Integer|nil):: Number of seconds between AMQP heartbeats, or nil if heartbeat disabled
894
939
  # "returns"(Hash|nil):: Message return activity stats with keys "total", "percent", "last", and "rate"
895
940
  # with percentage breakdown per return reason, or nil if none
896
941
  def stats(reset = false)
897
942
  stats = {
898
943
  "brokers" => @brokers.map { |b| b.stats },
899
944
  "exceptions" => @exceptions.stats,
945
+ "heartbeat" => @options[:heartbeat],
900
946
  "returns" => @returns.all
901
947
  }
902
948
  reset_stats if reset
@@ -1140,8 +1186,8 @@ module RightScale
1140
1186
  persistent = options[:persistent]
1141
1187
  mandatory = true
1142
1188
  remaining = (context.brokers - context.failed) & all_connected
1143
- Log.info("RETURN reason #{reason} token #{token} brokers #{context.brokers.inspect} failed #{context.failed.inspect} " +
1144
- " connected #{all_connected.inspect} remaining #{remaining.inspect}")
1189
+ Log.info("RETURN reason #{reason} token <#{token}> to #{to} from #{context.from} brokers #{context.brokers.inspect} " +
1190
+ "failed #{context.failed.inspect} remaining #{remaining.inspect} connected #{all_connected.inspect}")
1145
1191
  if remaining.empty?
1146
1192
  if (persistent || one_way) &&
1147
1193
  ["ACCESS_REFUSED", "NO_CONSUMERS"].include?(reason) &&
@@ -1170,6 +1216,9 @@ module RightScale
1170
1216
  "because no message context available for re-routing it to #{to}")
1171
1217
  end
1172
1218
  true
1219
+ rescue Exception => e
1220
+ Log.error("Failed to handle #{reason} return from #{identity} for message being routed to #{to}", e, :trace)
1221
+ @exceptions.track("return", e)
1173
1222
  end
1174
1223
 
1175
1224
  # Helper for deferring block execution until specified number of actions have completed
@@ -402,7 +402,7 @@ module RightScale
402
402
  logger.formatter.datetime_format = "%b %d %H:%M:%S"
403
403
  else
404
404
  $stderr.puts "Logging to syslog" if opts[:print]
405
- logger = RightSupport::SystemLogger.new(@program_name || identity || 'RightAgent')
405
+ logger = RightSupport::Log::SystemLogger.new(@program_name || identity || 'RightAgent')
406
406
  end
407
407
 
408
408
  @logger = Multiplexer.new(logger)
@@ -0,0 +1,43 @@
1
+ #
2
+ # Copyright (c) 2011 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+
23
+ require 'rubygems'
24
+ require 'eventmachine'
25
+ require 'fileutils'
26
+
27
+ # load definition for File.normalize_path, etc.
28
+ require File.expand_path(File.join(File.dirname(__FILE__), 'platform'))
29
+
30
+ unless defined?(RIGHT_AGENT_BASE_DIR)
31
+ RIGHT_AGENT_BASE_DIR = File.normalize_path(File.dirname(__FILE__))
32
+ end
33
+
34
+ # require minimal gems needed to create a CommandClient and send a command.
35
+ #
36
+ # FIX: agent_controller is currently the only minimal-load use case so these
37
+ # requires are oriented toward that. any additional use cases may require a
38
+ # rethink of minimal loading.
39
+ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_config'))
40
+ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'command'))
41
+ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'log'))
42
+ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'pid_file'))
43
+ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'serialize', 'serializable'))
@@ -20,170 +20,44 @@
20
20
  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
 
23
- class MQ
24
-
25
- class Queue
26
- # Asks the broker to redeliver all unacknowledged messages on a
27
- # specified channel. Zero or more messages may be redelivered.
28
- #
29
- # * requeue (default false)
30
- # If this parameter is false, the message will be redelivered to the original recipient.
31
- # If this flag is true, the server will attempt to requeue the message, potentially then
32
- # delivering it to an alternative subscriber.
33
- #
34
- def recover(requeue = false)
35
- @mq.callback{
36
- @mq.send Protocol::Basic::Recover.new({ :requeue => requeue })
37
- }
38
- self
39
- end
40
- end
41
-
42
- # May raise a MQ::Error exception when the frame payload contains a
43
- # Protocol::Channel::Close object.
44
- #
45
- # This usually occurs when a client attempts to perform an illegal
46
- # operation. A short, and incomplete, list of potential illegal operations
47
- # follows:
48
- # * publish a message to a deleted exchange (NOT_FOUND)
49
- # * declare an exchange using the reserved 'amq.' naming structure (ACCESS_REFUSED)
50
- #
51
- def process_frame frame
52
- log :received, frame
53
-
54
- case frame
55
- when Frame::Header
56
- @header = frame.payload
57
- @body = ''
58
-
59
- when Frame::Body
60
- @body << frame.payload
61
- if @body.length >= @header.size
62
- if @method.is_a? Protocol::Basic::Return
63
- @on_return_message.call @method, @body if @on_return_message
64
- else
65
- @header.properties.update(@method.arguments)
66
- @consumer.receive @header, @body if @consumer
67
- end
68
- @body = @header = @consumer = @method = nil
69
- end
70
-
71
- when Frame::Method
72
- case method = frame.payload
73
- when Protocol::Channel::OpenOk
74
- send Protocol::Access::Request.new(:realm => '/data',
75
- :read => true,
76
- :write => true,
77
- :active => true,
78
- :passive => true)
79
-
80
- when Protocol::Access::RequestOk
81
- @ticket = method.ticket
82
- callback{
83
- send Protocol::Channel::Close.new(:reply_code => 200,
84
- :reply_text => 'bye',
85
- :method_id => 0,
86
- :class_id => 0)
87
- } if @closing
88
- succeed
89
-
90
- when Protocol::Basic::CancelOk
91
- if @consumer = consumers[ method.consumer_tag ]
92
- @consumer.cancelled
93
- else
94
- MQ.error "Basic.CancelOk for invalid consumer tag: #{method.consumer_tag}"
95
- end
96
-
97
- when Protocol::Queue::DeclareOk
98
- queues[ method.queue ].receive_status method
99
-
100
- when Protocol::Basic::Deliver, Protocol::Basic::GetOk
101
- @method = method
102
- @header = nil
103
- @body = ''
104
-
105
- if method.is_a? Protocol::Basic::GetOk
106
- @consumer = get_queue{|q| q.shift }
107
- MQ.error "No pending Basic.GetOk requests" unless @consumer
108
- else
109
- @consumer = consumers[ method.consumer_tag ]
110
- MQ.error "Basic.Deliver for invalid consumer tag: #{method.consumer_tag}" unless @consumer
111
- end
112
-
113
- when Protocol::Basic::GetEmpty
114
- if @consumer = get_queue{|q| q.shift }
115
- @consumer.receive nil, nil
116
- else
117
- MQ.error "Basic.GetEmpty for invalid consumer"
118
- end
119
-
120
- when Protocol::Basic::Return
121
- @method = method
122
- @header = nil
123
- @body = ''
124
-
125
- when Protocol::Channel::Close
126
- raise Error, "#{method.reply_text} in #{Protocol.classes[method.class_id].methods[method.method_id]} on #{@channel}"
127
-
128
- when Protocol::Channel::CloseOk
129
- @closing = false
130
- conn.callback{ |c|
131
- c.channels.delete @channel
132
- c.close if c.channels.empty?
23
+ begin
24
+ # Clean up AMQP connection when an error is raised after a broker request failure,
25
+ # otherwise AMQP becomes unusable
26
+ AMQP.module_eval do
27
+ def self.start *args, &blk
28
+ begin
29
+ EM.run{
30
+ @conn ||= connect *args
31
+ @conn.callback(&blk) if blk
32
+ @conn
133
33
  }
134
-
135
- when Protocol::Basic::ConsumeOk
136
- if @consumer = consumers[ method.consumer_tag ]
137
- @consumer.confirm_subscribe
138
- else
139
- MQ.error "Basic.ConsumeOk for invalid consumer tag: #{method.consumer_tag}"
140
- end
34
+ rescue Exception => e
35
+ @conn = nil
36
+ raise e
141
37
  end
142
38
  end
143
39
  end
144
40
 
145
- # Provide callback to be activated when a message is returned
146
- def return_message(&blk)
147
- @on_return_message = blk
148
- end
149
-
150
- end
151
-
152
- # monkey patch to the amqp gem that adds :no_declare => true option for new Queue objects.
153
- # This allows an instance that has no configuration privileges to enroll without blowing
154
- # up the AMQP gem when it tries to subscribe to its queue before it has been created.
155
- # Exchange :no_declare support is already in the eventmachine-0.12.10 gem.
156
- # temporary until we get this into amqp proper
157
- MQ::Queue.class_eval do
158
- def initialize mq, name, opts = {}
159
- @mq = mq
160
- @opts = opts
161
- @bindings ||= {}
162
- @mq.queues[@name = name] ||= self
163
- unless opts[:no_declare]
164
- @mq.callback{
165
- @mq.send AMQP::Protocol::Queue::Declare.new({ :queue => name,
166
- :nowait => true }.merge(opts))
167
- }
168
- end
169
- end
170
- end
171
-
172
- begin
173
- # Monkey patch AMQP reconnect backoff
174
41
  AMQP::Client.module_eval do
42
+ # Add callback for connection failure
175
43
  def initialize opts = {}
176
44
  @settings = opts
177
45
  extend AMQP.client
178
46
 
47
+ @_channel_mutex = Mutex.new
48
+
179
49
  @on_disconnect ||= proc{ @connection_status.call(:failed) if @connection_status }
180
50
 
181
51
  timeout @settings[:timeout] if @settings[:timeout]
182
52
  errback{ @on_disconnect.call } unless @reconnecting
183
53
 
184
- @connected = false
54
+ # TCP connection "openness"
55
+ @tcp_connection_established = false
56
+ # AMQP connection "openness"
57
+ @connected = false
185
58
  end
186
59
 
60
+ # Add backoff controls to the reconnect algorithm
187
61
  def reconnect(force = false)
188
62
  if @reconnecting and not force
189
63
  # Wait after first reconnect attempt and in between each subsequent attempt
@@ -216,53 +90,88 @@ begin
216
90
  "#{RightScale::AgentIdentity.new('rs', 'broker', @settings[:port].to_i, @settings[:host].gsub('-', '~')).to_s}")
217
91
  log 'reconnecting'
218
92
  EM.reconnect(@settings[:host], @settings[:port], self)
93
+ rescue Exception => e
94
+ RightScale::Log.error("Exception caught during AMQP reconnect", e, :trace)
95
+ reconnect if @reconnecting
219
96
  end
220
- end
221
97
 
222
- # Monkey patch AMQP to clean up @conn when an error is raised after a broker request failure,
223
- # otherwise AMQP becomes unusable
224
- AMQP.module_eval do
225
- def self.start *args, &blk
226
- begin
227
- EM.run{
228
- @conn ||= connect *args
229
- @conn.callback(&blk) if blk
230
- @conn
231
- }
232
- rescue Exception => e
233
- @conn = nil
234
- raise e
235
- end
236
- end
237
- end
238
-
239
- # This monkey patch catches exceptions that would otherwise cause EM to stop or be in a bad
240
- # state if a top level EM error handler was setup. Instead close the connection and leave EM
241
- # alone.
242
- # Don't log an error if the environment variable IGNORE_AMQP_FAILURES is set (used in the
243
- # enroll script)
244
- AMQP::Client.module_eval do
98
+ # Catch exceptions that would otherwise cause EM to stop or be in a bad state if a top
99
+ # level EM error handler was setup. Instead close the connection and leave EM alone.
100
+ # Don't log an error if the environment variable IGNORE_AMQP_FAILURES is set
245
101
  alias :orig_receive_data :receive_data
246
102
  def receive_data(*args)
247
103
  begin
248
104
  orig_receive_data(*args)
249
105
  rescue Exception => e
250
- RightScale::Log.error("Exception caught while processing AMQP frame, closing connection",
251
- e, :trace) unless ENV['IGNORE_AMQP_FAILURES']
106
+ unless ENV['IGNORE_AMQP_FAILURES']
107
+ RightScale::Log.error("Exception caught while processing AMQP frame, closing connection", e, :trace)
108
+ end
252
109
  close_connection
253
110
  end
254
111
  end
112
+
113
+ # Make it log to RightScale when logging enabled
114
+ def log(*args)
115
+ return unless @settings[:logging] or AMQP.logging
116
+ require 'pp'
117
+ RightScale::Log.info("AMQP #{args.pretty_inspect.chomp}")
118
+ end
119
+ end
120
+
121
+ AMQP::Channel.class_eval do
122
+ # Detect message return and make callback
123
+ def check_content_completion
124
+ if @body.length >= @header.size
125
+ if @method.is_a? AMQP::Protocol::Basic::Return
126
+ @on_return_message.call @method, @body if @on_return_message
127
+ else
128
+ @header.properties.update(@method.arguments)
129
+ @consumer.receive @header, @body if @consumer
130
+ end
131
+ @body = @header = @consumer = @method = nil
132
+ end
133
+ end
134
+
135
+ # Provide callback to be activated when a message is returned
136
+ def return_message(&blk)
137
+ @on_return_message = blk
138
+ end
139
+
140
+ # Apply :no_declare option
141
+ def validate_parameters_match!(entity, parameters)
142
+ unless entity.opts == parameters || parameters[:passive] || parameters[:no_declare] || entity.opts[:no_declare]
143
+ raise AMQP::IncompatibleOptionsError.new(entity.name, entity.opts, parameters)
144
+ end
145
+ end
146
+
147
+ # Make it log to RightScale when logging enabled
148
+ def log(*args)
149
+ return unless AMQP.logging
150
+ require 'pp'
151
+ RightScale::Log.info("AMQP #{args.pretty_inspect.chomp}")
152
+ end
255
153
  end
256
154
 
257
- # Add a new callback to amqp gem that triggers once the handshake with the broker completed
258
- # The 'connected' status callback happens before the handshake is done and if it results in
259
- # a lot of activity it might prevent EM from being able to call the code handling the
260
- # incoming handshake packet in a timely fashion causing the broker to close the connection
261
- AMQP::BasicClient.module_eval do
262
- alias :orig_process_frame :process_frame
263
- def process_frame(frame)
264
- orig_process_frame(frame)
265
- @connection_status.call(:ready) if @connection_status && frame.payload.is_a?(AMQP::Protocol::Connection::Start)
155
+ # Add :no_declare => true option for new Queue objects to allow an instance that has
156
+ # no configuration privileges to enroll without blowing up the AMQP gem when it tries
157
+ # to subscribe to its queue before it has been created (already supported in gem for
158
+ # Exchange)
159
+ AMQP::Queue.class_eval do
160
+ def initialize(mq, name, opts = {}, &block)
161
+ @mq = mq
162
+ @opts = self.class.add_default_options(name, opts, block)
163
+ @bindings ||= {}
164
+ @name = name unless name.empty?
165
+ @status = @opts[:nowait] ? :unknown : :unfinished
166
+ unless opts[:no_declare]
167
+ @mq.callback{
168
+ @mq.send AMQP::Protocol::Queue::Declare.new(@opts)
169
+ }
170
+ end
171
+
172
+ self.callback = block
173
+
174
+ block.call(self) if @opts[:nowait] && block
266
175
  end
267
176
  end
268
177