right_agent 0.6.6 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. data/lib/right_agent/agent.rb +26 -25
  2. data/lib/right_agent/agent_config.rb +28 -2
  3. data/lib/right_agent/command/command_constants.rb +2 -2
  4. data/lib/right_agent/core_payload_types/executable_bundle.rb +3 -21
  5. data/lib/right_agent/core_payload_types/login_user.rb +19 -4
  6. data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -1
  7. data/lib/right_agent/core_payload_types/right_script_instantiation.rb +7 -1
  8. data/lib/right_agent/dispatcher.rb +6 -19
  9. data/lib/right_agent/idempotent_request.rb +72 -17
  10. data/lib/right_agent/monkey_patches/ruby_patch.rb +0 -1
  11. data/lib/right_agent/monkey_patches.rb +0 -1
  12. data/lib/right_agent/operation_result.rb +27 -4
  13. data/lib/right_agent/packets.rb +47 -23
  14. data/lib/right_agent/platform/darwin.rb +33 -2
  15. data/lib/right_agent/platform/linux.rb +98 -2
  16. data/lib/right_agent/platform/windows.rb +41 -6
  17. data/lib/right_agent/platform.rb +11 -2
  18. data/lib/right_agent/scripts/agent_controller.rb +2 -1
  19. data/lib/right_agent/scripts/agent_deployer.rb +2 -2
  20. data/lib/right_agent/scripts/stats_manager.rb +7 -3
  21. data/lib/right_agent/sender.rb +45 -28
  22. data/lib/right_agent.rb +2 -5
  23. data/right_agent.gemspec +5 -3
  24. data/spec/agent_config_spec.rb +1 -1
  25. data/spec/agent_spec.rb +26 -20
  26. data/spec/core_payload_types/login_user_spec.rb +7 -3
  27. data/spec/idempotent_request_spec.rb +218 -48
  28. data/spec/operation_result_spec.rb +19 -0
  29. data/spec/packets_spec.rb +42 -1
  30. data/spec/platform/darwin.rb +11 -0
  31. data/spec/platform/linux.rb +23 -0
  32. data/spec/platform/linux_volume_manager_spec.rb +43 -43
  33. data/spec/platform/platform_spec.rb +35 -32
  34. data/spec/platform/windows.rb +11 -0
  35. data/spec/sender_spec.rb +21 -25
  36. metadata +47 -40
  37. data/lib/right_agent/broker_client.rb +0 -686
  38. data/lib/right_agent/ha_broker_client.rb +0 -1327
  39. data/lib/right_agent/monkey_patches/amqp_patch.rb +0 -274
  40. data/lib/right_agent/monkey_patches/ruby_patch/string_patch.rb +0 -107
  41. data/lib/right_agent/stats_helper.rb +0 -745
  42. data/spec/broker_client_spec.rb +0 -962
  43. data/spec/ha_broker_client_spec.rb +0 -1695
  44. data/spec/monkey_patches/amqp_patch_spec.rb +0 -100
  45. data/spec/monkey_patches/string_patch_spec.rb +0 -99
  46. data/spec/stats_helper_spec.rb +0 -686
@@ -1,1327 +0,0 @@
1
- #
2
- # Copyright (c) 2009-2011 RightScale Inc
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # "Software"), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
- # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
- # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
- # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
- # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
-
23
- module RightScale
24
-
25
- # Client for multiple AMQP brokers to achieve a high availability service
26
- class HABrokerClient
27
-
28
- include StatsHelper
29
-
30
- class NoUserData < Exception; end
31
- class NoBrokerHosts < Exception; end
32
- class NoConnectedBrokers < Exception; end
33
-
34
- # Message publishing context
35
- class Context
36
-
37
- # (String) Message class name in lower snake case
38
- attr_reader :name
39
-
40
- # (String) Request type if applicable
41
- attr_reader :type
42
-
43
- # (String) Original sender of message if applicable
44
- attr_reader :from
45
-
46
- # (String) Generated message identifier if applicable
47
- attr_reader :token
48
-
49
- # (Boolean) Whether the packet is one that does not have an associated response
50
- attr_reader :one_way
51
-
52
- # (Hash) Options used to publish message
53
- attr_reader :options
54
-
55
- # (Array) Identity of candidate brokers when message was published
56
- attr_reader :brokers
57
-
58
- # (Array) Identity of brokers that have failed to deliver message with last one at end
59
- attr_reader :failed
60
-
61
- # Create context
62
- #
63
- # === Parameters
64
- # packet(Packet):: Packet being published
65
- # options(Hash):: Publish options
66
- # brokers(Array):: Identity of candidate brokers
67
- def initialize(packet, options, brokers)
68
- @name = (packet.respond_to?(:name) ? packet.name : packet.class.name.snake_case)
69
- @type = (packet.type if packet.respond_to?(:type) && packet.type != packet.class)
70
- @from = (packet.from if packet.respond_to?(:from))
71
- @token = (packet.token if packet.respond_to?(:token))
72
- @one_way = (packet.respond_to?(:one_way) ? packet.one_way : true)
73
- @options = options
74
- @brokers = brokers
75
- @failed = []
76
- end
77
-
78
- # Record delivery failure
79
- #
80
- # === Parameters
81
- # identity(String):: Identity of broker that failed delivery
82
- #
83
- # === Return
84
- # true:: Always return true
85
- def record_failure(identity)
86
- @failed << identity
87
- end
88
-
89
- end
90
-
91
- # Default number of seconds between reconnect attempts
92
- RECONNECT_INTERVAL = 60
93
-
94
- # (Array(Broker)) Priority ordered list of AMQP broker clients (exposed only for unit test purposes)
95
- attr_accessor :brokers
96
-
97
- # (Integer|nil) Home island identifier, or nil if unknown
98
- attr_reader :home_island
99
-
100
- # Create connections to all configured AMQP brokers
101
- # The constructed broker client list is in priority order with brokers in home island first
102
- #
103
- # === Parameters
104
- # serializer(Serializer):: Serializer used for marshaling packets being published; if nil,
105
- # has same effect as setting options :no_serialize and :no_unserialize
106
- # options(Hash):: Configuration options
107
- # :user(String):: User name
108
- # :pass(String):: Password
109
- # :vhost(String):: Virtual host path name
110
- # :insist(Boolean):: Whether to suppress redirection of connection
111
- # :reconnect_interval(Integer):: Number of seconds between reconnect attempts, defaults to RECONNECT_INTERVAL
112
- # :heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
113
- # connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
114
- # :host{String):: Comma-separated list of AMQP broker host names; if only one, it is reapplied
115
- # to successive ports; if none, defaults to localhost; each host may be followed by ':'
116
- # and a short string to be used as a broker index; the index defaults to the list index,
117
- # e.g., "host_a:0, host_c:2"
118
- # :port(String|Integer):: Comma-separated list of AMQP broker port numbers corresponding to :host list;
119
- # if only one, it is incremented and applied to successive hosts; if none, defaults to AMQP::PORT
120
- # :prefetch(Integer):: Maximum number of messages the AMQP broker is to prefetch for the agent
121
- # before it receives an ack. Value 1 ensures that only last unacknowledged gets redelivered
122
- # if the agent crashes. Value 0 means unlimited prefetch.
123
- # :islands(IslandData):: Islands with host and port settings for which connections are to be
124
- # created (takes precedence over any specified :host and :port option)
125
- # :home_island(Integer):: Identifier for home island for this server usable for accessing
126
- # :islands (takes precedence over any specified :host and :port option)
127
- # :order(Symbol):: Broker selection order when publishing a message: :random or :priority,
128
- # defaults to :priority, value can be overridden on publish call
129
- # :exception_callback(Proc):: Callback activated on exception events with parameters
130
- # exception(Exception):: Exception
131
- # message(Packet):: Message being processed
132
- # client(HABrokerClient):: Reference to this client
133
- # :exception_on_receive_callback(Proc):: Callback activated on a receive exception with parameters
134
- # message(String):: Message content that caused an exception
135
- # exception(Exception):: Exception that was raised
136
- #
137
- # === Raise
138
- # ArgumentError:: If :host and :port are not matched lists or :home_island is not found
139
- def initialize(serializer, options = {})
140
- @options = options.dup
141
- @options[:update_status_callback] = lambda { |b, c| update_status(b, c) }
142
- @options[:reconnect_interval] ||= RECONNECT_INTERVAL
143
- @connection_status = {}
144
- @serializer = serializer
145
- @published = Published.new
146
- reset_stats
147
- @select = @options[:order] || :priority
148
- islands = @options[:islands]
149
- if islands
150
- islands.each_value do |i|
151
- @brokers = connect_island(i.broker_hosts, i.broker_ports, i) if i.id == @options[:home_island]
152
- end
153
- @home_island = @options[:home_island]
154
- raise ArgumentError, "Could not find home island #{@home_island}" unless @brokers
155
- islands.each_value do |i|
156
- @brokers += connect_island(i.broker_hosts, i.broker_ports, i) if i.id != @home_island
157
- end
158
- else
159
- @brokers = connect_island(@options[:host], @options[:port])
160
- end
161
- @closed = false
162
- @brokers_hash = {}
163
- @brokers.each { |b| @brokers_hash[b.identity] = b }
164
- return_message { |i, r, m, t, c| handle_return(i, r, m, t, c) }
165
- end
166
-
167
- # Parse agent user data to extract broker host and port configuration
168
- # Agents below r_s_version 9 only support using one broker
169
- #
170
- # === Parameters
171
- # user_data(String):: Agent user data in <name>=<value>&<name>=<value>&... form
172
- # with required name RS_rn_url and optional names RS_rn_host and RS_rn_port
173
- #
174
- # === Return
175
- # (Array):: Broker hosts and ports as comma-separated list in priority order in the form
176
- # <hostname>:<index>,<hostname>:<index>,...
177
- # <port>:<index>,<port>:<index>,... or nil if none specified
178
- #
179
- # === Raise
180
- # NoUserData:: If the user data is missing
181
- # NoBrokerHosts:: If no brokers could be extracted from the user data
182
- def self.parse_user_data(user_data)
183
- raise NoUserData.new("User data is missing") if user_data.nil? || user_data.empty?
184
- hosts = ""
185
- ports = nil
186
- user_data.split("&").each do |data|
187
- name, value = data.split("=")
188
- if name == "RS_rn_url"
189
- h = value.split("@").last.split("/").first
190
- # Translate host name used by very old agents using only one broker
191
- h = "broker1-1.rightscale.com" if h == "broker.rightscale.com"
192
- hosts = h + hosts
193
- end
194
- if name == "RS_rn_host"
195
- hosts << value
196
- end
197
- if name == "RS_rn_port"
198
- ports = value
199
- end
200
- end
201
- raise NoBrokerHosts.new("No brokers found in user data") if hosts.empty?
202
- [hosts, ports]
203
- end
204
-
205
- # Parse host and port information to form list of broker address information
206
- #
207
- # === Parameters
208
- # host{String):: Comma-separated list of broker host names; if only one, it is reapplied
209
- # to successive ports; if none, defaults to localhost; each host may be followed by ':'
210
- # and a short string to be used as a broker index; the index defaults to the list index,
211
- # e.g., "host_a:0, host_c:2"
212
- # port(String|Integer):: Comma-separated list of broker port numbers corresponding to :host list;
213
- # if only one, it is incremented and applied to successive hosts; if none, defaults to AMQP::PORT
214
- #
215
- # === Returns
216
- # (Array(Hash)):: List of broker addresses with keys :host, :port, :index
217
- #
218
- # === Raise
219
- # ArgumentError:: If host and port are not matched lists
220
- def self.addresses(host, port)
221
- hosts = if host && !host.empty? then host.split(/,\s*/) else [ "localhost" ] end
222
- ports = if port && port.size > 0 then port.to_s.split(/,\s*/) else [ ::AMQP::PORT ] end
223
- if hosts.size != ports.size && hosts.size != 1 && ports.size != 1
224
- raise ArgumentError.new("Unmatched AMQP host/port lists -- hosts: #{host.inspect} ports: #{port.inspect}")
225
- end
226
- i = -1
227
- if hosts.size > 1
228
- hosts.map do |host|
229
- i += 1
230
- h = host.split(/:\s*/)
231
- port = if ports[i] then ports[i].to_i else ports[0].to_i end
232
- port = port.to_s.split(/:\s*/)[0]
233
- {:host => h[0], :port => port.to_i, :index => (h[1] || i.to_s).to_i}
234
- end
235
- else
236
- ports.map do |port|
237
- i += 1
238
- p = port.to_s.split(/:\s*/)
239
- host = if hosts[i] then hosts[i] else hosts[0] end
240
- host = host.split(/:\s*/)[0]
241
- {:host => host, :port => p[0].to_i, :index => (p[1] || i.to_s).to_i}
242
- end
243
- end
244
- end
245
-
246
- # Parse host and port information to form list of broker identities
247
- #
248
- # === Parameters
249
- # host{String):: Comma-separated list of broker host names; if only one, it is reapplied
250
- # to successive ports; if none, defaults to localhost; each host may be followed by ':'
251
- # and a short string to be used as a broker index; the index defaults to the list index,
252
- # e.g., "host_a:0, host_c:2"
253
- # port(String|Integer):: Comma-separated list of broker port numbers corresponding to :host list;
254
- # if only one, it is incremented and applied to successive hosts; if none, defaults to AMQP::PORT
255
- #
256
- # === Returns
257
- # (Array):: Identity of each broker
258
- #
259
- # === Raise
260
- # ArgumentError:: If host and port are not matched lists
261
- def self.identities(host, port = nil)
262
- addresses(host, port).map { |a| identity(a[:host], a[:port]) }
263
- end
264
-
265
- # Construct a broker serialized identity from its host and port of the form
266
- # rs-broker-host-port, with any '-'s in host replaced by '~'
267
- #
268
- # === Parameters
269
- # host{String):: IP host name or address for individual broker
270
- # port(Integer):: TCP port number for individual broker, defaults to ::AMQP::PORT
271
- #
272
- # === Returns
273
- # (String):: Broker serialized identity
274
- def self.identity(host, port = ::AMQP::PORT)
275
- AgentIdentity.new('rs', 'broker', port.to_i, host.gsub('-', '~')).to_s
276
- end
277
-
278
- # Break broker serialized identity down into individual parts if exists
279
- #
280
- # === Parameters
281
- # id(Integer|String):: Broker alias or serialized identity
282
- #
283
- # === Return
284
- # (Array):: Host, port, index, priority, and island_id, or all nil if broker not found
285
- def identity_parts(id)
286
- @brokers.each do |b|
287
- return [b.host, b.port, b.index, priority(b.identity, b.island_id)[0], b.island_id] if b.identity == id || b.alias == id
288
- end
289
- [nil, nil, nil, nil, nil]
290
- end
291
-
292
- # Convert broker identities to aliases
293
- #
294
- # === Parameters
295
- # identities(Array):: Broker identities
296
- #
297
- # === Return
298
- # (Array):: Broker aliases
299
- def aliases(identities)
300
- identities.map { |i| alias_(i) }
301
- end
302
-
303
- # Convert broker serialized identity to its alias
304
- #
305
- # === Parameters
306
- # identity(String):: Broker serialized identity
307
- #
308
- # === Return
309
- # (String|nil):: Broker alias, or nil if not a known broker
310
- def alias_(identity)
311
- @brokers_hash[identity].alias rescue nil
312
- end
313
-
314
- # Form string of hosts and associated indices for an island
315
- #
316
- # === Parameters
317
- # island_id(Integer|nil):: Island identifier, defaults to home island
318
- #
319
- # === Return
320
- # (String):: Comma separated list of host:index
321
- def hosts(island_id = nil)
322
- in_island(island_id).map { |b| "#{b.host}:#{b.index}" }.join(",")
323
- end
324
-
325
- # Form string of hosts and associated indices for an island
326
- #
327
- # === Parameters
328
- # island_id(Integer|nil):: Island identifier, defaults to home island
329
- #
330
- # === Return
331
- # (String):: Comma separated list of host:index
332
- def ports(island_id = nil)
333
- in_island(island_id).map { |b| "#{b.port}:#{b.index}" }.join(",")
334
- end
335
-
336
- # Get broker serialized identity if client exists
337
- #
338
- # === Parameters
339
- # id(Integer|String):: Broker alias or serialized identity
340
- #
341
- # === Return
342
- # (String|nil):: Broker serialized identity if client found, otherwise nil
343
- def get(id)
344
- @brokers.each { |b| return b.identity if b.identity == id || b.alias == id }
345
- nil
346
- end
347
-
348
- # Check whether connected to broker
349
- #
350
- # === Parameters
351
- # identity{String):: Broker serialized identity
352
- #
353
- # === Return
354
- # (Boolean):: true if connected to broker, otherwise false, or nil if broker unknown
355
- def connected?(identity)
356
- @brokers_hash[identity].connected? rescue nil
357
- end
358
-
359
- # Get serialized identity of connected brokers for an island
360
- #
361
- # === Parameters
362
- # island_id(Integer|nil):: Island identifier, defaults to home island
363
- #
364
- # === Return
365
- # (Array):: Serialized identity of connected brokers
366
- def connected(island_id = nil)
367
- in_island(island_id).inject([]) { |c, b| if b.connected? then c << b.identity else c end }
368
- end
369
-
370
- # Get serialized identity of connected brokers for all islands
371
- #
372
- # === Return
373
- # (Array):: Serialized identity of connected brokers
374
- def all_connected
375
- @brokers.inject([]) { |c, b| if b.connected? then c << b.identity else c end }
376
- end
377
-
378
- # Get serialized identity of brokers that are usable, i.e., connecting or confirmed connected
379
- #
380
- # === Return
381
- # (Array):: Serialized identity of usable brokers
382
- def usable
383
- each_usable.map { |b| b.identity }
384
- end
385
-
386
- # Get serialized identity of unusable brokers
387
- #
388
- # === Return
389
- # (Array):: Serialized identity of unusable brokers
390
- def unusable
391
- @brokers.map { |b| b.identity } - each_usable.map { |b| b.identity }
392
- end
393
-
394
- # Get serialized identity of all brokers
395
- #
396
- # === Return
397
- # (Array):: Serialized identity of all brokers
398
- def all
399
- @brokers.map { |b| b.identity }
400
- end
401
-
402
- # Get serialized identity of brokers in home island
403
- #
404
- # === Return
405
- # (Array):: Serialized identity of brokers
406
- def home
407
- island
408
- end
409
-
410
- # Get serialized identity of brokers in given island
411
- #
412
- # === Parameters
413
- # island_id(Integer|nil):: Island identifier, defaults to home island
414
- #
415
- # === Return
416
- # (Array):: Serialized identity of brokers
417
- def island(island_id = nil)
418
- in_island(island_id).map { |b| b.identity }
419
- end
420
-
421
- # Get serialized identity of failed broker clients, i.e., ones that were never successfully
422
- # connected, not ones that are just disconnected
423
- #
424
- # === Return
425
- # (Array):: Serialized identity of failed broker clients
426
- def failed
427
- @brokers.inject([]) { |c, b| b.failed? ? c << b.identity : c }
428
- end
429
-
430
- # Change connection heartbeat frequency to be used for any new connections
431
- #
432
- # === Parameters
433
- # heartbeat(Integer):: Number of seconds between AMQP connection heartbeats used to keep
434
- # connection alive (e.g., when AMQP broker is behind a firewall), nil or 0 means disable
435
- #
436
- # === Return
437
- # (Integer|nil):: New heartbeat setting
438
- def heartbeat=(heartbeat)
439
- @options[:heartbeat] = heartbeat
440
- end
441
-
442
- # Make new connection to broker at specified address unless already connected
443
- # or currently connecting
444
- #
445
- # === Parameters
446
- # host{String):: IP host name or address for individual broker
447
- # port(Integer):: TCP port number for individual broker
448
- # index(Integer):: Unique index for broker within given island for use in forming alias
449
- # priority(Integer|nil):: Priority position of this broker in list for use by this agent
450
- # with nil or a value that would leave a gap in the list meaning add to end of list
451
- # island(IslandData|nil):: Island containing this broker, defaults to home island
452
- # force(Boolean):: Reconnect even if already connected
453
- #
454
- # === Block
455
- # Optional block with following parameters to be called after initiating the connection
456
- # unless already connected to this broker:
457
- # identity(String):: Broker serialized identity
458
- #
459
- # === Return
460
- # (Boolean):: true if connected, false if no connect attempt made
461
- #
462
- # === Raise
463
- # Exception:: If host and port do not match an existing broker but index does
464
- def connect(host, port, index, priority = nil, island = nil, force = false, &blk)
465
- identity = self.class.identity(host, port)
466
- existing = @brokers_hash[identity]
467
- if existing && existing.usable? && !force
468
- Log.info("Ignored request to reconnect #{identity} because already #{existing.status.to_s}")
469
- false
470
- else
471
- old_identity = identity
472
- @brokers.each do |b|
473
- if index == b.index && (island.nil? || in_island?(b, island.id))
474
- # Changing host and/or port of existing broker client
475
- old_identity = b.identity
476
- break
477
- end
478
- end unless existing
479
-
480
- address = {:host => host, :port => port, :index => index}
481
- broker = BrokerClient.new(identity, address, @serializer, @exceptions, @options, island, existing)
482
- island_id = island && island.id
483
- p, i = priority(old_identity, island_id)
484
- if priority && priority < p
485
- @brokers.insert(i + priority, broker)
486
- elsif priority && priority > p
487
- Log.info("Reduced priority setting for broker #{identity} from #{priority} to #{p} to avoid gap in list")
488
- @brokers.insert(i + p, broker)
489
- else
490
- i += p
491
- if @brokers[i] && @brokers[i].island_id == island_id
492
- @brokers[i].close
493
- @brokers[i] = broker
494
- elsif @brokers[i]
495
- @brokers.insert(i, broker)
496
- else
497
- @brokers[i] = broker
498
- end
499
- end
500
- @brokers_hash[identity] = broker
501
- yield broker.identity if block_given?
502
- true
503
- end
504
- end
505
-
506
- # Connect to any brokers in islands for which not currently connected
507
- # Remove any broker clients for islands in which they are no longer configured
508
- #
509
- # === Parameters
510
- # islands(Array):: List of islands as IslandData object
511
- #
512
- # === Return
513
- # identities(Array):: Identity of newly connected brokers
514
- def connect_update(islands)
515
- old = all
516
- new = []
517
- islands.each_value do |i|
518
- priority = 0
519
- self.class.addresses(i.broker_hosts, i.broker_ports).each do |a|
520
- identity = self.class.identity(a[:host], a[:port])
521
- if @brokers_hash[identity]
522
- old.delete(identity)
523
- else
524
- begin
525
- new << identity if connect(a[:host], a[:port], a[:index], priority, i)
526
- rescue Exception => e
527
- Log.error("Failed to connect to broker #{identity}", e, :trace)
528
- @exceptions.track("connect update", e)
529
- end
530
- end
531
- priority += 1
532
- end
533
- end
534
-
535
- old.each do |identity|
536
- if b = @brokers_hash[identity]
537
- remove(b.host, b.port)
538
- else
539
- Log.error("Could not remove broker #{identity} during connection update because not found, " +
540
- "current broker configuration: #{status.inspect}")
541
- end
542
- end
543
- { :add => new, :remove => old, :home => home }
544
- end
545
-
546
- # Subscribe an AMQP queue to an AMQP exchange on all broker clients that are connected or still connecting
547
- # Allow connecting here because subscribing may happen before all have confirmed connected
548
- # Do not wait for confirmation from broker client that subscription is complete
549
- # When a message is received, acknowledge, unserialize, and log it as specified
550
- # If the message is unserialized and it is not of the right type, it is dropped after logging a warning
551
- #
552
- # === Parameters
553
- # queue(Hash):: AMQP queue being subscribed with keys :name and :options,
554
- # which are the standard AMQP ones plus
555
- # :no_declare(Boolean):: Whether to skip declaring this queue on the broker
556
- # to cause its creation; for use when client does not have permission to create or
557
- # knows the queue already exists and wants to avoid declare overhead
558
- # exchange(Hash|nil):: AMQP exchange to subscribe to with keys :type, :name, and :options,
559
- # nil means use empty exchange by directly subscribing to queue; the :options are the
560
- # standard AMQP ones plus
561
- # :no_declare(Boolean):: Whether to skip declaring this exchange on the broker
562
- # to cause its creation; for use when client does not have create permission or
563
- # knows the exchange already exists and wants to avoid declare overhead
564
- # options(Hash):: Subscribe options:
565
- # :ack(Boolean):: Explicitly acknowledge received messages to AMQP
566
- # :no_unserialize(Boolean):: Do not unserialize message, this is an escape for special
567
- # situations like enrollment, also implicitly disables receive filtering and logging;
568
- # this option is implicitly invoked if initialize without a serializer
569
- # (packet class)(Array(Symbol)):: Filters to be applied in to_s when logging packet to :info,
570
- # only packet classes specified are accepted, others are not processed but are logged with error
571
- # :category(String):: Packet category description to be used in error messages
572
- # :log_data(String):: Additional data to display at end of log entry
573
- # :no_log(Boolean):: Disable receive logging unless debug level
574
- # :exchange2(Hash):: Additional exchange to which same queue is to be bound
575
- # :brokers(Array):: Identity of brokers for which to subscribe, defaults to all usable if nil or empty
576
- #
577
- # === Block
578
- # Block with following parameters to be called each time exchange matches a message to the queue:
579
- # identity(String):: Serialized identity of broker delivering the message
580
- # message(Packet|String):: Message received, which is unserialized unless :no_unserialize was specified
581
- #
582
- # === Return
583
- # identities(Array):: Identity of brokers where successfully subscribed
584
- def subscribe(queue, exchange = nil, options = {}, &blk)
585
- identities = []
586
- brokers = options.delete(:brokers)
587
- each_usable(brokers) { |b| identities << b.identity if b.subscribe(queue, exchange, options, &blk) }
588
- Log.info("Could not subscribe to queue #{queue.inspect} on exchange #{exchange.inspect} " +
589
- "on brokers #{each_usable(brokers).inspect} when selected #{brokers.inspect} " +
590
- "from usable #{usable.inspect}") if identities.empty?
591
- identities
592
- end
593
-
594
- # Unsubscribe from the specified queues on usable broker clients
595
- # Silently ignore unknown queues
596
- #
597
- # === Parameters
598
- # queue_names(Array):: Names of queues previously subscribed to
599
- # timeout(Integer):: Number of seconds to wait for all confirmations, defaults to no timeout
600
- #
601
- # === Block
602
- # Optional block with no parameters to be called after all queues are unsubscribed
603
- #
604
- # === Return
605
- # true:: Always return true
606
- def unsubscribe(queue_names, timeout = nil, &blk)
607
- count = each_usable.inject(0) do |c, b|
608
- c + b.queues.inject(0) { |c, q| c + (queue_names.include?(q.name) ? 1 : 0) }
609
- end
610
- if count == 0
611
- blk.call if blk
612
- else
613
- handler = CountedDeferrable.new(count, timeout)
614
- handler.callback { blk.call if blk }
615
- each_usable { |b| b.unsubscribe(queue_names) { handler.completed_one } }
616
- end
617
- true
618
- end
619
-
620
- # Declare queue or exchange object but do not subscribe to it
621
- #
622
- # === Parameters
623
- # type(Symbol):: Type of object: :queue, :direct, :fanout or :topic
624
- # name(String):: Name of object
625
- # options(Hash):: Standard AMQP declare options plus
626
- # :brokers(Array):: Identity of brokers for which to declare, defaults to all usable if nil or empty
627
- #
628
- # === Return
629
- # identities(Array):: Identity of brokers where successfully declared
630
- def declare(type, name, options = {})
631
- identities = []
632
- brokers = options.delete(:brokers)
633
- each_usable(brokers) { |b| identities << b.identity if b.declare(type, name, options) }
634
- Log.info("Could not declare #{type.to_s} #{name.inspect} on brokers #{each_usable(brokers).inspect} " +
635
- "when selected #{brokers.inspect} from usable #{usable.inspect}") if identities.empty?
636
- identities
637
- end
638
-
639
- # Publish message to AMQP exchange of first connected broker
640
- # Only use home island brokers by default
641
- #
642
- # === Parameters
643
- # exchange(Hash):: AMQP exchange to subscribe to with keys :type, :name, and :options,
644
- # which are the standard AMQP ones plus
645
- # :no_declare(Boolean):: Whether to skip declaring this exchange or queue on the broker
646
- # to cause its creation; for use when client does not have create permission or
647
- # knows the object already exists and wants to avoid declare overhead
648
- # :declare(Boolean):: Whether to delete this exchange or queue from the AMQP cache
649
- # to force it to be declared on the broker and thus be created if it does not exist
650
- # packet(Packet):: Message to serialize and publish
651
- # options(Hash):: Publish options -- standard AMQP ones plus
652
- # :fanout(Boolean):: true means publish to all connected brokers
653
- # :brokers(Array):: Identity of brokers selected for use, defaults to all home brokers
654
- # if nil or empty
655
- # :order(Symbol):: Broker selection order: :random or :priority,
656
- # defaults to @select if :brokers is nil, otherwise defaults to :priority
657
- # :no_serialize(Boolean):: Do not serialize packet because it is already serialized,
658
- # this is an escape for special situations like enrollment, also implicitly disables
659
- # publish logging; this option is implicitly invoked if initialize without a serializer
660
- # :log_filter(Array(Symbol)):: Filters to be applied in to_s when logging packet to :info
661
- # :log_data(String):: Additional data to display at end of log entry
662
- # :no_log(Boolean):: Disable publish logging unless debug level
663
- #
664
- # === Return
665
- # identities(Array):: Identity of brokers where packet was successfully published
666
- #
667
- # === Raise
668
- # NoConnectedBrokers:: If cannot find a connected broker
669
- def publish(exchange, packet, options = {})
670
- identities = []
671
- no_serialize = options[:no_serialize] || @serializer.nil?
672
- message = if no_serialize then packet else @serializer.dump(packet) end
673
- brokers = use(options)
674
- brokers.each do |b|
675
- if b.publish(exchange, packet, message, options.merge(:no_serialize => no_serialize))
676
- identities << b.identity
677
- if options[:mandatory] && !no_serialize
678
- context = Context.new(packet, options, brokers.map { |b| b.identity })
679
- @published.store(message, context)
680
- end
681
- break unless options[:fanout]
682
- end
683
- end
684
- if identities.empty?
685
- selected = "selected " if options[:brokers]
686
- list = aliases(brokers.map { |b| b.identity }).join(", ")
687
- raise NoConnectedBrokers, "None of #{selected}brokers [#{list}] are usable for publishing"
688
- end
689
- identities
690
- end
691
-
692
- # Register callback to be activated when a broker returns a message that could not be delivered
693
- # A message published with :mandatory => true is returned if the exchange does not have any associated queues
694
- # or if all the associated queues do not have any consumers
695
- # A message published with :immediate => true is returned for the same reasons as :mandatory plus if all
696
- # of the queues associated with the exchange are not immediately ready to consume the message
697
- # Remove any previously registered callback
698
- #
699
- # === Block
700
- # Required block to be called when a message is returned with parameters
701
- # identity(String):: Broker serialized identity
702
- # reason(String):: Reason for return
703
- # "NO_ROUTE" - queue does not exist
704
- # "NO_CONSUMERS" - queue exists but it has no consumers, or if :immediate was specified,
705
- # all consumers are not immediately ready to consume
706
- # "ACCESS_REFUSED" - queue not usable because broker is in the process of stopping service
707
- # message(String):: Returned serialized message
708
- # to(String):: Queue to which message was published
709
- # context(Context|nil):: Message publishing context, or nil if not available
710
- #
711
- # === Return
712
- # true:: Always return true
713
- def return_message(&blk)
714
- each_usable do |b|
715
- b.return_message do |to, reason, message|
716
- context = @published.fetch(message)
717
- context.record_failure(b.identity) if context
718
- blk.call(b.identity, reason, message, to, context)
719
- end
720
- end
721
- true
722
- end
723
-
724
- # Provide callback to be activated when a message cannot be delivered
725
- #
726
- # === Block
727
- # Required block with parameters
728
- # reason(String):: Non-delivery reason
729
- # "NO_ROUTE" - queue does not exist
730
- # "NO_CONSUMERS" - queue exists but it has no consumers, or if :immediate was specified,
731
- # all consumers are not immediately ready to consume
732
- # "ACCESS_REFUSED" - queue not usable because broker is in the process of stopping service
733
- # type(String|nil):: Request type, or nil if not applicable
734
- # token(String|nil):: Generated message identifier, or nil if not applicable
735
- # from(String|nil):: Identity of original sender of message, or nil if not applicable
736
- # to(String):: Queue to which message was published
737
- #
738
- # === Return
739
- # true:: Always return true
740
- def non_delivery(&blk)
741
- @non_delivery = blk
742
- true
743
- end
744
-
745
- # Delete queue in all usable brokers or all selected brokers that are usable
746
- #
747
- # === Parameters
748
- # name(String):: Queue name
749
- # options(Hash):: Queue declare options plus
750
- # :brokers(Array):: Identity of brokers in which queue is to be deleted
751
- #
752
- # === Return
753
- # identities(Array):: Identity of brokers where queue was deleted
754
- def delete(name, options = {})
755
- identities = []
756
- u = usable
757
- brokers = options.delete(:brokers)
758
- ((brokers || u) & u).each { |i| identities << i if (b = @brokers_hash[i]) && b.delete(name, options) }
759
- identities
760
- end
761
-
762
- # Delete queue resources from AMQP in all usable brokers
763
- #
764
- # === Parameters
765
- # name(String):: Queue name
766
- # options(Hash):: Queue declare options plus
767
- # :brokers(Array):: Identity of brokers in which queue is to be deleted
768
- #
769
- # === Return
770
- # identities(Array):: Identity of brokers where queue was deleted
771
- def delete_amqp_resources(name, options = {})
772
- identities = []
773
- u = usable
774
- ((options[:brokers] || u) & u).each { |i| identities << i if (b = @brokers_hash[i]) && b.delete_amqp_resources(:queue, name) }
775
- identities
776
- end
777
-
778
- # Remove a broker client from the configuration
779
- # Invoke connection status callbacks only if connection is not already disabled
780
- # There is no check whether this is the last usable broker client
781
- #
782
- # === Parameters
783
- # host{String):: IP host name or address for individual broker
784
- # port(Integer):: TCP port number for individual broker
785
- #
786
- # === Block
787
- # Optional block with following parameters to be called after removing the connection
788
- # unless broker is not configured
789
- # identity(String):: Broker serialized identity
790
- #
791
- # === Return
792
- # identity(String|nil):: Serialized identity of broker removed, or nil if unknown
793
- def remove(host, port, &blk)
794
- identity = self.class.identity(host, port)
795
- if broker = @brokers_hash[identity]
796
- Log.info("Removing #{identity}, alias #{broker.alias} from broker list")
797
- broker.close(propagate = true, normal = true, log = false)
798
- @brokers_hash.delete(identity)
799
- @brokers.reject! { |b| b.identity == identity }
800
- yield identity if block_given?
801
- else
802
- Log.info("Ignored request to remove #{identity} from broker list because unknown")
803
- identity = nil
804
- end
805
- identity
806
- end
807
-
808
- # Declare a broker client as unusable
809
- #
810
- # === Parameters
811
- # identities(Array):: Identity of brokers
812
- #
813
- # === Return
814
- # true:: Always return true
815
- #
816
- # === Raises
817
- # Exception:: If identified broker is unknown
818
- def declare_unusable(identities)
819
- identities.each do |id|
820
- broker = @brokers_hash[id]
821
- raise Exception, "Cannot mark unknown broker #{id} unusable" unless broker
822
- broker.close(propagate = true, normal = false, log = false)
823
- end
824
- end
825
-
826
- # Close all broker client connections
827
- #
828
- # === Block
829
- # Optional block with no parameters to be called after all connections are closed
830
- #
831
- # === Return
832
- # true:: Always return true
833
- def close(&blk)
834
- if @closed
835
- blk.call if blk
836
- else
837
- @closed = true
838
- @connection_status = {}
839
- handler = CountedDeferrable.new(@brokers.size)
840
- handler.callback { blk.call if blk }
841
- @brokers.each do |b|
842
- begin
843
- b.close(propagate = false) { handler.completed_one }
844
- rescue Exception => e
845
- handler.completed_one
846
- Log.error("Failed to close broker #{b.alias}", e, :trace)
847
- @exceptions.track("close", e)
848
- end
849
- end
850
- end
851
- true
852
- end
853
-
854
- # Close an individual broker client connection
855
- #
856
- # === Parameters
857
- # identity(String):: Broker serialized identity
858
- # propagate(Boolean):: Whether to propagate connection status updates
859
- #
860
- # === Block
861
- # Optional block with no parameters to be called after connection closed
862
- #
863
- # === Return
864
- # true:: Always return true
865
- #
866
- # === Raise
867
- # Exception:: If broker unknown
868
- def close_one(identity, propagate = true, &blk)
869
- broker = @brokers_hash[identity]
870
- raise Exception, "Cannot close unknown broker #{identity}" unless broker
871
- broker.close(propagate, &blk)
872
- true
873
- end
874
-
875
- # Register callback to be activated when there is a change in connection status
876
- # Connection status change is measured individually for each island
877
- # Can be called more than once without affecting previous callbacks
878
- #
879
- # === Parameters
880
- # options(Hash):: Connection status monitoring options
881
- # :one_off(Integer):: Seconds to wait for status change; only send update once;
882
- # if timeout, report :timeout as the status
883
- # :boundary(Symbol):: :any if only report change on any (0/1) boundary,
884
- # :all if only report change on all (n-1/n) boundary, defaults to :any
885
- # :brokers(Array):: Only report a status change for these identified brokers
886
- #
887
- # === Block
888
- # Required block activated when connected count crosses a status boundary with following parameters
889
- # status(Symbol):: Status of connection: :connected, :disconnected, or :failed, with
890
- # :failed indicating that all selected brokers or all brokers have failed
891
- # island_id(Integer):: Island in which there was a change (optional parameter)
892
- #
893
- # === Return
894
- # id(String):: Identifier associated with connection status request
895
- def connection_status(options = {}, &callback)
896
- id = AgentIdentity.generate
897
- @connection_status[id] = {:boundary => options[:boundary], :brokers => options[:brokers], :callback => callback}
898
- if timeout = options[:one_off]
899
- @connection_status[id][:timer] = EM::Timer.new(timeout) do
900
- if @connection_status[id]
901
- if @connection_status[id][:callback].arity == 2
902
- @connection_status[id][:callback].call(:timeout, nil)
903
- else
904
- @connection_status[id][:callback].call(:timeout)
905
- end
906
- @connection_status.delete(id)
907
- end
908
- end
909
- end
910
- id
911
- end
912
-
913
- # Get status summary
914
- #
915
- # === Return
916
- # (Array(Hash)):: Status of each configured broker with keys
917
- # :identity(String):: Broker serialized identity
918
- # :alias(String):: Broker alias used in logs
919
- # :status(Symbol):: Status of connection
920
- # :disconnects(Integer):: Number of times lost connection
921
- # :failures(Integer):: Number of times connect failed
922
- # :retries(Integer):: Number of attempts to connect after failure
923
- def status
924
- @brokers.map { |b| b.summary }
925
- end
926
-
927
- # Get broker client statistics
928
- #
929
- # === Parameters:
930
- # reset(Boolean):: Whether to reset the statistics after getting the current ones
931
- #
932
- # === Return
933
- # stats(Hash):: Broker client stats with keys
934
- # "brokers"(Array):: Stats for each broker client in priority order
935
- # "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
936
- # "total"(Integer):: Total exceptions for this category
937
- # "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
938
- # "heartbeat"(Integer|nil):: Number of seconds between AMQP heartbeats, or nil if heartbeat disabled
939
- # "returns"(Hash|nil):: Message return activity stats with keys "total", "percent", "last", and "rate"
940
- # with percentage breakdown per return reason, or nil if none
941
- def stats(reset = false)
942
- stats = {
943
- "brokers" => @brokers.map { |b| b.stats },
944
- "exceptions" => @exceptions.stats,
945
- "heartbeat" => @options[:heartbeat],
946
- "returns" => @returns.all
947
- }
948
- reset_stats if reset
949
- stats
950
- end
951
-
952
- # Reset broker client statistics
953
- # Do not reset disconnect and failure stats because they might then be
954
- # inconsistent with underlying connection status
955
- #
956
- # === Return
957
- # true:: Always return true
958
- def reset_stats
959
- @returns = ActivityStats.new
960
- @exceptions = ExceptionStats.new(self, @options[:exception_callback])
961
- true
962
- end
963
-
964
- protected
965
-
966
- # Determine whether broker is in given island
967
- #
968
- # === Parameters
969
- # broker(BrokerClient):: Broker client
970
- # island_id(Integer|nil):: Island identifier, defaults to home island
971
- #
972
- # === Return
973
- # (Boolean):: true if in island, otherwise false
974
- def in_island?(broker, island_id = nil)
975
- (!island_id && broker.in_home_island) || (island_id && island_id == broker.island_id)
976
- end
977
-
978
- # Get clients for brokers in a given island
979
- #
980
- # === Parameters
981
- # island_id(Integer|nil):: Island identifier, defaults to home island
982
- #
983
- # === Return
984
- # (Array):: Broker clients in given island
985
- def in_island(island_id = nil)
986
- @brokers.select { |b| in_island?(b, island_id) }
987
- end
988
-
989
- # Connect to all brokers in island
990
- #
991
- # === Parameters
992
- # host{String):: Comma-separated list of AMQP broker host names; if only one, it is reapplied
993
- # to successive ports; if none, defaults to localhost; each host may be followed by ':'
994
- # and a short string to be used as a broker index; the index defaults to the list index,
995
- # e.g., "host_a:0, host_c:2"
996
- # port(String|Integer):: Comma-separated list of AMQP broker port numbers corresponding to :host list;
997
- # if only one, it is incremented and applied to successive hosts; if none, defaults to AMQP::PORT
998
- # island(IslandData|nil):: Island containing these brokers, defaults to home island
999
- #
1000
- # === Return
1001
- # (Array):: Broker clients created
1002
- def connect_island(host, port, island = nil)
1003
- self.class.addresses(host, port).map do |a|
1004
- identity = self.class.identity(a[:host], a[:port])
1005
- BrokerClient.new(identity, a, @serializer, @exceptions, @options, island, nil)
1006
- end
1007
- end
1008
-
1009
- # Determine priority of broker within island
1010
- # If broker not found, assign next available priority
1011
- #
1012
- # === Parameters
1013
- # identity(String):: Broker identity
1014
- # island_id(Integer|nil):: Island identifier, defaults to home island
1015
- #
1016
- # === Return
1017
- # (Array):: Priority and broker array index where island starts
1018
- def priority(identity, island_id = nil)
1019
- index = 0
1020
- priority = 0
1021
- found_island = false
1022
- @brokers.each do |b|
1023
- if in_island?(b, island_id)
1024
- found_island = true
1025
- break if b.identity == identity
1026
- priority += 1
1027
- elsif found_island
1028
- break
1029
- end
1030
- index += 1 unless found_island
1031
- end
1032
- [priority, index]
1033
- end
1034
-
1035
- # Iterate over clients that are usable, i.e., connecting or confirmed connected
1036
- #
1037
- # === Parameters
1038
- # identities(Array):: Identity of brokers to be considered, nil or empty array means all brokers
1039
- #
1040
- # === Block
1041
- # Optional block with following parameters to be called for each usable broker client
1042
- # broker(BrokerClient):: Broker client
1043
- #
1044
- # === Return
1045
- # (Array):: Usable broker clients
1046
- def each_usable(identities = nil)
1047
- choices = if identities && !identities.empty?
1048
- choices = identities.inject([]) { |c, i| if b = @brokers_hash[i] then c << b else c end }
1049
- else
1050
- @brokers
1051
- end
1052
- choices.select do |b|
1053
- if b.usable?
1054
- yield(b) if block_given?
1055
- true
1056
- end
1057
- end
1058
- end
1059
-
1060
- # Select the broker clients to be used in the desired order
1061
- # Only uses home island brokers if :brokers options not used
1062
- #
1063
- # === Parameters
1064
- # options(Hash):: Selection options:
1065
- # :brokers(Array):: Identity of brokers selected for use, defaults to all home brokers if nil or empty
1066
- # :order(Symbol):: Broker selection order: :random or :priority,
1067
- # defaults to @select if :brokers is nil, otherwise defaults to :priority
1068
- #
1069
- # === Return
1070
- # (Array):: Allowed BrokerClients in the order to be used
1071
- def use(options)
1072
- choices = []
1073
- select = options[:order]
1074
- if options[:brokers] && !options[:brokers].empty?
1075
- options[:brokers].each do |identity|
1076
- if choice = @brokers_hash[identity]
1077
- choices << choice
1078
- else
1079
- Log.error("Invalid broker identity #{identity.inspect}, check server configuration")
1080
- end
1081
- end
1082
- else
1083
- choices = in_island
1084
- select ||= @select
1085
- end
1086
- if select == :random
1087
- choices.sort_by { rand }
1088
- else
1089
- choices
1090
- end
1091
- end
1092
-
1093
- # Callback from broker client with connection status update
1094
- # Makes client callback with :connected or :disconnected status if boundary crossed
1095
- # for given island, or with :failed if all selected brokers or all brokers have failed
1096
- #
1097
- # === Parameters
1098
- # broker(BrokerClient):: Broker client reporting status update
1099
- # connected_before(Boolean):: Whether client was connected before this update
1100
- #
1101
- # === Return
1102
- # true:: Always return true
1103
- def update_status(broker, connected_before)
1104
- after = connected(broker.island_id)
1105
- before = after.clone
1106
- before.delete(broker.identity) if broker.connected? && !connected_before
1107
- before.push(broker.identity) if !broker.connected? && connected_before
1108
- unless before == after
1109
- island = broker.in_home_island ? " in home island" : " in island #{broker.island_alias}" if broker.island_id
1110
- Log.info("[status] Broker #{broker.alias} is now #{broker.status}, " +
1111
- "connected brokers#{island}: [#{aliases(after).join(", ")}]")
1112
- end
1113
- @connection_status.reject! do |k, v|
1114
- reject = false
1115
- if v[:brokers].nil? || v[:brokers].include?(broker.identity)
1116
- b, a, n, f = if v[:brokers].nil?
1117
- [before, after, in_island(broker.island_id).size, all]
1118
- else
1119
- [before & v[:brokers], after & v[:brokers], (island(broker.island_id) & v[:brokers]).size, v[:brokers]]
1120
- end
1121
- update = if v[:boundary] == :all
1122
- if b.size < n && a.size == n
1123
- :connected
1124
- elsif b.size == n && a.size < n
1125
- :disconnected
1126
- elsif (f - failed).empty?
1127
- :failed
1128
- end
1129
- else
1130
- if b.size == 0 && a.size > 0
1131
- :connected
1132
- elsif b.size > 0 && a.size == 0
1133
- :disconnected
1134
- elsif (f - failed).empty?
1135
- :failed
1136
- end
1137
- end
1138
- if update
1139
- if v[:callback].arity == 2
1140
- v[:callback].call(update, broker.island_id)
1141
- else
1142
- v[:callback].call(update)
1143
- end
1144
- if v[:timer]
1145
- v[:timer].cancel
1146
- reject = true
1147
- end
1148
- end
1149
- end
1150
- reject
1151
- end
1152
- true
1153
- end
1154
-
1155
- # Handle message returned by broker because it could not deliver it
1156
- # If agent still active, resend using another broker
1157
- # If this is last usable broker and persistent is enabled, allow message to be queued
1158
- # on next send even if the queue has no consumers so there is a chance of message
1159
- # eventually being delivered
1160
- # If persistent or one-way request and all usable brokers have failed, try one more time
1161
- # without mandatory flag to give message opportunity to be queued
1162
- # If there are no more usable broker clients, send non-delivery message to original sender
1163
- #
1164
- # === Parameters
1165
- # identity(String):: Identity of broker that could not deliver message
1166
- # reason(String):: Reason for return
1167
- # "NO_ROUTE" - queue does not exist
1168
- # "NO_CONSUMERS" - queue exists but it has no consumers, or if :immediate was specified,
1169
- # all consumers are not immediately ready to consume
1170
- # "ACCESS_REFUSED" - queue not usable because broker is in the process of stopping service
1171
- # message(String):: Returned message in serialized packet format
1172
- # to(String):: Queue to which message was published
1173
- # context(Context):: Message publishing context
1174
- #
1175
- # === Return
1176
- # true:: Always return true
1177
- def handle_return(identity, reason, message, to, context)
1178
- @brokers_hash[identity].update_status(:stopping) if reason == "ACCESS_REFUSED"
1179
-
1180
- if context
1181
- @returns.update("#{alias_(identity)} (#{reason.to_s.downcase})")
1182
- name = context.name
1183
- options = context.options || {}
1184
- token = context.token
1185
- one_way = context.one_way
1186
- persistent = options[:persistent]
1187
- mandatory = true
1188
- remaining = (context.brokers - context.failed) & all_connected
1189
- Log.info("RETURN reason #{reason} token <#{token}> to #{to} from #{context.from} brokers #{context.brokers.inspect} " +
1190
- "failed #{context.failed.inspect} remaining #{remaining.inspect} connected #{all_connected.inspect}")
1191
- if remaining.empty?
1192
- if (persistent || one_way) &&
1193
- ["ACCESS_REFUSED", "NO_CONSUMERS"].include?(reason) &&
1194
- !(remaining = context.brokers & all_connected).empty?
1195
- # Retry because persistent, and this time w/o mandatory so that gets queued even though no consumers
1196
- mandatory = false
1197
- else
1198
- t = token ? " <#{token}>" : ""
1199
- Log.info("NO ROUTE #{aliases(context.brokers).join(", ")} [#{name}]#{t} to #{to}")
1200
- @non_delivery.call(reason, context.type, token, context.from, to) if @non_delivery
1201
- end
1202
- end
1203
-
1204
- unless remaining.empty?
1205
- t = token ? " <#{token}>" : ""
1206
- p = persistent ? ", persistent" : ""
1207
- m = mandatory ? ", mandatory" : ""
1208
- Log.info("RE-ROUTE #{aliases(remaining).join(", ")} [#{context.name}]#{t} to #{to}#{p}#{m}")
1209
- exchange = {:type => :queue, :name => to, :options => {:no_declare => true}}
1210
- publish(exchange, message, options.merge(:no_serialize => true, :brokers => remaining,
1211
- :persistent => persistent, :mandatory => mandatory))
1212
- end
1213
- else
1214
- @returns.update("#{alias_(identity)} (#{reason.to_s.downcase} - missing context)")
1215
- Log.info("Dropping message returned from broker #{identity} for reason #{reason} " +
1216
- "because no message context available for re-routing it to #{to}")
1217
- end
1218
- true
1219
- rescue Exception => e
1220
- Log.error("Failed to handle #{reason} return from #{identity} for message being routed to #{to}", e, :trace)
1221
- @exceptions.track("return", e)
1222
- end
1223
-
1224
- # Helper for deferring block execution until specified number of actions have completed
1225
- # or timeout occurs
1226
- class CountedDeferrable
1227
-
1228
- include EM::Deferrable
1229
-
1230
- # Defer action until completion count reached or timeout occurs
1231
- #
1232
- # === Parameter
1233
- # count(Integer):: Number of completions required for action
1234
- # timeout(Integer|nil):: Number of seconds to wait for all completions and if
1235
- # reached, proceed with action; nil means no timing
1236
- def initialize(count, timeout = nil)
1237
- @timer = EM::Timer.new(timeout) { succeed } if timeout
1238
- @count = count
1239
- end
1240
-
1241
- # Completed one part of task
1242
- #
1243
- # === Return
1244
- # true:: Always return true
1245
- def completed_one
1246
- if (@count -= 1) == 0
1247
- @timer.cancel if @timer
1248
- succeed
1249
- end
1250
- true
1251
- end
1252
-
1253
- end # CountedDeferrable
1254
-
1255
- # Cache for context of recently published messages for use with message returns
1256
- # Applies LRU for managing cache size but only deletes entries when old enough
1257
- class Published
1258
-
1259
- # Number of seconds since a cache entry was last used before it is deleted
1260
- MAX_AGE = 30
1261
-
1262
- # Initialize cache
1263
- def initialize
1264
- @cache = {}
1265
- @lru = []
1266
- end
1267
-
1268
- # Store message context in cache
1269
- #
1270
- # === Parameters
1271
- # message(String):: Serialized message that was published
1272
- # context(Context):: Message publishing context
1273
- #
1274
- # === Return
1275
- # true:: Always return true
1276
- def store(message, context)
1277
- key = identify(message)
1278
- now = Time.now.to_i
1279
- if entry = @cache[key]
1280
- entry[0] = now
1281
- @lru.push(@lru.delete(key))
1282
- else
1283
- @cache[key] = [now, context]
1284
- @lru.push(key)
1285
- @cache.delete(@lru.shift) while (now - @cache[@lru.first][0]) > MAX_AGE
1286
- end
1287
- true
1288
- end
1289
-
1290
- # Fetch context of previously published message
1291
- #
1292
- # === Parameters
1293
- # message(String):: Serialized message that was published
1294
- #
1295
- # === Return
1296
- # (Context|nil):: Context of message, or nil if not found in cache
1297
- def fetch(message)
1298
- key = identify(message)
1299
- if entry = @cache[key]
1300
- entry[0] = Time.now.to_i
1301
- @lru.push(@lru.delete(key))
1302
- entry[1]
1303
- end
1304
- end
1305
-
1306
- # Obtain a unique identifier for this message
1307
- #
1308
- # === Parameters
1309
- # message(String):: Serialized message that was published
1310
- #
1311
- # === Returns
1312
- # (String):: Unique id for message
1313
- def identify(message)
1314
- # If possible use significant part of serialized signature without decoding the message,
1315
- # otherwise use entire serialized message
1316
- if s = (message =~ /signature/)
1317
- message[s, 1000]
1318
- else
1319
- message
1320
- end
1321
- end
1322
-
1323
- end # Published
1324
-
1325
- end # HABrokerClient
1326
-
1327
- end # RightScale